diff --git a/.cursor/rules/sdk/docs/kv-cache-system.mdc b/.cursor/rules/sdk/docs/kv-cache-system.mdc index 4c300113ce..cb1bf247ee 100644 --- a/.cursor/rules/sdk/docs/kv-cache-system.mdc +++ b/.cursor/rules/sdk/docs/kv-cache-system.mdc @@ -87,7 +87,7 @@ When a new cache key is used for the first time: - `cacheKey` — User-provided or auto-generated session identifier - `modelId` — Unique identifier for the loaded model instance -- `configHash` — Hash of system prompt + tool names (ensures cache validity) +- `configHash` — Hash of system prompt + complete canonical tool definitions (ensures cache validity) ### Auto-Cache Retention @@ -177,7 +177,7 @@ Injecting the prime via closure keeps `kv-cache-session.ts` free of model-regist ### Config Hash Generation (generateConfigHash) -Cache validity is tied to a SHA-256 hash of system prompt content + sorted tool names. Model config is NOT included (per addon team: doesn't affect cache validity). Changing tools mid-session creates a new cache with the new tools anchored. Dynamic-mode tools intentionally do NOT participate in the hash so the cache can survive per-turn tool sets. +Cache validity is tied to a SHA-256 hash of the system prompt content + complete canonical tool definitions. Object keys are sorted recursively to avoid cache misses caused only by insertion order; tool-array order is preserved because it matches the prompt sent to the model. Model config is NOT included (per addon team: doesn't affect cache validity). Changing any prompt-affecting tool field mid-session creates a new cache with the new tools anchored. ### Auto-Cache Rename Flow @@ -233,7 +233,7 @@ Enable with `loggerLevel: "debug"` in config. Logs from `cache-logger.ts` show: ## MCP Compatibility -KV Cache works with MCP tools. Only tool names are hashed (sorted alphabetically) — tool order, descriptions, and parameter changes don't affect cache key. Adding/removing tools creates a new cache. +KV Cache works with MCP tools. Complete canonical tool definitions are hashed: object-key insertion order does not affect the cache key, while tool-array order and changes to names, descriptions, or parameters create a new cache. ## Common Issues diff --git a/.cursor/rules/sdk/public-constants-contract.mdc b/.cursor/rules/sdk/public-constants-contract.mdc index 2a23af2df1..874f8cc21d 100644 --- a/.cursor/rules/sdk/public-constants-contract.mdc +++ b/.cursor/rules/sdk/public-constants-contract.mdc @@ -12,7 +12,7 @@ alwaysApply: false `packages/sdk/index.ts` re-exports plain constants alongside RPC methods — `ModelType`, `MODEL_TYPES`, `PLUGIN_LLM` and friends, `SUPPORTED_AUDIO_FORMATS`, -`TOOLS_MODE`, `VERBOSITY`. Non-JS SDKs (Python, ...) are generated from +`VERBOSITY`. Non-JS SDKs (Python, ...) are generated from `packages/sdk/contract/{schema.json,manifest.json,models.json}` — they never read `index.ts` directly. A constant that isn't in `schema.json`'s `$defs` is invisible to every other language, no matter how prominently it's diff --git a/packages/inference/package.json b/packages/inference/package.json index c86dadaa62..778a40b280 100644 --- a/packages/inference/package.json +++ b/packages/inference/package.json @@ -193,7 +193,7 @@ "@qvac/diffusion-cpp": "^0.17.0", "@qvac/embed-llamacpp": "^0.30.1", "@qvac/langdetect-text": "^0.1.2", - "@qvac/llm-llamacpp": "^0.39.3", + "@qvac/llm-llamacpp": "^0.43.0", "@qvac/ocr-ggml": "^0.13.1", "@qvac/translation-nmtcpp": "^8.3.0", "@qvac/tts-ggml": "^0.6.0", @@ -249,7 +249,7 @@ "@qvac/diffusion-cpp": "^0.17.0", "@qvac/embed-llamacpp": "^0.30.1", "@qvac/langdetect-text": "^0.1.2", - "@qvac/llm-llamacpp": "^0.39.3", + "@qvac/llm-llamacpp": "^0.43.0", "@qvac/ocr-ggml": "^0.13.1", "@qvac/translation-nmtcpp": "^8.3.0", "@qvac/tts-ggml": "^0.6.0", diff --git a/packages/inference/src/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.ts b/packages/inference/src/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.ts index 3ce3b28061..f0f874a995 100644 --- a/packages/inference/src/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.ts +++ b/packages/inference/src/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.ts @@ -5,7 +5,6 @@ import type { ResponseFormat, Tool } from '@/schemas/index' -import { TOOLS_MODE } from '@/schemas/tools' import { getModel, getModelConfig, type AnyModel } from '@/runtime/model-registry' import type { DisposableScope } from '@/runtime/disposable-scope' import type { Logger } from '@/logging/types' @@ -19,7 +18,7 @@ import { type CompletionGenerationParams } from '@/plugins/builtin/llamacpp-completion/ops/completion-stream' import { normalizeCompletionStats } from '@/plugins/builtin/llamacpp-completion/ops/completion-stats' -import { appendToolsToHistory, prependToolsToHistory } from '@/utils/tool-integration' +import { prependToolsToHistory } from '@/utils/tool-integration' const logger = getEngineLogger() @@ -69,7 +68,6 @@ type BatchModelStreamResult = { type BatchPromptRenderOptions = { toolsEnabled: boolean - toolsMode?: string | undefined } function runBatchModel(model: AnyModel, prompts: AddonBatchPrompt[]) { @@ -104,10 +102,7 @@ function renderPromptHistory( let historyWithTools: Array = prompt.history if (tools) { - historyWithTools = - options.toolsMode === TOOLS_MODE.dynamic - ? appendToolsToHistory(prompt.history, tools) - : prependToolsToHistory(prompt.history, tools) + historyWithTools = prependToolsToHistory(prompt.history, tools) } // Uses the same attachment expansion as single completion: each @@ -160,8 +155,7 @@ export async function* batchCompletion( const model = getModel(modelId) const modelConfig = getModelConfig(modelId) const renderOptions: BatchPromptRenderOptions = { - toolsEnabled: (modelConfig as { tools?: boolean }).tools === true, - toolsMode: (modelConfig as { toolsMode?: string }).toolsMode + toolsEnabled: (modelConfig as { tools?: boolean }).tools === true } const onAbort = () => { diff --git a/packages/inference/src/plugins/builtin/llamacpp-completion/ops/completion-stream.ts b/packages/inference/src/plugins/builtin/llamacpp-completion/ops/completion-stream.ts index dbefec7b22..d764f9111f 100644 --- a/packages/inference/src/plugins/builtin/llamacpp-completion/ops/completion-stream.ts +++ b/packages/inference/src/plugins/builtin/llamacpp-completion/ops/completion-stream.ts @@ -9,7 +9,6 @@ import type { ToolCall, ToolDialect } from '@/schemas/index' -import { TOOLS_MODE } from '@/schemas/tools' import { logCacheDisabled, logCacheInit, @@ -29,11 +28,7 @@ import { type TurnHandle } from '@/plugins/builtin/llamacpp-completion/ops/kv-cache-session' import type { DisposableScope } from '@/runtime/disposable-scope' -import { - appendToolsToHistory, - detectToolDialect, - prependToolsToHistory -} from '@/utils/tool-integration' +import { detectToolDialect, prependToolsToHistory } from '@/utils/tool-integration' import { parseToolCalls } from '@/utils/tools/index' import { getResponseFormatJsonSchema } from '@/utils/response-format' import { buildAutoCacheSaveHistory, type CacheMessage } from '@/utils/index' @@ -205,30 +200,19 @@ type HistoryMsg = { attachments?: { path: string }[] | undefined } -type ToolPlacement = 'static' | 'dynamic' - /** - * Attach the tool block to a turn payload at the position its placement - * requires. - * - * Static mirrors the no-kv-cache path (`prependToolsToHistory`) and keeps the - * block ahead of the conversation. Dynamic must leave it immediately after the - * last anchor message, which is what the addon's `ToolsCompactController` - * validates before it will anchor and later trim the block. + * Attach the tool block ahead of a turn payload, mirroring the no-kv-cache + * path (`prependToolsToHistory`). */ -function withToolBlock( - messages: ChatHistory[], - toolBlock: ChatHistory[], - placement: ToolPlacement -): ChatHistory[] { +function withToolBlock(messages: ChatHistory[], toolBlock: ChatHistory[]): ChatHistory[] { if (toolBlock.length === 0) return messages - return placement === 'static' ? [...toolBlock, ...messages] : [...messages, ...toolBlock] + return [...toolBlock, ...messages] } interface CachePayload { messages: ChatHistory[] /** - * Whether the prefix will hold a rendered static tool block once this turn + * Whether the prefix will hold a rendered tool block once this turn * commits — either it already did, or this payload carries one the template * will render. */ @@ -251,12 +235,10 @@ function rendersToolBlock(messages: HistoryMsg[], toolBlock: ChatHistory[]): boo /** * Pick the messages that need to reach the model for the next turn. * - * `placement` selects both the slicing strategy and where the tool block sits - * in the payload. Tools are never baked into the primed prefix — a prefix with - * no user turn is not a renderable conversation for every template — so they - * travel with a turn instead. + * Tools are never baked into the primed prefix — a prefix with no user turn is + * not a renderable conversation for every template — so they travel with a + * turn instead. * - * Static placement: * - Empty history: nothing to slice; send whatever non-system messages * exist. (The call site always reports the cache as existing, so this * is the only way into this branch.) @@ -269,20 +251,6 @@ function rendersToolBlock(messages: HistoryMsg[], toolBlock: ChatHistory[]): boo * the bad boundary doesn't propagate into the next turn. * - The tool block travels only with the turn that writes it into the * cache; see `skipToolBlock` below. - * - * Dynamic placement: - * - The addon anchors the tool block after the last user message and - * trims tools + the assistant's tool-call output from the cache once - * the chain resolves. After that trim, the cache only holds messages - * up to the last user turn, so we ship the right slice - * plus the (possibly new) tool set: - * * tool-chain continuation (last role is "tool"): send the trailing - * consecutive tool messages, no tool block — tools are still - * anchored in the cache from the previous round. - * * new user turn after a chain (prev role is "assistant"): send - * [assistant, user] so the model sees its own final reply before - * the new prompt, then re-anchor the tool block. - * * otherwise: send just the last message + tool block. */ function prepareMessagesForCache( session: KvCacheSession, @@ -290,7 +258,6 @@ function prepareMessagesForCache( cacheExists: boolean, history: HistoryMsg[], tools?: Tool[], - placement: ToolPlacement = 'static', toolBlockEvictable = false ): CachePayload { const toolBlock = tools?.length ? transformMessages(tools) : [] @@ -298,76 +265,43 @@ function prepareMessagesForCache( if (!(cacheExists && history.length > 0)) { const historyWithoutSystem = history.filter((msg) => msg.role !== 'system') return { - messages: withToolBlock(transformMessages(historyWithoutSystem), toolBlock, placement), - toolBlockCached: placement === 'static' && rendersToolBlock(historyWithoutSystem, toolBlock) + messages: withToolBlock(transformMessages(historyWithoutSystem), toolBlock), + toolBlockCached: rendersToolBlock(historyWithoutSystem, toolBlock) } } - if (placement === 'static') { - // Static path — slice from the turn's `savedCount` so callers can - // stage multiple messages between completions. `decideCachedHistorySlice` - // also guards against the QVAC-17780 stale-count regression: if the - // saved boundary would slice the history down to an empty payload - // (e.g. after a cancelled mid-decode), it falls back to the full - // non-system history and signals the caller to drop the bad entry. - // The session owns the entry; `dropStaleSavedCount` clears it - // without touching the on-disk file (the file is still trustworthy - // — only the boundary count is wrong). - const { messages, clearStaleCount } = decideCachedHistorySlice( - turn.savedCount, - cacheExists, - history - ) - - if (clearStaleCount) { - session.dropStaleSavedCount(turn) - } - - // Static never trims the block back out of the cache, so re-sending it - // every turn would leave one copy per turn and grow the prefix with the - // conversation. Skip it only when the prefix is known to hold a rendered - // one: `toolBlockCached` records that a previous turn actually got it into - // the cache, which a committed message count does not prove. A stale - // boundary means we are resending the whole conversation anyway, and an - // evictable block can no longer be assumed present. - const skipToolBlock = turn.toolBlockCached && !clearStaleCount && !toolBlockEvictable - const blockToSend = skipToolBlock ? [] : toolBlock + // Slice from the turn's `savedCount` so callers can + // stage multiple messages between completions. `decideCachedHistorySlice` + // also guards against the QVAC-17780 stale-count regression: if the + // saved boundary would slice the history down to an empty payload + // (e.g. after a cancelled mid-decode), it falls back to the full + // non-system history and signals the caller to drop the bad entry. + // The session owns the entry; `dropStaleSavedCount` clears it + // without touching the on-disk file (the file is still trustworthy + // — only the boundary count is wrong). + const { messages, clearStaleCount } = decideCachedHistorySlice( + turn.savedCount, + cacheExists, + history + ) - return { - messages: withToolBlock(transformMessages(messages), blockToSend, placement), - toolBlockCached: skipToolBlock || rendersToolBlock(messages, blockToSend) - } + if (clearStaleCount) { + session.dropStaleSavedCount(turn) } - // Dynamic path. The addon trimmed tools after the previous round, so the - // cache no longer holds the saved-count we'd rely on for slicing — pick - // the right fragment based on the role of the last history message. Nothing - // tool-specific survives that trim, so the prefix never counts as holding a - // block and every turn re-anchors its own. - const lastMsg = history[history.length - 1]! - - if (lastMsg.role === 'tool') { - const trailingTools: HistoryMsg[] = [] - for (let i = history.length - 1; i >= 0; i--) { - const msg = history[i]! - if (msg.role !== 'tool') break - trailingTools.unshift(msg) - } - return { messages: transformMessages(trailingTools), toolBlockCached: false } - } - - if (lastMsg.role === 'user') { - const prevMsg = history[history.length - 2] - const tail = prevMsg?.role === 'assistant' ? [prevMsg, lastMsg] : [lastMsg] - return { - messages: withToolBlock(transformMessages(tail), toolBlock, placement), - toolBlockCached: false - } - } + // The block is never trimmed back out of the cache, so re-sending it every + // turn would leave one copy per turn and grow the prefix with the + // conversation. Skip it only when the prefix is known to hold a rendered + // one: `toolBlockCached` records that a previous turn actually got it into + // the cache, which a committed message count does not prove. A stale + // boundary means we are resending the whole conversation anyway, and an + // evictable block can no longer be assumed present. + const skipToolBlock = turn.toolBlockCached && !clearStaleCount && !toolBlockEvictable + const blockToSend = skipToolBlock ? [] : toolBlock return { - messages: withToolBlock(transformMessages([lastMsg]), toolBlock, placement), - toolBlockCached: false + messages: withToolBlock(transformMessages(messages), blockToSend), + toolBlockCached: skipToolBlock || rendersToolBlock(messages, blockToSend) } } @@ -458,16 +392,12 @@ export async function* completion( const modelConfig = getModelConfig(modelId) const toolsEnabled = (modelConfig as { tools?: boolean }).tools === true - const toolsMode = (modelConfig as { toolsMode?: string }).toolsMode const toolsActive = !!tools?.length && toolsEnabled - const dynamicTools = toolsActive && toolsMode === TOOLS_MODE.dynamic - const staticTools = toolsActive && !dynamicTools // Sliding is opt-in (`n_discarded` defaults to 0). Once on, the addon's - // discard window opens at the end of the primed prefix — which is where a - // static tool block sits, since the prime is the system prompt alone — and - // the clamp that would protect it only runs in dynamic mode. So while - // sliding is possible the block cannot be assumed to survive, and it has to - // travel with every turn. + // discard window opens at the end of the primed prefix — which is where the + // tool block sits, since the prime is the system prompt alone — and nothing + // protects it. So while sliding is possible the block cannot be assumed to + // survive, and it has to travel with every turn. const toolBlockEvictable = ((modelConfig as { n_discarded?: number }).n_discarded ?? 0) > 0 const dialect = @@ -536,10 +466,8 @@ export async function* completion( if (!kvCache) { // KV-cache disabled — straight passthrough, no session involvement. let historyWithTools: Array = history - if (staticTools && tools) { + if (toolsActive && tools) { historyWithTools = prependToolsToHistory(history, tools) - } else if (dynamicTools && tools) { - historyWithTools = appendToolsToHistory(history, tools) } const transformedHistory = transformMessages(historyWithTools) @@ -564,11 +492,10 @@ export async function* completion( const session = createKvCacheSession(modelId, { logger: requestLogger }) const systemPromptFromHistory = extractSystemPrompt(history) - // Static bakes the tool block into the cache on the turn that first sends it - // and never trims it, so a late or changed tool set has to land on a fresh - // cache rather than a warm prefix holding the old block. Dynamic trims its - // block after each chain, so nothing tool-specific survives in its cache. - const configHash = generateConfigHash(systemPromptFromHistory, staticTools ? tools : undefined) + // The tool block is baked into the cache on the turn that first sends it and + // never trimmed, so a late or changed tool set has to land on a fresh cache + // rather than a warm prefix holding the old block. + const configHash = generateConfigHash(systemPromptFromHistory, toolsActive ? tools : undefined) const systemPromptToUse = systemPromptFromHistory || @@ -622,7 +549,6 @@ export async function* completion( /* cacheExists */ true, history, toolsActive ? tools : undefined, - dynamicTools ? 'dynamic' : 'static', toolBlockEvictable ) const messagesToSend = payload.messages diff --git a/packages/inference/src/plugins/builtin/llamacpp-completion/ops/kv-cache-session.ts b/packages/inference/src/plugins/builtin/llamacpp-completion/ops/kv-cache-session.ts index 0730919153..bdaf30209c 100644 --- a/packages/inference/src/plugins/builtin/llamacpp-completion/ops/kv-cache-session.ts +++ b/packages/inference/src/plugins/builtin/llamacpp-completion/ops/kv-cache-session.ts @@ -219,7 +219,7 @@ export interface BeginCustomTurnInput { kind: 'custom' /** User-provided session key (`completion({ kvCache: "session-a" })`). */ customKey: string - /** Hash of system prompt + (static) tool names. */ + /** Hash of system prompt + complete tool definitions. */ configHash: string /** * Prime the cache by sending the system prompt to the addon. Tools are not @@ -233,7 +233,7 @@ export interface BeginCustomTurnInput { export interface BeginAutoTurnInput { kind: 'auto' - /** Hash of system prompt + (static) tool names. */ + /** Hash of system prompt + complete tool definitions. */ configHash: string /** Conversation history used to compute the pre-response cache key. */ history: CacheMessage[] diff --git a/packages/inference/src/plugins/builtin/llamacpp-completion/transform.ts b/packages/inference/src/plugins/builtin/llamacpp-completion/transform.ts index 06756ad0c6..e3a6f86f46 100644 --- a/packages/inference/src/plugins/builtin/llamacpp-completion/transform.ts +++ b/packages/inference/src/plugins/builtin/llamacpp-completion/transform.ts @@ -1,4 +1,4 @@ -import { TOOLS_MODE, type LlmConfig } from '@/schemas/index' +import { type LlmConfig } from '@/schemas/index' /** * Converts an LlmConfig into the flat string-keyed map the C++ addon expects. @@ -35,12 +35,5 @@ export function transformLlmConfig(llmConfig: LlmConfig) { delete transformed['opencl_cache_dir'] } - if ('tools_mode' in transformed) { - if (transformed['tools_mode'] === TOOLS_MODE.dynamic) { - transformed['tools_compact'] = 'true' - } - delete transformed['tools_mode'] - } - return transformed } diff --git a/packages/inference/src/plugins/ops/kv-cache-utils.ts b/packages/inference/src/plugins/ops/kv-cache-utils.ts index 8bb46e6233..69403248e2 100644 --- a/packages/inference/src/plugins/ops/kv-cache-utils.ts +++ b/packages/inference/src/plugins/ops/kv-cache-utils.ts @@ -23,26 +23,30 @@ export function extractSystemPrompt(messages: CacheMessage[]): string | null { return systemMessage ? systemMessage.content : null } -interface ToolLike { - name: string -} - -function getToolNamesForHash(tools: unknown): string[] { - if (!Array.isArray(tools)) return [] - return (tools as ToolLike[]) - .map((t) => t.name) - .filter((n) => typeof n === 'string') - .sort() +// Cache hash based on the system prompt + complete tool definitions. +// Callers pass tools only when the tool block is written into the cache and +// left there, so a different tool set gets its own cache instead of reusing a +// prefix that holds the old block. Every prompt-affecting field participates, +// not just the name: canonical serialization avoids cache misses caused only +// by object-key insertion order, while tool-array order is preserved because +// that is the order sent to the model. +function canonicalizeHashInput(value: unknown): unknown { + if (Array.isArray(value)) return value.map(canonicalizeHashInput) + if (typeof value !== 'object' || value === null) return value + + const canonical: Record = {} + for (const key of Object.keys(value).sort()) { + canonical[key] = canonicalizeHashInput((value as Record)[key]) + } + return canonical } -// Cache hash based on system prompt + tool names. -// Callers pass tools only when the tool block is written into the cache and -// left there (static placement), so a different tool set gets its own cache -// instead of reusing a prefix that holds the old block. export function generateConfigHash(systemPrompt: string | null, tools?: unknown): string { const hash = crypto.createHash('sha-256') - const toolNames = getToolNamesForHash(tools) - hash.update(Buffer.from(JSON.stringify({ systemPrompt, toolNames }), 'utf8')) + const canonicalConfig = JSON.stringify( + canonicalizeHashInput({ systemPrompt, tools: Array.isArray(tools) ? tools : [] }) + ) + hash.update(Buffer.from(canonicalConfig, 'utf8')) return hash.digest('hex').substring(0, 16) } diff --git a/packages/inference/src/schemas/index.ts b/packages/inference/src/schemas/index.ts index 2ccb4a4aa6..566e6c601d 100644 --- a/packages/inference/src/schemas/index.ts +++ b/packages/inference/src/schemas/index.ts @@ -8,12 +8,10 @@ export { toolSchema, toolCallSchema, toolCallErrorSchema, - TOOLS_MODE, type Tool, type ToolCall, type ToolCallError, - type ToolCallWithCall, - type ToolsMode + type ToolCallWithCall } from '@/schemas/tools' export * from '@/schemas/delegate' export * from '@/schemas/model-ops' diff --git a/packages/inference/src/schemas/llamacpp-config.ts b/packages/inference/src/schemas/llamacpp-config.ts index 14b736e740..5188c76d5f 100644 --- a/packages/inference/src/schemas/llamacpp-config.ts +++ b/packages/inference/src/schemas/llamacpp-config.ts @@ -1,6 +1,5 @@ import { z } from 'zod' import { modelSrcInputSchema } from '@/schemas/model-src-utils' -import { TOOLS_MODE } from '@/schemas/tools' /** * Upper bound for `reasoning_budget`. Mirrors the llm-llamacpp addon, which @@ -51,12 +50,6 @@ export const llmConfigBaseSchema = z.object({ */ parallel: z.number().int().min(1).optional(), tools: z.boolean().optional(), - toolsMode: z - .enum([TOOLS_MODE.static, TOOLS_MODE.dynamic]) - .describe( - 'Controls tool placement in the prompt. "static" (default) prepends the tool set once and reuses it across the session. "dynamic" anchors tools after the last user message and trims them from the kv-cache after the chain resolves so each user prompt can carry its own tools.' - ) - .optional(), 'cache-type-k': z.string().optional(), 'cache-type-v': z.string().optional(), 'main-gpu': z.union([z.number().int().min(0), z.enum(['integrated', 'dedicated'])]).optional(), diff --git a/packages/inference/src/schemas/tools.ts b/packages/inference/src/schemas/tools.ts index 18df8733a2..9a752fa3c9 100644 --- a/packages/inference/src/schemas/tools.ts +++ b/packages/inference/src/schemas/tools.ts @@ -1,23 +1,5 @@ import { z } from 'zod' -/** - * `static` (default) — tools are prepended once after the system message and - * shared across the chat session. - * `dynamic` — tools are anchored after the last user message and trimmed - * from the kv-cache once the tool-call chain resolves, so each user prompt - * can carry its own tool set without poisoning the cache. - * - * Implementation detail: maps to the addon's `tools_compact` boolean. We - * use the higher-level `static`/`dynamic` naming so the addon-side - * mapping can change without breaking the public API. - */ -export const TOOLS_MODE = { - static: 'static', - dynamic: 'dynamic' -} as const - -export type ToolsMode = (typeof TOOLS_MODE)[keyof typeof TOOLS_MODE] - const jsonSchemaEnumValueSchema = z.union([z.string(), z.number(), z.boolean(), z.null()]) export const toolSchema = z.object({ diff --git a/packages/inference/src/surface.ts b/packages/inference/src/surface.ts index f688e3de90..585a118701 100644 --- a/packages/inference/src/surface.ts +++ b/packages/inference/src/surface.ts @@ -82,8 +82,6 @@ export { type LoadedInstance, type CacheFileInfo, toolSchema, - TOOLS_MODE, - type ToolsMode, type McpClient, type McpClientInput, type OCRClientParams, diff --git a/packages/inference/src/utils/tool-integration.ts b/packages/inference/src/utils/tool-integration.ts index 875cf16703..f116a9dc32 100644 --- a/packages/inference/src/utils/tool-integration.ts +++ b/packages/inference/src/utils/tool-integration.ts @@ -9,9 +9,9 @@ interface HistoryMessage { } /** - * Static tools mode: prepend tools right after the system message (or at the - * very start when no system message is present). The tool block stays in the - * kv-cache for the whole chat session. + * Prepend tools right after the system message (or at the very start when no + * system message is present). The tool block stays in the kv-cache for the + * whole chat session. */ export function prependToolsToHistory( history: HistoryMessage[], @@ -26,19 +26,6 @@ export function prependToolsToHistory( return [...tools, ...history] } -/** - * Dynamic tools mode: append tools after the last history message. The - * addon's compact-tools mode anchors the block after the last user message - * and trims it from the kv-cache once the tool-call chain resolves, so a - * subsequent turn can ship a different tool set without poisoning the cache. - */ -export function appendToolsToHistory( - history: HistoryMessage[], - tools: Tool[] -): Array { - return [...history, ...tools] -} - export function detectToolDialect(modelId: string): ToolDialect { const info = getModelInfo(modelId) if (!info) return 'hermes' diff --git a/packages/inference/test/completion-kvcache-tools.test.ts b/packages/inference/test/completion-kvcache-tools.test.ts index 43df81685e..53fc658518 100644 --- a/packages/inference/test/completion-kvcache-tools.test.ts +++ b/packages/inference/test/completion-kvcache-tools.test.ts @@ -330,9 +330,8 @@ test('completion: kv-cache resends the tool block after a turn that could not re // With `n_discarded > 0` the addon may slide its context window, and the // discard region opens exactly where a static tool block sits — the protected -// prefix ends at the primed system prompt, and the clamp that would guard the -// block only runs in dynamic mode. While the block can be evicted it has to -// travel with every turn. +// prefix ends at the primed system prompt. While the block can be evicted it +// has to travel with every turn. test('completion: kv-cache resends the tool block when the context window can slide', async (t) => { await setIsolatedHome() clearRegistry() diff --git a/packages/inference/test/kv-cache-session.test.ts b/packages/inference/test/kv-cache-session.test.ts index 781806f5d1..9a10d35297 100644 --- a/packages/inference/test/kv-cache-session.test.ts +++ b/packages/inference/test/kv-cache-session.test.ts @@ -77,6 +77,91 @@ async function loadSession() { return { fs, path, mod, utils, retention, cleanup, writeFakeCache } } +test('generateConfigHash: includes complete canonical tool definitions', async (t) => { + const { mod, cleanup } = await loadSession() + try { + const calculator = { + type: 'function', + name: 'calculator', + description: 'Performs arithmetic', + parameters: { + type: 'object', + properties: { + operation: { type: 'string', enum: ['add', 'subtract'] }, + value: { type: 'number' } + }, + required: ['operation', 'value'] + } + } + const changedSchema = { + ...calculator, + parameters: { + ...calculator.parameters, + properties: { + ...calculator.parameters.properties, + operation: { type: 'string', enum: ['multiply', 'divide'] } + } + } + } + const reorderedKeys = { + parameters: { + required: ['operation', 'value'], + properties: { + value: { type: 'number' }, + operation: { enum: ['add', 'subtract'], type: 'string' } + }, + type: 'object' + }, + description: 'Performs arithmetic', + name: 'calculator', + type: 'function' + } + + const originalHash = mod.generateConfigHash('system prompt', [calculator]) + const changedHash = mod.generateConfigHash('system prompt', [changedSchema]) + const reorderedHash = mod.generateConfigHash('system prompt', [reorderedKeys]) + + t.not(originalHash, changedHash, 'same-named tools with different schemas use different caches') + t.is(originalHash, reorderedHash, 'object-key insertion order does not affect cache identity') + + const other = { ...calculator, name: 'search' } + t.not( + mod.generateConfigHash('system prompt', [calculator, other]), + mod.generateConfigHash('system prompt', [other, calculator]), + 'tool-array order participates in cache identity' + ) + } finally { + cleanup() + } +}) + +// `configHash` is the on-disk `.bin` filename, so the digest of a tool-free +// session is a compatibility surface: any change to the hash payload or its +// serialization renames every plain-chat cache file and re-primes it cold. +// Pinning the shipped digests keeps that a deliberate decision. +test('generateConfigHash: no-tools digests stay pinned', async (t) => { + const { mod, cleanup } = await loadSession() + try { + t.is( + mod.generateConfigHash('you are a helpful assistant.', undefined), + '3f5906d163f40776', + 'omitted tools keep the shipped digest' + ) + t.is( + mod.generateConfigHash('you are a helpful assistant.', []), + '3f5906d163f40776', + 'an empty tool array hashes like omitted tools' + ) + t.is( + mod.generateConfigHash(null, undefined), + '99ba47708d700919', + 'a missing system prompt keeps the shipped digest' + ) + } finally { + cleanup() + } +}) + test('kv-cache-session: beginTurn primes the cache on first use, reuses on second', async (t) => { const { mod, cleanup, writeFakeCache } = await loadSession() try { diff --git a/packages/sdk-python/src/tetherto/qvac_sdk/_generated/__init__.py b/packages/sdk-python/src/tetherto/qvac_sdk/_generated/__init__.py index 01d81460e6..5a622fadd4 100644 --- a/packages/sdk-python/src/tetherto/qvac_sdk/_generated/__init__.py +++ b/packages/sdk-python/src/tetherto/qvac_sdk/_generated/__init__.py @@ -73,7 +73,6 @@ TextToSpeechResponse, TextToSpeechStreamRequest, TextToSpeechStreamResponse, - ToolsMode, TranscribeRequest, TranscribeResponse, TranscribeStreamRequest, @@ -168,7 +167,6 @@ "TextToSpeechResponse", "TextToSpeechStreamRequest", "TextToSpeechStreamResponse", - "ToolsMode", "TranscribeRequest", "TranscribeResponse", "TranscribeStreamRequest", diff --git a/packages/sdk-python/src/tetherto/qvac_sdk/_generated/models/__init__.py b/packages/sdk-python/src/tetherto/qvac_sdk/_generated/models/__init__.py index bc0b3e044a..506d4fac87 100644 --- a/packages/sdk-python/src/tetherto/qvac_sdk/_generated/models/__init__.py +++ b/packages/sdk-python/src/tetherto/qvac_sdk/_generated/models/__init__.py @@ -487,7 +487,6 @@ LoadModelSrcRequestLlamacppCompletionModelConfigProjectionModelSrc, LoadModelSrcRequestLlamacppCompletionModelConfigProjectionModelSrcAddon, LoadModelSrcRequestLlamacppCompletionModelConfigSplitMode, - LoadModelSrcRequestLlamacppCompletionModelConfigToolsMode, LoadModelSrcRequestLlamacppCompletionModelConfigVerbosity, LoadModelSrcRequestLlamacppEmbedding, LoadModelSrcRequestLlamacppEmbeddingDelegate, @@ -712,7 +711,6 @@ TextToSpeechStreamResponse, TextToSpeechStreamResponseStats, Threads, - ToolsMode, TranscribeRequest, TranscribeRequestAudioChunkBase64, TranscribeRequestAudioChunkFilePath, @@ -1242,7 +1240,6 @@ "LoadModelSrcRequestLlamacppCompletionModelConfigProjectionModelSrc", "LoadModelSrcRequestLlamacppCompletionModelConfigProjectionModelSrcAddon", "LoadModelSrcRequestLlamacppCompletionModelConfigSplitMode", - "LoadModelSrcRequestLlamacppCompletionModelConfigToolsMode", "LoadModelSrcRequestLlamacppCompletionModelConfigVerbosity", "LoadModelSrcRequestLlamacppEmbedding", "LoadModelSrcRequestLlamacppEmbeddingDelegate", @@ -1467,7 +1464,6 @@ "TextToSpeechStreamResponse", "TextToSpeechStreamResponseStats", "Threads", - "ToolsMode", "TranscribeRequest", "TranscribeRequestAudioChunkBase64", "TranscribeRequestAudioChunkFilePath", diff --git a/packages/sdk-python/src/tetherto/qvac_sdk/_generated/models/_internal.py b/packages/sdk-python/src/tetherto/qvac_sdk/_generated/models/_internal.py index 43df639fb5..f477e17047 100644 --- a/packages/sdk-python/src/tetherto/qvac_sdk/_generated/models/_internal.py +++ b/packages/sdk-python/src/tetherto/qvac_sdk/_generated/models/_internal.py @@ -2070,11 +2070,6 @@ class SupportedAudioFormat(Enum): raw = ".raw" -class ToolsMode(Enum): - static = "static" - dynamic = "dynamic" - - class Verbosity(Enum): error = 0 warn = 1 @@ -6380,11 +6375,6 @@ class LoadModelSrcRequestLlamacppCompletionModelConfigVerbosity(Enum): number_3 = 3 -class LoadModelSrcRequestLlamacppCompletionModelConfigToolsMode(Enum): - static = "static" - dynamic = "dynamic" - - class MainGpu(RootModel[int]): root: Annotated[int, Field(ge=0, le=9007199254740991)] @@ -6477,14 +6467,6 @@ class LoadModelSrcRequestLlamacppCompletionModelConfig(GeneratedBaseModel): n_discarded: float | None = None parallel: Annotated[int | None, Field(ge=1, le=9007199254740991)] = None tools: bool | None = None - tools_mode: Annotated[ - LoadModelSrcRequestLlamacppCompletionModelConfigToolsMode | None, - Field( - alias="toolsMode", - description='Controls tool placement in the prompt. "static" (default) prepends the tool set once and reuses it across the session. "dynamic" anchors tools after the last user message and trims them from the kv-cache after the chain resolves so each user prompt can carry its own tools.', - title="LoadModelSrcRequestLlamacppCompletionModelConfigToolsMode", - ), - ] = None cache_type_k: Annotated[str | None, Field(alias="cache-type-k")] = None cache_type_v: Annotated[str | None, Field(alias="cache-type-v")] = None main_gpu: Annotated[ diff --git a/packages/sdk/contract/README.md b/packages/sdk/contract/README.md index 6d6763c7cd..5a5688798b 100644 --- a/packages/sdk/contract/README.md +++ b/packages/sdk/contract/README.md @@ -6,7 +6,7 @@ these artifacts. - `schema.json` — JSON Schema (draft 2020-12) for every request and response wire type, plus every public constant registered in `@/schemas/constants- -registry` (`ModelType`, `ToolsMode`, `Verbosity`, `PluginId`, +registry` (`ModelType`, `Verbosity`, `PluginId`, `SupportedAudioFormat`) as its own `constants.` def, tagged with `x-enum-varnames` so codegen preserves the original key names (plain JSON Schema `enum:` only carries values). Requests use the schema input shape, diff --git a/packages/sdk/contract/schema.json b/packages/sdk/contract/schema.json index b0b23febed..89459b18d4 100644 --- a/packages/sdk/contract/schema.json +++ b/packages/sdk/contract/schema.json @@ -2742,12 +2742,6 @@ "enum": [".mp3", ".m4a", ".ogg", ".wav", ".flac", ".aac", ".raw"], "x-enum-varnames": ["MP3", "M4A", "OGG", "WAV", "FLAC", "AAC", "RAW"] }, - "constants.ToolsMode": { - "title": "ToolsMode", - "type": "string", - "enum": ["static", "dynamic"], - "x-enum-varnames": ["static", "dynamic"] - }, "constants.Verbosity": { "title": "Verbosity", "type": "number", @@ -7946,12 +7940,6 @@ "tools": { "type": "boolean" }, - "toolsMode": { - "type": "string", - "enum": ["static", "dynamic"], - "description": "Controls tool placement in the prompt. \"static\" (default) prepends the tool set once and reuses it across the session. \"dynamic\" anchors tools after the last user message and trims them from the kv-cache after the chain resolves so each user prompt can carry its own tools.", - "title": "LoadModelSrcRequestLlamacppCompletionModelConfigToolsMode" - }, "cache-type-k": { "type": "string" }, diff --git a/packages/sdk/e2e/tests/desktop/consumer.ts b/packages/sdk/e2e/tests/desktop/consumer.ts index ede7cd499a..7c39126eca 100644 --- a/packages/sdk/e2e/tests/desktop/consumer.ts +++ b/packages/sdk/e2e/tests/desktop/consumer.ts @@ -165,12 +165,6 @@ resources.define('tools', { config: { ctx_size: 4096, tools: true } }) -resources.define('tools-dynamic', { - constant: QWEN3_1_7B_INST_Q4, - type: 'llamacpp-completion', - config: { ctx_size: 4096, tools: true, toolsMode: 'dynamic' } -}) - resources.define('tools-qwen35', { constant: QWEN3_5_0_8B_MULTIMODAL_Q4_K_M, type: 'llamacpp-completion', diff --git a/packages/sdk/e2e/tests/electron/consumer.ts b/packages/sdk/e2e/tests/electron/consumer.ts index bbbcd07010..2b9144c04c 100644 --- a/packages/sdk/e2e/tests/electron/consumer.ts +++ b/packages/sdk/e2e/tests/electron/consumer.ts @@ -150,12 +150,6 @@ resources.define('tools', { config: { ctx_size: 4096, tools: true } }) -resources.define('tools-dynamic', { - constant: QWEN3_1_7B_INST_Q4, - type: 'llamacpp-completion', - config: { ctx_size: 4096, tools: true, toolsMode: 'dynamic' } -}) - resources.define('tools-qwen35', { constant: QWEN3_5_0_8B_MULTIMODAL_Q4_K_M, type: 'llamacpp-completion', diff --git a/packages/sdk/e2e/tests/kv-cache-tests.ts b/packages/sdk/e2e/tests/kv-cache-tests.ts index fa78395904..d77648ac64 100644 --- a/packages/sdk/e2e/tests/kv-cache-tests.ts +++ b/packages/sdk/e2e/tests/kv-cache-tests.ts @@ -252,48 +252,13 @@ export const kvCacheToolsSequentialSave: TestDefinition = { } ], messages: ['What is 10 + 20?', 'Now what is 5 + 5?'], - stream: true + stream: true, + generationParams: { temp: 0, top_k: 1, seed: 42 } }, expectation: { validation: 'type', expectedType: 'string' }, metadata: { category: 'kv-cache', dependency: 'tools', estimatedDurationMs: 90000 } } -// Dynamic tools mode + custom kvCache key across a multi-round tool chain, -// with a model evict/reload in the middle. No other test covers this -// intersection: `toolsMode: "dynamic"` exercises the per-turn fragment cache -// path (trailing-tool / [assistant,user] slicing in `completion-stream.ts`), -// and evict/reload simulates a model reload after priming (in-memory savedCount -// and addon anchoring cleared, on-disk `.bin` retained). The executor asserts -// that tool calls still parse on cached/reloaded rounds and that the on-disk -// cache is reused (`cacheTokens > 0`). -export const kvCacheToolsDynamicReuse: TestDefinition = { - testId: 'kv-cache-tools-dynamic-reuse', - params: { - cacheKey: 'tools-dynamic-reuse-session', - firstUserMessage: 'What is 10 + 20?', - secondUserMessage: 'Now what is 5 + 5?', - toolResult: '30', - tools: [ - { - type: 'function', - name: 'calculator', - description: 'Performs basic math operations', - parameters: { - type: 'object', - properties: { - operation: { type: 'string', enum: ['add', 'subtract', 'multiply', 'divide'] }, - a: { type: 'number' }, - b: { type: 'number' } - }, - required: ['operation', 'a', 'b'] - } - } - ] - }, - expectation: { validation: 'type', expectedType: 'string' }, - metadata: { category: 'kv-cache', dependency: 'tools-dynamic', estimatedDurationMs: 120000 } -} - export const kvCacheCancelThenNewPrompt: TestDefinition = { testId: 'kv-cache-cancel-then-new-prompt', params: { @@ -330,6 +295,5 @@ export const kvCacheTests = [ kvCacheRemoveThinkingCompaction, kvCacheNoSystemPrompt, kvCacheToolsSequentialSave, - kvCacheToolsDynamicReuse, kvCacheCancelThenNewPrompt ] diff --git a/packages/sdk/e2e/tests/mobile/consumer.ts b/packages/sdk/e2e/tests/mobile/consumer.ts index f1db5146c0..0d52bde85d 100644 --- a/packages/sdk/e2e/tests/mobile/consumer.ts +++ b/packages/sdk/e2e/tests/mobile/consumer.ts @@ -136,12 +136,6 @@ resources.define('tools', { config: { ctx_size: 4096, tools: true } }) -resources.define('tools-dynamic', { - constant: QWEN3_1_7B_INST_Q4, - type: 'llamacpp-completion', - config: { ctx_size: 4096, tools: true, toolsMode: 'dynamic' } -}) - resources.define('ocr', { constant: OCR_LATIN, type: 'ggml-ocr', diff --git a/packages/sdk/e2e/tests/shared/executors/kv-cache-executor.ts b/packages/sdk/e2e/tests/shared/executors/kv-cache-executor.ts index ee73916585..273409dfde 100644 --- a/packages/sdk/e2e/tests/shared/executors/kv-cache-executor.ts +++ b/packages/sdk/e2e/tests/shared/executors/kv-cache-executor.ts @@ -26,8 +26,6 @@ export class KvCacheExecutor extends AbstractModelExecutor return [test.testId, this.removeThinkingCompaction.bind(this)] if (test.testId === 'kv-cache-tools-sequential-save') return [test.testId, this.toolsSequentialSave.bind(this)] - if (test.testId === 'kv-cache-tools-dynamic-reuse') - return [test.testId, this.toolsDynamicReuse.bind(this)] if (test.testId === 'kv-cache-cancel-then-new-prompt') return [test.testId, this.cancelThenNewPrompt.bind(this)] if ( @@ -456,10 +454,24 @@ export class KvCacheExecutor extends AbstractModelExecutor } async toolsSequentialSave( - params: { cacheKey: string; tools: unknown[]; messages: string[]; stream: boolean }, + params: { + cacheKey: string + tools: unknown[] + messages: string[] + stream: boolean + generationParams?: Record + }, expectation: Expectation ): Promise { let toolsModelId = await this.resources.ensureLoaded('tools') + const declaredTools = new Map( + ( + params.tools as Array<{ + name: string + parameters?: { required?: string[] } + }> + ).map((tool) => [tool.name, tool.parameters?.required ?? []]) + ) try { try { @@ -483,7 +495,8 @@ export class KvCacheExecutor extends AbstractModelExecutor history: [...history], stream: true, kvCache: params.cacheKey, - tools: params.tools as never + tools: params.tools as never, + ...(params.generationParams && { generationParams: params.generationParams }) }) let response = '' @@ -491,6 +504,30 @@ export class KvCacheExecutor extends AbstractModelExecutor response += token } + const toolCalls = result.toolCalls ? await result.toolCalls : [] + const declaredCall = toolCalls.find((call) => declaredTools.has(call.name)) + if (!declaredCall) { + return { + passed: false, + output: + `Tool completion ${i + 1} emitted no call matching a declared tool after ` + + `${i === 0 ? 'cache creation' : 'model reload and cache reuse'}. ` + + `Got: [${toolCalls.map((call) => call.name).join(', ')}]` + } + } + + const requiredArgs = declaredTools.get(declaredCall.name) ?? [] + const missingArgs = requiredArgs.filter((key) => !(key in declaredCall.arguments)) + if (missingArgs.length > 0) { + return { + passed: false, + output: + `Tool completion ${i + 1} call '${declaredCall.name}' is missing required ` + + `arguments after ${i === 0 ? 'cache creation' : 'model reload and cache reuse'}: ` + + `${missingArgs.join(', ')}` + } + } + const stats = await result.stats const cacheTokens = ((stats as Record)?.cacheTokens as number) ?? 0 @@ -525,139 +562,4 @@ export class KvCacheExecutor extends AbstractModelExecutor return { passed: false, output: `Tools sequential save failed: ${errorMsg}` } } } - - /** - * Dynamic tools mode (`toolsMode: "dynamic"`) + a custom kvCache key across a - * three-round tool chain, with a model evict/reload after the prime turn. - * - * Covers a combination no other kv-cache test exercises: - * - * - Round 1 (prime): a user prompt must yield a PARSEABLE tool call under - * dynamic mode while the kvCache key is being primed. - * - evict + reload: drops the addon's in-memory KV session and the SDK's - * in-memory `savedCount` / anchoring, leaving only the on-disk `.bin`. - * - Round 2 (continuation, history ends in a `tool` message): exercises the - * dynamic "trailing tool messages" fragment branch. Must REUSE the - * on-disk cache (`cacheTokens > 0`) after the reload, and stay coherent. - * - Round 3 (new prompt, history ends `assistant` then `user`): exercises - * the dynamic "[assistant, user]" fragment branch on a warm cache. Must - * again yield a PARSEABLE tool call — proving cache reuse did not corrupt - * tool parsing. - */ - async toolsDynamicReuse( - params: { - cacheKey: string - tools: unknown[] - firstUserMessage: string - secondUserMessage: string - toolResult: string - }, - expectation: Expectation - ): Promise { - const resourceKey = 'tools-dynamic' - let modelId = await this.resources.ensureLoaded(resourceKey) - - const runTurn = (history: ChatMessage[]) => - callWhenAddonIdle(async () => { - const result = completion({ - modelId, - history, - stream: false, - kvCache: params.cacheKey, - tools: params.tools as never - }) - const text = await result.text - const toolCalls = result.toolCalls - ? ((await result.toolCalls) as Array<{ id: string; name: string }>) - : [] - const stats = (await result.stats) as Record | undefined - const cacheTokens = (stats?.cacheTokens as number) ?? 0 - return { text, toolCalls, cacheTokens } - }) - - try { - try { - await deleteCache({ kvCacheKey: params.cacheKey }) - } catch { - /* ignore ENOENT */ - } - - const system: ChatMessage = { - role: 'system', - content: 'You are a helpful assistant with access to tools. Be brief.' - } - - // ---- Round 1: prime. Expect a parseable tool call under dynamic mode. - const r1History: ChatMessage[] = [system, { role: 'user', content: params.firstUserMessage }] - const r1 = await runTurn(r1History) - if (r1.toolCalls.length === 0) { - return { - passed: false, - output: - `Round 1 (prime) under dynamic mode emitted no parseable tool call. ` + - `Dynamic tool-call format instruction not surfaced, or kvCache prime corrupted the prompt. ` + - `text=${JSON.stringify(r1.text).slice(0, 200)}` - } - } - - // Feed back the assistant tool-call turn + a tool result (standard - // agentic loop). History now ends in a `tool` message. - const r2History: ChatMessage[] = [ - ...r1History, - { role: 'assistant', content: r1.text }, - { - role: 'tool', - content: `[Tool: ${r1.toolCalls[0]!.name} (${r1.toolCalls[0]!.id})]\n${params.toolResult}` - } - ] - - // ---- Evict + reload: clear in-memory KV session, savedCount, anchoring. - // Only the on-disk `.bin` survives — the reload-desync scenario. - await this.resources.evict(resourceKey) - modelId = await this.resources.ensureLoaded(resourceKey) - - // ---- Round 2: continuation on the reloaded cache (trailing-tool branch). - // Must reuse the on-disk cache. - const r2 = await runTurn(r2History) - if (r2.cacheTokens <= 0) { - return { - passed: false, - output: - `Round 2 (post-reload continuation) did not reuse the on-disk dynamic-tools cache: ` + - `cacheTokens=${r2.cacheTokens}. The on-disk cache file was not picked up after reload.` - } - } - - // ---- Round 3: new user prompt after the chain ([assistant, user] branch) - // on a warm cache. Must still yield a parseable tool call. - const r3History: ChatMessage[] = [ - ...r2History, - { role: 'assistant', content: r2.text }, - { role: 'user', content: params.secondUserMessage } - ] - const r3 = await runTurn(r3History) - if (r3.toolCalls.length === 0) { - return { - passed: false, - output: - `Round 3 (new prompt on warm dynamic-tools cache) emitted no parseable tool call — ` + - `cache reuse corrupted tool parsing. r2CacheTokens=${r2.cacheTokens}, ` + - `r3CacheTokens=${r3.cacheTokens}, text=${JSON.stringify(r3.text).slice(0, 200)}` - } - } - - const summary = - `Dynamic tools + kvCache reuse OK [${resourceKey}]: ` + - `r1Calls=${r1.toolCalls.length}, ` + - `r2CacheTokens=${r2.cacheTokens} (post-reload reuse), ` + - `r3Calls=${r3.toolCalls.length} (warm), r3CacheTokens=${r3.cacheTokens}` - // The harness only surfaces `output` on failure, so log the numbers - // explicitly — otherwise a passing run hides the reuse magnitude. - console.log(`[kv-cache-tools-dynamic-reuse] ${summary}`) - return ValidationHelpers.validate(summary, expectation) - } catch (error) { - const errorMsg = error instanceof Error ? error.message : String(error) - return { passed: false, output: `Dynamic tools reuse failed: ${errorMsg}` } - } - } } diff --git a/packages/sdk/e2e/tests/shared/executors/tools-executor.ts b/packages/sdk/e2e/tests/shared/executors/tools-executor.ts index 3f73708694..0f89e5a1b8 100644 --- a/packages/sdk/e2e/tests/shared/executors/tools-executor.ts +++ b/packages/sdk/e2e/tests/shared/executors/tools-executor.ts @@ -20,13 +20,12 @@ export class ToolsExecutor extends AbstractModelExecutor { description: string parameters: Record }> - toolsMode?: 'static' | 'dynamic' toolDialect?: ToolDialect resourceKey?: string stream?: boolean expectedToolCall?: { name: string; argKeys?: string[] } } - const resourceKey = p.resourceKey ?? (p.toolsMode === 'dynamic' ? 'tools-dynamic' : 'tools') + const resourceKey = p.resourceKey ?? 'tools' const toolsModelId = await this.resources.ensureLoaded(resourceKey) try { diff --git a/packages/sdk/e2e/tests/tools-tests.ts b/packages/sdk/e2e/tests/tools-tests.ts index 8a151f1d32..5f47713b5d 100644 --- a/packages/sdk/e2e/tests/tools-tests.ts +++ b/packages/sdk/e2e/tests/tools-tests.ts @@ -21,7 +21,6 @@ const createToolsTest = ( validation: 'type' expectedType: 'string' | 'number' | 'array' } - toolsMode?: 'static' | 'dynamic' toolDialect?: ToolDialect resourceKey?: string suites?: string[] @@ -35,15 +34,13 @@ const createToolsTest = ( validation: 'type' as const, expectedType: 'string' as const } - const dependency = - options.resourceKey ?? (options.toolsMode === 'dynamic' ? 'tools-dynamic' : 'tools') + const dependency = options.resourceKey ?? 'tools' return { testId, params: { history: [{ role: 'user', content: userPrompt }], tools, stream: false, - ...(options.toolsMode && { toolsMode: options.toolsMode }), ...(options.toolDialect && { toolDialect: options.toolDialect }), ...(options.resourceKey && { resourceKey: options.resourceKey }), ...(options.expectedToolCall && { expectedToolCall: options.expectedToolCall }) @@ -96,36 +93,6 @@ export const toolsSimpleFunction = createToolsTest( } ) -export const toolsSimpleFunctionDynamic = createToolsTest( - 'tools-simple-function-dynamic', - "What's 25 degrees Celsius in Fahrenheit?", - [ - { - type: 'function', - name: 'convert_temperature', - description: 'Convert temperature between Celsius and Fahrenheit', - parameters: { - type: 'object', - properties: { - value: { type: 'number', description: 'Temperature value' }, - from_unit: { - type: 'string', - enum: ['celsius', 'fahrenheit'], - description: 'Source unit' - }, - to_unit: { - type: 'string', - enum: ['celsius', 'fahrenheit'], - description: 'Target unit' - } - }, - required: ['value', 'from_unit', 'to_unit'] - } - } - ], - { toolsMode: 'dynamic' } -) - export const toolsMultipleFunctions = createToolsTest( 'tools-multiple-functions', 'Get the weather for London and calculate the time difference with New York', @@ -298,7 +265,6 @@ export const toolsSimpleFunctionGemma4 = createToolsTest( export const toolsTests = [ toolsSimpleFunction, - toolsSimpleFunctionDynamic, toolsMultipleFunctions, toolsSimpleFunctionQwen35, toolsSimpleFunctionGemma4, diff --git a/packages/sdk/examples/llamacpp-dynamic-tools.ts b/packages/sdk/examples/llamacpp-dynamic-tools.ts deleted file mode 100644 index 149771ecb6..0000000000 --- a/packages/sdk/examples/llamacpp-dynamic-tools.ts +++ /dev/null @@ -1,189 +0,0 @@ -/** - * Dynamic-tools mode example. - * - * In `dynamic` mode each user prompt can carry its own tool set: the addon - * anchors tools after the last user message, runs the tool-call chain, then - * trims the tools + chain output from the kv-cache so a later turn can ship - * a different tool list without poisoning the cache. Compare with - * `llamacpp-native-tools.ts`, which uses the default `static` mode where - * one shared tool set lives at the top of the session. - * - * Run with: - * bun run build - * bun run bare:example dist/examples/llamacpp-dynamic-tools.js - */ -import { z } from 'zod' -import { - completion, - loadModel, - unloadModel, - type ToolCall, - type CompletionParams, - type ToolInput, - QWEN3_1_7B_INST_Q4 -} from '@qvac/sdk' - -const weatherSchema = z.object({ - city: z.string().describe('City name') -}) - -const horoscopeSchema = z.object({ - sign: z.string().describe('An astrological sign, e.g. Taurus or Aquarius') -}) - -const dateSchema = z.object({}) - -const toolSchemas = { - get_weather: weatherSchema, - get_horoscope: horoscopeSchema, - get_date: dateSchema -} as const - -const weatherTools: ToolInput[] = [ - { - name: 'get_weather', - description: 'Get current weather for a city', - parameters: weatherSchema - } -] - -const horoscopeTools: ToolInput[] = [ - { - name: 'get_horoscope', - description: "Get today's horoscope for an astrological sign", - parameters: horoscopeSchema - } -] - -const dateTools: ToolInput[] = [ - { - name: 'get_date', - description: "Get today's date", - parameters: dateSchema - } -] - -function executeToolCall(call: ToolCall): string { - if (call.name === 'get_weather') { - const args = call.arguments as { city: string } - return `The weather in ${args.city} is rainy, 8°C with heavy clouds.` - } - if (call.name === 'get_horoscope') { - const args = call.arguments as { sign: string } - return `Horoscope for ${args.sign}: a great day for new beginnings.` - } - if (call.name === 'get_date') { - return new Date().toISOString().slice(0, 10) - } - return `Unknown tool: ${call.name}` -} - -type ChatTurnParams = Pick & { - history: Array<{ role: string; content: string }> - tools: ToolInput[] -} - -async function chatTurn({ modelId, kvCache, history, tools }: ChatTurnParams) { - const result = completion({ - modelId, - history, - tools, - kvCache, - stream: true - }) - - const tokensTask = (async () => { - for await (const token of result.tokenStream) { - process.stdout.write(token) - } - })() - - const toolEventsTask = (async () => { - for await (const evt of result.toolCallStream) { - console.log(`\n▸ tool call: ${evt.call.name}(${JSON.stringify(evt.call.arguments)})`) - } - })() - - await Promise.all([tokensTask, toolEventsTask]) - - const text = await result.text - const toolCalls: ToolCall[] = await result.toolCalls - - if (toolCalls.length === 0) { - history.push({ role: 'assistant', content: text }) - return - } - - for (const call of toolCalls) { - const schema = toolSchemas[call.name as keyof typeof toolSchemas] - if (schema) { - const parsed = schema.safeParse(call.arguments) - if (!parsed.success) { - console.log(`✖ validation failed for ${call.name}:`, parsed.error) - } - } - } - - history.push({ role: 'assistant', content: text }) - for (const call of toolCalls) { - history.push({ role: 'tool', content: executeToolCall(call) }) - } - - // Follow-up turn so the model can incorporate the tool results. - await chatTurn({ modelId, kvCache, history, tools }) -} - -async function main() { - const modelId = await loadModel({ - modelSrc: QWEN3_1_7B_INST_Q4, - modelConfig: { - ctx_size: 4096, - tools: true, - toolsMode: 'dynamic' - }, - onProgress: (p) => { - const mb = (n: number) => (n / 1e6).toFixed(1) - const line = `▸ Downloading ${p.percentage.toFixed(0)}% (${mb(p.downloaded)}/${mb(p.total)} MB)` - process.stderr.write(process.stderr.isTTY ? `\r${line}` : `${line}\n`) - if (p.percentage >= 100) process.stderr.write('\n') - } - }) - console.log(`▸ Model loaded: ${modelId}`) - - const kvCache = `dynamic-tools-${Date.now()}` - const history: Array<{ role: string; content: string }> = [ - { - role: 'system', - content: - 'You are a helpful assistant that uses tools when they are available. ' + - "User's cat is named Windy and dog is named Butch." - } - ] - - // Turn 1 — only weather tools available. - history.push({ role: 'user', content: "What's the weather in Tokyo?" }) - console.log('\n▸ Turn 1 (tools=weather):\n') - await chatTurn({ modelId, kvCache, history, tools: weatherTools }) - - // Turn 2 — same session, swap to horoscope tools. Dynamic mode lets the - // model see a different tool set without invalidating the kv-cache. - history.push({ - role: 'user', - content: 'Now check my horoscope for Aquarius.' - }) - console.log('\n\n▸ Turn 2 (tools=horoscope):\n') - await chatTurn({ modelId, kvCache, history, tools: horoscopeTools }) - - // Turn 3 — swap to a parameterless tool to confirm empty-arg flows work. - history.push({ role: 'user', content: "What's today's date?" }) - console.log('\n\n▸ Turn 3 (tools=date):\n') - await chatTurn({ modelId, kvCache, history, tools: dateTools }) - - console.log('\n\n▸ Done.') - await unloadModel({ modelId, clearStorage: false }) -} - -main().catch((err) => { - console.error('✖', err) - process.exit(1) -}) diff --git a/packages/sdk/index.ts b/packages/sdk/index.ts index f104e438c0..0dd8ba12a6 100644 --- a/packages/sdk/index.ts +++ b/packages/sdk/index.ts @@ -146,8 +146,6 @@ export { type LoadedInstance, type CacheFileInfo, toolSchema, - TOOLS_MODE, - type ToolsMode, type McpClient, type McpClientInput, type OCRClientParams, diff --git a/packages/sdk/package.json b/packages/sdk/package.json index d7c900eb03..c85dc5bb7f 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -215,7 +215,7 @@ "@qvac/embed-llamacpp": "^0.32.0", "@qvac/error": "^0.1.1", "@qvac/langdetect-text": "^0.1.2", - "@qvac/llm-llamacpp": "^0.42.0", + "@qvac/llm-llamacpp": "^0.43.0", "@qvac/logging": "^0.1.0", "@qvac/ocr-ggml": "^0.16.0", "@qvac/rag": "^0.6.4", diff --git a/packages/sdk/schemas/constants-registry.ts b/packages/sdk/schemas/constants-registry.ts index f2848afdd1..902e20da58 100644 --- a/packages/sdk/schemas/constants-registry.ts +++ b/packages/sdk/schemas/constants-registry.ts @@ -1,6 +1,5 @@ import { z } from 'zod' import { ModelType } from './model-types' -import { TOOLS_MODE } from './tools' import { VERBOSITY } from './llamacpp-config' import { PLUGIN_LLM, @@ -43,7 +42,6 @@ import { SUPPORTED_AUDIO_FORMATS } from '@/constants/audio' */ export const constantsRegistry = { ModelType: z.enum(ModelType), - ToolsMode: z.enum(TOOLS_MODE), Verbosity: z.enum(VERBOSITY), PluginId: z.enum({ LLM: PLUGIN_LLM, diff --git a/packages/sdk/schemas/index.ts b/packages/sdk/schemas/index.ts index 20d5dbd355..d4eac5243d 100644 --- a/packages/sdk/schemas/index.ts +++ b/packages/sdk/schemas/index.ts @@ -8,12 +8,10 @@ export { toolSchema, toolCallSchema, toolCallErrorSchema, - TOOLS_MODE, type Tool, type ToolCall, type ToolCallError, - type ToolCallWithCall, - type ToolsMode + type ToolCallWithCall } from './tools' export * from './delegate' export * from './delete-cache' diff --git a/packages/sdk/schemas/llamacpp-config.ts b/packages/sdk/schemas/llamacpp-config.ts index 839d9da03b..3481358ccb 100644 --- a/packages/sdk/schemas/llamacpp-config.ts +++ b/packages/sdk/schemas/llamacpp-config.ts @@ -1,6 +1,5 @@ import { z } from 'zod' import { modelSrcInputSchema } from './model-src-utils' -import { TOOLS_MODE } from './tools' /** * Upper bound for `reasoning_budget`. Mirrors the llm-llamacpp addon, which @@ -51,12 +50,6 @@ export const llmConfigBaseSchema = z.object({ */ parallel: z.number().int().min(1).optional(), tools: z.boolean().optional(), - toolsMode: z - .enum([TOOLS_MODE.static, TOOLS_MODE.dynamic]) - .describe( - 'Controls tool placement in the prompt. "static" (default) prepends the tool set once and reuses it across the session. "dynamic" anchors tools after the last user message and trims them from the kv-cache after the chain resolves so each user prompt can carry its own tools.' - ) - .optional(), 'cache-type-k': z.string().optional(), 'cache-type-v': z.string().optional(), 'main-gpu': z.union([z.number().int().min(0), z.enum(['integrated', 'dedicated'])]).optional(), diff --git a/packages/sdk/schemas/tools.ts b/packages/sdk/schemas/tools.ts index 3f7418e6de..9a752fa3c9 100644 --- a/packages/sdk/schemas/tools.ts +++ b/packages/sdk/schemas/tools.ts @@ -1,23 +1,5 @@ import { z } from 'zod' -/** - * `static` (default) — tools are prepended once after the system message and - * shared across the chat session. - * `dynamic` — tools are anchored after the last user message and trimmed - * from the kv-cache once the tool-call chain resolves, so each user prompt - * can carry its own tool set without poisoning the cache. - * - * Implementation detail: maps to the addon's `tools_compact` boolean. The - * SDK uses the higher-level `static`/`dynamic` naming so the addon-side - * mapping can change without breaking the public API. - */ -export const TOOLS_MODE = { - static: 'static', - dynamic: 'dynamic' -} as const - -export type ToolsMode = (typeof TOOLS_MODE)[keyof typeof TOOLS_MODE] - const jsonSchemaEnumValueSchema = z.union([z.string(), z.number(), z.boolean(), z.null()]) export const toolSchema = z.object({ diff --git a/packages/sdk/server/bare/ops/kv-cache-utils.ts b/packages/sdk/server/bare/ops/kv-cache-utils.ts index fec22a1b44..e720520f95 100644 --- a/packages/sdk/server/bare/ops/kv-cache-utils.ts +++ b/packages/sdk/server/bare/ops/kv-cache-utils.ts @@ -23,26 +23,30 @@ export function extractSystemPrompt(messages: CacheMessage[]): string | null { return systemMessage ? systemMessage.content : null } -interface ToolLike { - name: string -} - -function getToolNamesForHash(tools: unknown): string[] { - if (!Array.isArray(tools)) return [] - return (tools as ToolLike[]) - .map((t) => t.name) - .filter((n) => typeof n === 'string') - .sort() +// Cache hash based on the system prompt + complete tool definitions. +// Callers pass tools only when the tool block is written into the cache and +// left there, so a different tool set gets its own cache instead of reusing a +// prefix that holds the old block. Every prompt-affecting field participates, +// not just the name: canonical serialization avoids cache misses caused only +// by object-key insertion order, while tool-array order is preserved because +// that is the order sent to the model. +function canonicalizeHashInput(value: unknown): unknown { + if (Array.isArray(value)) return value.map(canonicalizeHashInput) + if (typeof value !== 'object' || value === null) return value + + const canonical: Record = {} + for (const key of Object.keys(value).sort()) { + canonical[key] = canonicalizeHashInput((value as Record)[key]) + } + return canonical } -// Cache hash based on system prompt + tool names. -// Callers pass tools only when the tool block is written into the cache and -// left there (static placement), so a different tool set gets its own cache -// instead of reusing a prefix that holds the old block. export function generateConfigHash(systemPrompt: string | null, tools?: unknown): string { const hash = crypto.createHash('sha-256') - const toolNames = getToolNamesForHash(tools) - hash.update(Buffer.from(JSON.stringify({ systemPrompt, toolNames }), 'utf8')) + const canonicalConfig = JSON.stringify( + canonicalizeHashInput({ systemPrompt, tools: Array.isArray(tools) ? tools : [] }) + ) + hash.update(Buffer.from(canonicalConfig, 'utf8')) return hash.digest('hex').substring(0, 16) } diff --git a/packages/sdk/server/bare/plugins/llamacpp-completion/ops/batch-completion-stream.ts b/packages/sdk/server/bare/plugins/llamacpp-completion/ops/batch-completion-stream.ts index bef9f7d39c..5e47e61f90 100644 --- a/packages/sdk/server/bare/plugins/llamacpp-completion/ops/batch-completion-stream.ts +++ b/packages/sdk/server/bare/plugins/llamacpp-completion/ops/batch-completion-stream.ts @@ -1,6 +1,5 @@ import type { AbortSignal } from 'bare-abort-controller' import type { BatchCompletionStreamPrompt, CompletionStats, ResponseFormat, Tool } from '@/schemas' -import { TOOLS_MODE } from '@/schemas/tools' import { getModel, getModelConfig, type AnyModel } from '@/server/bare/registry/model-registry' import type { DisposableScope } from '@/server/bare/runtime/disposable-scope' import type { Logger } from '@/logging/types' @@ -14,7 +13,7 @@ import { type CompletionGenerationParams } from '@/server/bare/plugins/llamacpp-completion/ops/completion-stream' import { normalizeCompletionStats } from '@/server/bare/plugins/llamacpp-completion/ops/completion-stats' -import { appendToolsToHistory, prependToolsToHistory } from '@/server/utils/tool-integration' +import { prependToolsToHistory } from '@/server/utils/tool-integration' const logger = getServerLogger() @@ -64,7 +63,6 @@ type BatchModelStreamResult = { type BatchPromptRenderOptions = { toolsEnabled: boolean - toolsMode?: string | undefined } function runBatchModel(model: AnyModel, prompts: AddonBatchPrompt[]) { @@ -99,10 +97,7 @@ function renderPromptHistory( let historyWithTools: Array = prompt.history if (tools) { - historyWithTools = - options.toolsMode === TOOLS_MODE.dynamic - ? appendToolsToHistory(prompt.history, tools) - : prependToolsToHistory(prompt.history, tools) + historyWithTools = prependToolsToHistory(prompt.history, tools) } // Uses the same attachment expansion as single completion: each SDK @@ -155,8 +150,7 @@ export async function* batchCompletion( const model = getModel(modelId) const modelConfig = getModelConfig(modelId) const renderOptions: BatchPromptRenderOptions = { - toolsEnabled: (modelConfig as { tools?: boolean }).tools === true, - toolsMode: (modelConfig as { toolsMode?: string }).toolsMode + toolsEnabled: (modelConfig as { tools?: boolean }).tools === true } const onAbort = () => { diff --git a/packages/sdk/server/bare/plugins/llamacpp-completion/ops/completion-stream.ts b/packages/sdk/server/bare/plugins/llamacpp-completion/ops/completion-stream.ts index 2dd1eac8be..ecd612f13f 100644 --- a/packages/sdk/server/bare/plugins/llamacpp-completion/ops/completion-stream.ts +++ b/packages/sdk/server/bare/plugins/llamacpp-completion/ops/completion-stream.ts @@ -9,7 +9,6 @@ import type { ToolCall, ToolDialect } from '@/schemas' -import { TOOLS_MODE } from '@/schemas/tools' import { logCacheDisabled, logCacheInit, @@ -29,11 +28,7 @@ import { type TurnHandle } from '@/server/bare/plugins/llamacpp-completion/ops/kv-cache-session' import type { DisposableScope } from '@/server/bare/runtime/disposable-scope' -import { - appendToolsToHistory, - detectToolDialect, - prependToolsToHistory -} from '@/server/utils/tool-integration' +import { detectToolDialect, prependToolsToHistory } from '@/server/utils/tool-integration' import { parseToolCalls } from '@/server/utils/tools' import { getResponseFormatJsonSchema } from '@/server/utils/response-format' import { buildAutoCacheSaveHistory, type CacheMessage } from '@/server/utils' @@ -205,30 +200,19 @@ type HistoryMsg = { attachments?: { path: string }[] | undefined } -type ToolPlacement = 'static' | 'dynamic' - /** - * Attach the tool block to a turn payload at the position its placement - * requires. - * - * Static mirrors the no-kv-cache path (`prependToolsToHistory`) and keeps the - * block ahead of the conversation. Dynamic must leave it immediately after the - * last anchor message, which is what the addon's `ToolsCompactController` - * validates before it will anchor and later trim the block. + * Attach the tool block ahead of a turn payload, mirroring the no-kv-cache + * path (`prependToolsToHistory`). */ -function withToolBlock( - messages: ChatHistory[], - toolBlock: ChatHistory[], - placement: ToolPlacement -): ChatHistory[] { +function withToolBlock(messages: ChatHistory[], toolBlock: ChatHistory[]): ChatHistory[] { if (toolBlock.length === 0) return messages - return placement === 'static' ? [...toolBlock, ...messages] : [...messages, ...toolBlock] + return [...toolBlock, ...messages] } interface CachePayload { messages: ChatHistory[] /** - * Whether the prefix will hold a rendered static tool block once this turn + * Whether the prefix will hold a rendered tool block once this turn * commits — either it already did, or this payload carries one the template * will render. */ @@ -251,12 +235,10 @@ function rendersToolBlock(messages: HistoryMsg[], toolBlock: ChatHistory[]): boo /** * Pick the messages that need to reach the model for the next turn. * - * `placement` selects both the slicing strategy and where the tool block sits - * in the payload. Tools are never baked into the primed prefix — a prefix with - * no user turn is not a renderable conversation for every template — so they - * travel with a turn instead. + * Tools are never baked into the primed prefix — a prefix with no user turn is + * not a renderable conversation for every template — so they travel with a + * turn instead. * - * Static placement: * - Empty history: nothing to slice; send whatever non-system messages * exist. (The call site always reports the cache as existing, so this * is the only way into this branch.) @@ -269,20 +251,6 @@ function rendersToolBlock(messages: HistoryMsg[], toolBlock: ChatHistory[]): boo * the bad boundary doesn't propagate into the next turn. * - The tool block travels only with the turn that writes it into the * cache; see `skipToolBlock` below. - * - * Dynamic placement: - * - The addon anchors the tool block after the last user message and - * trims tools + the assistant's tool-call output from the cache once - * the chain resolves. After that trim, the cache only holds messages - * up to the last user turn, so the SDK has to ship the right slice - * plus the (possibly new) tool set: - * * tool-chain continuation (last role is "tool"): send the trailing - * consecutive tool messages, no tool block — tools are still - * anchored in the cache from the previous round. - * * new user turn after a chain (prev role is "assistant"): send - * [assistant, user] so the model sees its own final reply before - * the new prompt, then re-anchor the tool block. - * * otherwise: send just the last message + tool block. */ function prepareMessagesForCache( session: KvCacheSession, @@ -290,7 +258,6 @@ function prepareMessagesForCache( cacheExists: boolean, history: HistoryMsg[], tools?: Tool[], - placement: ToolPlacement = 'static', toolBlockEvictable = false ): CachePayload { const toolBlock = tools?.length ? transformMessages(tools) : [] @@ -298,76 +265,43 @@ function prepareMessagesForCache( if (!(cacheExists && history.length > 0)) { const historyWithoutSystem = history.filter((msg) => msg.role !== 'system') return { - messages: withToolBlock(transformMessages(historyWithoutSystem), toolBlock, placement), - toolBlockCached: placement === 'static' && rendersToolBlock(historyWithoutSystem, toolBlock) + messages: withToolBlock(transformMessages(historyWithoutSystem), toolBlock), + toolBlockCached: rendersToolBlock(historyWithoutSystem, toolBlock) } } - if (placement === 'static') { - // Static path — slice from the turn's `savedCount` so callers can - // stage multiple messages between completions. `decideCachedHistorySlice` - // also guards against the QVAC-17780 stale-count regression: if the - // saved boundary would slice the history down to an empty payload - // (e.g. after a cancelled mid-decode), it falls back to the full - // non-system history and signals the caller to drop the bad entry. - // The session owns the entry; `dropStaleSavedCount` clears it - // without touching the on-disk file (the file is still trustworthy - // — only the boundary count is wrong). - const { messages, clearStaleCount } = decideCachedHistorySlice( - turn.savedCount, - cacheExists, - history - ) - - if (clearStaleCount) { - session.dropStaleSavedCount(turn) - } - - // Static never trims the block back out of the cache, so re-sending it - // every turn would leave one copy per turn and grow the prefix with the - // conversation. Skip it only when the prefix is known to hold a rendered - // one: `toolBlockCached` records that a previous turn actually got it into - // the cache, which a committed message count does not prove. A stale - // boundary means we are resending the whole conversation anyway, and an - // evictable block can no longer be assumed present. - const skipToolBlock = turn.toolBlockCached && !clearStaleCount && !toolBlockEvictable - const blockToSend = skipToolBlock ? [] : toolBlock + // Slice from the turn's `savedCount` so callers can + // stage multiple messages between completions. `decideCachedHistorySlice` + // also guards against the QVAC-17780 stale-count regression: if the + // saved boundary would slice the history down to an empty payload + // (e.g. after a cancelled mid-decode), it falls back to the full + // non-system history and signals the caller to drop the bad entry. + // The session owns the entry; `dropStaleSavedCount` clears it + // without touching the on-disk file (the file is still trustworthy + // — only the boundary count is wrong). + const { messages, clearStaleCount } = decideCachedHistorySlice( + turn.savedCount, + cacheExists, + history + ) - return { - messages: withToolBlock(transformMessages(messages), blockToSend, placement), - toolBlockCached: skipToolBlock || rendersToolBlock(messages, blockToSend) - } + if (clearStaleCount) { + session.dropStaleSavedCount(turn) } - // Dynamic path. The addon trimmed tools after the previous round, so the - // cache no longer holds the saved-count we'd rely on for slicing — pick - // the right fragment based on the role of the last history message. Nothing - // tool-specific survives that trim, so the prefix never counts as holding a - // block and every turn re-anchors its own. - const lastMsg = history[history.length - 1]! - - if (lastMsg.role === 'tool') { - const trailingTools: HistoryMsg[] = [] - for (let i = history.length - 1; i >= 0; i--) { - const msg = history[i]! - if (msg.role !== 'tool') break - trailingTools.unshift(msg) - } - return { messages: transformMessages(trailingTools), toolBlockCached: false } - } - - if (lastMsg.role === 'user') { - const prevMsg = history[history.length - 2] - const tail = prevMsg?.role === 'assistant' ? [prevMsg, lastMsg] : [lastMsg] - return { - messages: withToolBlock(transformMessages(tail), toolBlock, placement), - toolBlockCached: false - } - } + // The block is never trimmed back out of the cache, so re-sending it every + // turn would leave one copy per turn and grow the prefix with the + // conversation. Skip it only when the prefix is known to hold a rendered + // one: `toolBlockCached` records that a previous turn actually got it into + // the cache, which a committed message count does not prove. A stale + // boundary means we are resending the whole conversation anyway, and an + // evictable block can no longer be assumed present. + const skipToolBlock = turn.toolBlockCached && !clearStaleCount && !toolBlockEvictable + const blockToSend = skipToolBlock ? [] : toolBlock return { - messages: withToolBlock(transformMessages([lastMsg]), toolBlock, placement), - toolBlockCached: false + messages: withToolBlock(transformMessages(messages), blockToSend), + toolBlockCached: skipToolBlock || rendersToolBlock(messages, blockToSend) } } @@ -458,16 +392,12 @@ export async function* completion( const modelConfig = getModelConfig(modelId) const toolsEnabled = (modelConfig as { tools?: boolean }).tools === true - const toolsMode = (modelConfig as { toolsMode?: string }).toolsMode const toolsActive = !!tools?.length && toolsEnabled - const dynamicTools = toolsActive && toolsMode === TOOLS_MODE.dynamic - const staticTools = toolsActive && !dynamicTools // Sliding is opt-in (`n_discarded` defaults to 0). Once on, the addon's - // discard window opens at the end of the primed prefix — which is where a - // static tool block sits, since the prime is the system prompt alone — and - // the clamp that would protect it only runs in dynamic mode. So while - // sliding is possible the block cannot be assumed to survive, and it has to - // travel with every turn. + // discard window opens at the end of the primed prefix — which is where the + // tool block sits, since the prime is the system prompt alone — and nothing + // protects it. So while sliding is possible the block cannot be assumed to + // survive, and it has to travel with every turn. const toolBlockEvictable = ((modelConfig as { n_discarded?: number }).n_discarded ?? 0) > 0 const dialect = @@ -536,10 +466,8 @@ export async function* completion( if (!kvCache) { // KV-cache disabled — straight passthrough, no session involvement. let historyWithTools: Array = history - if (staticTools && tools) { + if (toolsActive && tools) { historyWithTools = prependToolsToHistory(history, tools) - } else if (dynamicTools && tools) { - historyWithTools = appendToolsToHistory(history, tools) } const transformedHistory = transformMessages(historyWithTools) @@ -564,11 +492,10 @@ export async function* completion( const session = createKvCacheSession(modelId, { logger: requestLogger }) const systemPromptFromHistory = extractSystemPrompt(history) - // Static bakes the tool block into the cache on the turn that first sends it - // and never trims it, so a late or changed tool set has to land on a fresh - // cache rather than a warm prefix holding the old block. Dynamic trims its - // block after each chain, so nothing tool-specific survives in its cache. - const configHash = generateConfigHash(systemPromptFromHistory, staticTools ? tools : undefined) + // The tool block is baked into the cache on the turn that first sends it and + // never trimmed, so a late or changed tool set has to land on a fresh cache + // rather than a warm prefix holding the old block. + const configHash = generateConfigHash(systemPromptFromHistory, toolsActive ? tools : undefined) const systemPromptToUse = systemPromptFromHistory || @@ -622,7 +549,6 @@ export async function* completion( /* cacheExists */ true, history, toolsActive ? tools : undefined, - dynamicTools ? 'dynamic' : 'static', toolBlockEvictable ) const messagesToSend = payload.messages diff --git a/packages/sdk/server/bare/plugins/llamacpp-completion/ops/kv-cache-session.ts b/packages/sdk/server/bare/plugins/llamacpp-completion/ops/kv-cache-session.ts index 91041ed426..ac6cb71d39 100644 --- a/packages/sdk/server/bare/plugins/llamacpp-completion/ops/kv-cache-session.ts +++ b/packages/sdk/server/bare/plugins/llamacpp-completion/ops/kv-cache-session.ts @@ -219,7 +219,7 @@ export interface BeginCustomTurnInput { kind: 'custom' /** User-provided session key (`completion({ kvCache: "session-a" })`). */ customKey: string - /** Hash of system prompt + (static) tool names. */ + /** Hash of system prompt + complete tool definitions. */ configHash: string /** * Prime the cache by sending the system prompt to the addon. Tools are not @@ -233,7 +233,7 @@ export interface BeginCustomTurnInput { export interface BeginAutoTurnInput { kind: 'auto' - /** Hash of system prompt + (static) tool names. */ + /** Hash of system prompt + complete tool definitions. */ configHash: string /** Conversation history used to compute the pre-response cache key. */ history: CacheMessage[] diff --git a/packages/sdk/server/bare/plugins/llamacpp-completion/transform.ts b/packages/sdk/server/bare/plugins/llamacpp-completion/transform.ts index 9231e97700..e42246e482 100644 --- a/packages/sdk/server/bare/plugins/llamacpp-completion/transform.ts +++ b/packages/sdk/server/bare/plugins/llamacpp-completion/transform.ts @@ -1,4 +1,4 @@ -import { TOOLS_MODE, type LlmConfig } from '@/schemas' +import { type LlmConfig } from '@/schemas' /** * Converts an LlmConfig into the flat string-keyed map the C++ addon expects. @@ -35,12 +35,5 @@ export function transformLlmConfig(llmConfig: LlmConfig) { delete transformed['opencl_cache_dir'] } - if ('tools_mode' in transformed) { - if (transformed['tools_mode'] === TOOLS_MODE.dynamic) { - transformed['tools_compact'] = 'true' - } - delete transformed['tools_mode'] - } - return transformed } diff --git a/packages/sdk/server/utils/tool-integration.ts b/packages/sdk/server/utils/tool-integration.ts index 210a99b669..a39e86bda1 100644 --- a/packages/sdk/server/utils/tool-integration.ts +++ b/packages/sdk/server/utils/tool-integration.ts @@ -9,9 +9,9 @@ interface HistoryMessage { } /** - * Static tools mode: prepend tools right after the system message (or at the - * very start when no system message is present). The tool block stays in the - * kv-cache for the whole chat session. + * Prepend tools right after the system message (or at the very start when no + * system message is present). The tool block stays in the kv-cache for the + * whole chat session. */ export function prependToolsToHistory( history: HistoryMessage[], @@ -26,19 +26,6 @@ export function prependToolsToHistory( return [...tools, ...history] } -/** - * Dynamic tools mode: append tools after the last history message. The - * addon's compact-tools mode anchors the block after the last user message - * and trims it from the kv-cache once the tool-call chain resolves, so a - * subsequent turn can ship a different tool set without poisoning the cache. - */ -export function appendToolsToHistory( - history: HistoryMessage[], - tools: Tool[] -): Array { - return [...history, ...tools] -} - export function detectToolDialect(modelId: string): ToolDialect { const info = getModelInfo(modelId) if (!info) return 'hermes' diff --git a/packages/sdk/test/bare/completion-kvcache-tools.test.ts b/packages/sdk/test/bare/completion-kvcache-tools.test.ts index 3000a08fe9..4231ea5786 100644 --- a/packages/sdk/test/bare/completion-kvcache-tools.test.ts +++ b/packages/sdk/test/bare/completion-kvcache-tools.test.ts @@ -329,10 +329,9 @@ test('completion: kv-cache resends the tool block after a turn that could not re }) // With `n_discarded > 0` the addon may slide its context window, and the -// discard region opens exactly where a static tool block sits — the protected -// prefix ends at the primed system prompt, and the clamp that would guard the -// block only runs in dynamic mode. While the block can be evicted it has to -// travel with every turn. +// discard region opens exactly where the tool block sits — the protected +// prefix ends at the primed system prompt, and nothing guards the block. +// While the block can be evicted it has to travel with every turn. test('completion: kv-cache resends the tool block when the context window can slide', async (t) => { await setIsolatedHome() clearRegistry() diff --git a/packages/sdk/test/bare/runtime/kv-cache-session.test.ts b/packages/sdk/test/bare/runtime/kv-cache-session.test.ts index 65ccec471a..8cf3adf84f 100644 --- a/packages/sdk/test/bare/runtime/kv-cache-session.test.ts +++ b/packages/sdk/test/bare/runtime/kv-cache-session.test.ts @@ -77,6 +77,91 @@ async function loadSession() { return { fs, path, mod, utils, retention, cleanup, writeFakeCache } } +test('generateConfigHash: includes complete canonical tool definitions', async (t) => { + const { mod, cleanup } = await loadSession() + try { + const calculator = { + type: 'function', + name: 'calculator', + description: 'Performs arithmetic', + parameters: { + type: 'object', + properties: { + operation: { type: 'string', enum: ['add', 'subtract'] }, + value: { type: 'number' } + }, + required: ['operation', 'value'] + } + } + const changedSchema = { + ...calculator, + parameters: { + ...calculator.parameters, + properties: { + ...calculator.parameters.properties, + operation: { type: 'string', enum: ['multiply', 'divide'] } + } + } + } + const reorderedKeys = { + parameters: { + required: ['operation', 'value'], + properties: { + value: { type: 'number' }, + operation: { enum: ['add', 'subtract'], type: 'string' } + }, + type: 'object' + }, + description: 'Performs arithmetic', + name: 'calculator', + type: 'function' + } + + const originalHash = mod.generateConfigHash('system prompt', [calculator]) + const changedHash = mod.generateConfigHash('system prompt', [changedSchema]) + const reorderedHash = mod.generateConfigHash('system prompt', [reorderedKeys]) + + t.not(originalHash, changedHash, 'same-named tools with different schemas use different caches') + t.is(originalHash, reorderedHash, 'object-key insertion order does not affect cache identity') + + const other = { ...calculator, name: 'search' } + t.not( + mod.generateConfigHash('system prompt', [calculator, other]), + mod.generateConfigHash('system prompt', [other, calculator]), + 'tool-array order participates in cache identity' + ) + } finally { + cleanup() + } +}) + +// `configHash` is the on-disk `.bin` filename, so the digest of a tool-free +// session is a compatibility surface: any change to the hash payload or its +// serialization renames every plain-chat cache file and re-primes it cold. +// Pinning the shipped digests keeps that a deliberate decision. +test('generateConfigHash: no-tools digests stay pinned', async (t) => { + const { mod, cleanup } = await loadSession() + try { + t.is( + mod.generateConfigHash('you are a helpful assistant.', undefined), + '3f5906d163f40776', + 'omitted tools keep the shipped digest' + ) + t.is( + mod.generateConfigHash('you are a helpful assistant.', []), + '3f5906d163f40776', + 'an empty tool array hashes like omitted tools' + ) + t.is( + mod.generateConfigHash(null, undefined), + '99ba47708d700919', + 'a missing system prompt keeps the shipped digest' + ) + } finally { + cleanup() + } +}) + test('kv-cache-session: beginTurn primes the cache on first use, reuses on second', async (t) => { const { mod, cleanup, writeFakeCache } = await loadSession() try { diff --git a/plugins/opencode/src/managed-serve-host.ts b/plugins/opencode/src/managed-serve-host.ts index c92974ec47..5dc6c0356f 100644 --- a/plugins/opencode/src/managed-serve-host.ts +++ b/plugins/opencode/src/managed-serve-host.ts @@ -19,8 +19,7 @@ function createManagedServe(config: ManagedServeHostConfig): Promise