diff --git a/README.md b/README.md index ec9ef3c..c24d5e3 100644 --- a/README.md +++ b/README.md @@ -339,6 +339,7 @@ Provider diagnostics also include the VS Code/extension versions, extension host | `opencodego.streamIdleTimeoutSeconds` | `120` | Cancel if stream goes idle | | `opencodego.showUsageStatusBar` | `true` | Show usage summary in status bar | | `opencodego.showProviderPrefix` | `true` | Include `OpenCode Go` / `OpenCode Zen` in model names | +| `opencodego.visionProxyWholeConversation` | `false` | Vision proxy: describe the whole conversation instead of only the message with a new image (more context, more tokens) | | `opencodego.freeOnly` | `true` | Zen: free models only. `false` = include paid | | `opencodego.agentsWindow` | `true` | Expose agent-host model variants (`targetChatSessionType`) for the Agents window | | `opencodego.showAgentModelsInManagePanel` | `false` | Show agent vendors in Manage Language Models panel | diff --git a/docs/features/11-20260715-vision-proxy.md b/docs/features/11-20260715-vision-proxy.md index 39f9877..f8406f3 100644 --- a/docs/features/11-20260715-vision-proxy.md +++ b/docs/features/11-20260715-vision-proxy.md @@ -79,6 +79,20 @@ Storage keys (in extension `globalState`): - `opencodego.visionProxyModelId` — target Copilot model ID (e.g. `copilot:gpt-5.5`) - `opencodego.visionProxyPrompt` — description instruction sent to the vision model +### Description Cache & Whole-Conversation Mode + +Descriptions produced by the proxy are cached per image (SHA-256 of its bytes), so +images already described in earlier turns are reused without calling the vision model +again - saving Copilot quota and latency in multi-turn conversations (issue #119). + +A boolean setting controls how much context each description gets: + +- `opencodego.visionProxyWholeConversation` (default `false`). Off: the proxy + describes only the message that contains a new image and reuses cached + descriptions, keeping token usage low. On: the proxy sends the whole + conversation to the vision model so descriptions carry full context, at the + cost of more tokens. Descriptions are still cached in both modes. + ### Graceful Fallback If the proxy fails (model not found, API error, empty description), images are stripped with a placeholder `[Image unavailable — vision proxy unavailable]` instead of forwarding raw image data to a text-only model (which would 400). Original text parts are preserved. diff --git a/package-lock.json b/package-lock.json index aab7569..ae0cd4f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "opencode-copilot-chat", - "version": "0.5.0", + "version": "0.5.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "opencode-copilot-chat", - "version": "0.5.0", + "version": "0.5.1", "license": "MIT", "dependencies": { "@silvia-odwyer/photon-node": "^0.3.4" @@ -843,6 +843,7 @@ "integrity": "sha512-X6ypGChaWYk6PBtUg2BwuTZEFFcHJAtGTVJ9/lCTOufhZ4i9fNolQNnktq+kkMCwMj7V8Svsq7+TxSDslmhE0g==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.66.0", "@typescript-eslint/types": "8.66.0", @@ -1241,6 +1242,7 @@ "integrity": "sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -2138,6 +2140,7 @@ "integrity": "sha512-nuKKvN+oIBO0koN7Tm7dlkmnkc21mtt0QJLwAKzjLq14y6lRTdVG36MZHJ8eQHwdJMwZbQNMlPOYedMq/oVJvQ==", "dev": true, "license": "MIT", + "peer": true, "workspaces": [ "packages/*" ], @@ -5568,6 +5571,7 @@ "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -5686,6 +5690,7 @@ "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" diff --git a/package.json b/package.json index 0d90d0b..e04fd13 100644 --- a/package.json +++ b/package.json @@ -176,6 +176,11 @@ "default": true, "description": "Show the OpenCode Go or OpenCode Zen provider prefix in model names." }, + "opencodego.visionProxyWholeConversation": { + "type": "boolean", + "default": false, + "description": "Off (default): describe only the message with a new image and reuse cached image descriptions to save quota. On: describe the whole conversation for richer context (uses more tokens; descriptions are still cached)." + }, "opencodego.freeOnly": { "type": "boolean", "default": true, diff --git a/src/extension.ts b/src/extension.ts index a731e26..a2cc119 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -46,6 +46,7 @@ import { } from "./providerTypes"; import { isInternalDataPart } from "./chatParts"; import { getImageDataUrlBase64Bytes, MAX_IMAGE_BASE64_BYTES, normalizeImageDataUrl } from "./imageNormalizer"; +import { imageDescriptionKey, lookupImageDescriptions, storeImageDescriptions } from "./visionProxyCache"; import { providerModelDisplayName } from "./modelNames"; import { buildStableModelCapabilities } from "./modelCapabilities"; import { calculateModelLimits, type ModelLimits } from "./modelLimits"; @@ -2020,11 +2021,25 @@ class OpenCodeProvider implements vscode.LanguageModelChatProvider convertMessage(message, this.reasoningContentByToolCallId, rawModelId)), ); - const apiMessages = normalizeMessages(convertedMessages.flatMap((result) => result.messages)); const normalizedImageCount = convertedMessages.map((result) => result.normalizedImageCount).reduce((total, count) => total + count, 0); if (normalizedImageCount > 0) { this.log(`[vision] Normalized ${normalizedImageCount} image attachment(s) to provider-safe dimensions/encoding.`); } + + // Flatten the converted messages, tracking which original message produced + // each apiMessage. The vision proxy returns per-message descriptions keyed + // by the original message index, so this mapping lets us apply the correct + // description to the right apiMessage (convertMessage can emit several + // messages per input — e.g. tool results — which shifts indices). + const flatMessages: ApiMessage[] = []; + const flatSourceIndex: number[] = []; + for (let i = 0; i < convertedMessages.length; i++) { + for (const msg of convertedMessages[i].messages) { + flatMessages.push(msg); + flatSourceIndex.push(i); + } + } + const baseSettings = getSettings(); // Apply per-request Thinking selection (from Copilot Chat submenu) on top // of the workspace default. The override only affects the current model @@ -2040,35 +2055,48 @@ class OpenCodeProvider implements vscode.LanguageModelChatProvider(VISION_PROXY_MODEL_ID_KEY, "") || "" : ""; if (hasImageInput && !actuallySupportsVision && visionProxyModelId) { const visionProxyPrompt = this.context.globalState.get(VISION_PROXY_PROMPT_KEY, "") || DEFAULT_VISION_PROXY_PROMPT; + // When `opencodego.visionProxyWholeConversation` is on, describe the whole + // conversation instead of only the message with a new image, so descriptions + // keep conversation context (at the cost of more tokens). + const describeWholeConversation = vscode.workspace.getConfiguration("opencodego").get("visionProxyWholeConversation", false); let imagesHandled = false; try { - this.log(`[vision-proxy] Forwarding images to ${visionProxyModelId}`); - const description = await proxyVision(messages, visionProxyModelId, visionProxyPrompt, token); - if (description) { - for (let i = 0; i < apiMessages.length; i++) { - const msg = apiMessages[i]; + this.log(`[vision-proxy] Forwarding images to ${visionProxyModelId}${describeWholeConversation ? " (whole conversation)" : ""}`); + const { descriptions, cacheHits, cacheMisses } = await proxyVision(messages, visionProxyModelId, visionProxyPrompt, describeWholeConversation, token); + if (descriptions.size > 0) { + const fallbackDescription = descriptions.values().next().value ?? ""; + for (let i = 0; i < flatMessages.length; i++) { + const msg = flatMessages[i]; if (!Array.isArray(msg.content)) continue; - if (msg.content.some((p) => p.type === "image_url")) { - const textParts = msg.content - .filter((p): p is OpenAiContentPart & { text: string } => p.type === "text" && typeof p.text === "string") - .map((p) => p.text); - msg.content = [{ type: "text", text: `[Image described by vision proxy]: ${description}` }]; - if (textParts.length > 0) { - msg.content.push({ type: "text", text: textParts.join("\n") }); - } - imagesHandled = true; + if (!msg.content.some((p) => p.type === "image_url")) continue; + const textParts = msg.content + .filter((p): p is OpenAiContentPart & { text: string } => p.type === "text" && typeof p.text === "string") + .map((p) => p.text); + // Tool-result images are not described by the proxy, so they fall + // back to the first available description (matching the previous + // single-description behavior). + const description = descriptions.get(flatSourceIndex[i]) ?? fallbackDescription; + msg.content = [{ type: "text", text: `[Image described by vision proxy]: ${description}` }]; + if (textParts.length > 0) { + msg.content.push({ type: "text", text: textParts.join("\n") }); } + imagesHandled = true; } - this.log(`[vision-proxy] Replaced images using vision proxy model`); + this.log(`[vision-proxy] Replaced images using vision proxy model (${cacheHits} from cache, ${cacheMisses} newly described)`); } } catch (err) { this.log(`[vision-proxy] Error: ${err instanceof Error ? err.message : String(err)}`); @@ -2078,8 +2106,8 @@ class OpenCodeProvider implements vscode.LanguageModelChatProvider p.type === "image_url")) { const textParts = msg.content @@ -2095,6 +2123,8 @@ class OpenCodeProvider implements vscode.LanguageModelChatProvider; + /** Messages whose images were already cached — no vision model request was made. */ + cacheHits: number; + /** Messages that required a new vision-model request. */ + cacheMisses: number; +}; + /** - * Vision proxy: relay image messages through a vision-capable Copilot model - * and return the text description. This lets text-only models "see" images - * transparently (issue #74). + * Collect the parts of a single message that the vision proxy should see: + * image parts plus text parts, dropping tool parts. */ -async function proxyVision( - messages: readonly vscode.LanguageModelChatRequestMessage[], - visionModelId: string, +function collectRequestParts( + msg: vscode.LanguageModelChatRequestMessage, +): Array { + const parts: Array = []; + for (const part of msg.content) { + if (part instanceof vscode.LanguageModelDataPart && part.mimeType.startsWith("image/")) { + parts.push(part); + } else if (part instanceof vscode.LanguageModelTextPart) { + parts.push(part); + } else if (typeof part === "object" && part !== null && "value" in part) { + const valuePart = part as { value: unknown }; + parts.push(new vscode.LanguageModelTextPart(String(valuePart.value))); + } + } + return parts; +} + +/** + * Build a vision-model request for a single message: keep its image parts and + * text parts (dropping tool parts), then append the vision prompt. This lets + * the proxy describe ONLY the message that contains a new image, instead of + * re-sending the whole conversation on every turn. + */ +function buildVisionRequestMessage( + msg: vscode.LanguageModelChatRequestMessage, visionPrompt: string, - token: vscode.CancellationToken, -): Promise { - // Find the vision model by trying several matching strategies: - // 1. Exact id match (full internal model id) - // 2. Vendor:id partial (e.g. "opencodego:mimo-v2.5") - // 3. Name or id substring (e.g. "mimo-v2.5" or "Mimo V2.5") - // Filter out agent-host variants — they use a different transport and - // don't have vision support. Prefer non-agent models. - const nonAgent = (models: readonly vscode.LanguageModelChat[]) => models.filter((m) => !m.id.includes("-agent:")); - - let visionModels = nonAgent(await vscode.lm.selectChatModels({ id: visionModelId })); - if (!visionModels || visionModels.length === 0) { - // Try matching by name substring across all providers - const allVisible = nonAgent(await vscode.lm.selectChatModels({})); - visionModels = allVisible.filter( - (m) => - m.id.toLowerCase().includes(visionModelId.toLowerCase()) || - m.name.toLowerCase().includes(visionModelId.toLowerCase()) || - m.family.toLowerCase().includes(visionModelId.toLowerCase()), +): vscode.LanguageModelChatMessage[] { + const requestMessages: vscode.LanguageModelChatMessage[] = []; + const parts = collectRequestParts(msg); + if (parts.length > 0) { + requestMessages.push( + new vscode.LanguageModelChatMessage( + msg.role === vscode.LanguageModelChatMessageRole.Assistant + ? vscode.LanguageModelChatMessageRole.Assistant + : vscode.LanguageModelChatMessageRole.User, + parts, + ), ); } - if (!visionModels || visionModels.length === 0) { - throw new Error(`Vision model "${visionModelId}" not found. ` + `Run "OpenCode Go: Configure Vision Proxy" to see available models.`); + // Append the vision prompt + if (visionPrompt) { + requestMessages.push(vscode.LanguageModelChatMessage.User(visionPrompt)); } + return requestMessages; +} - // All models that matched are candidates. `selectChatModels` returns - // `LanguageModelChat` which does not expose capabilities in the stable - // API, so we just use the first match. Most vision models handle image - // input gracefully — models without vision will report the error. - const model = visionModels[0]; - - // Build a request preserving images and text from the original messages +/** + * Build a vision-model request over the WHOLE conversation: keep image and + * text parts from every message (dropping tool parts), then append the vision + * prompt. Used when `opencodego.visionProxyWholeConversation` is enabled, so + * descriptions carry full conversation context (at the cost of more tokens). + */ +function buildWholeConversationRequest( + messages: readonly vscode.LanguageModelChatRequestMessage[], + visionPrompt: string, +): vscode.LanguageModelChatMessage[] { const requestMessages: vscode.LanguageModelChatMessage[] = []; for (const msg of messages) { - const parts: Array = []; - for (const part of msg.content) { - if (part instanceof vscode.LanguageModelDataPart && part.mimeType.startsWith("image/")) { - parts.push(part); - } else if (part instanceof vscode.LanguageModelTextPart) { - parts.push(part); - } else if (typeof part === "object" && part !== null && "value" in part) { - const valuePart = part as { value: unknown }; - parts.push(new vscode.LanguageModelTextPart(String(valuePart.value))); - } - } + const parts = collectRequestParts(msg); if (parts.length > 0) { requestMessages.push( new vscode.LanguageModelChatMessage( @@ -3863,18 +3912,149 @@ async function proxyVision( ); } } - // Append the vision prompt if (visionPrompt) { requestMessages.push(vscode.LanguageModelChatMessage.User(visionPrompt)); } + return requestMessages; +} - const response = await model.sendRequest(requestMessages, {}, token); - let fullDescription = ""; - for await (const part of response.text) { - fullDescription += part; +/** + * Vision proxy: relay image messages through a vision-capable Copilot model + * and return the text description. This lets text-only models "see" images + * transparently (issue #74). + * + * By default (`describeWholeConversation` false), descriptions are cached per + * image (`imageDescriptionCache`). A message whose images are ALL already + * cached is reused without contacting the vision model; only messages that + * contain at least one new image trigger a `sendRequest()` - for that single + * message + prompt - and the result is stored in the cache for future turns. + * + * When `describeWholeConversation` is true (setting + * `opencodego.visionProxyWholeConversation`), the proxy sends ONE request over + * the whole conversation so descriptions keep full context; the combined + * description is still stored under every image hash. + */ +async function proxyVision( + messages: readonly vscode.LanguageModelChatRequestMessage[], + visionModelId: string, + visionPrompt: string, + describeWholeConversation: boolean, + token: vscode.CancellationToken, +): Promise { + const descriptions = new Map(); + let cacheHits = 0; + let cacheMisses = 0; + + // Find the vision model lazily — only when a message actually needs a new + // description. When every image is already cached we never call + // `vscode.lm.selectChatModels()` or `model.sendRequest()`. + let visionModel: vscode.LanguageModelChat | undefined; + const resolveVisionModel = async (): Promise => { + if (visionModel) { + return visionModel; + } + // Matching strategies: + // 1. Exact id match (full internal model id) + // 2. Vendor:id partial (e.g. "opencodego:mimo-v2.5") + // 3. Name or id substring (e.g. "mimo-v2.5" or "Mimo V2.5") + // Filter out agent-host variants — they use a different transport and + // don't have vision support. Prefer non-agent models. + const nonAgent = (models: readonly vscode.LanguageModelChat[]) => models.filter((m) => !m.id.includes("-agent:")); + + let visionModels = nonAgent(await vscode.lm.selectChatModels({ id: visionModelId })); + if (!visionModels || visionModels.length === 0) { + // Try matching by name substring across all providers + const allVisible = nonAgent(await vscode.lm.selectChatModels({})); + visionModels = allVisible.filter( + (m) => + m.id.toLowerCase().includes(visionModelId.toLowerCase()) || + m.name.toLowerCase().includes(visionModelId.toLowerCase()) || + m.family.toLowerCase().includes(visionModelId.toLowerCase()), + ); + } + if (!visionModels || visionModels.length === 0) { + throw new Error(`Vision model "${visionModelId}" not found. ` + `Run "OpenCode Go: Configure Vision Proxy" to see available models.`); + } + + // All models that matched are candidates. `selectChatModels` returns + // `LanguageModelChat` which does not expose capabilities in the stable + // API, so we just use the first match. Most vision models handle image + // input gracefully — models without vision will report the error. + visionModel = visionModels[0]; + return visionModel; + }; + + // Whole-conversation mode (opencodego.visionProxyWholeConversation): one + // request over all messages, so descriptions carry full conversation context. + if (describeWholeConversation) { + const imageIndices: number[] = []; + const allHashes: string[] = []; + for (let index = 0; index < messages.length; index++) { + const msg = messages[index]; + const imageParts = msg.content.filter( + (part): part is vscode.LanguageModelDataPart => + part instanceof vscode.LanguageModelDataPart && part.mimeType.startsWith("image/"), + ); + if (imageParts.length === 0) { + continue; + } + imageIndices.push(index); + allHashes.push(...imageParts.map((part) => imageDescriptionKey(dataPartToBase64(part.data)))); + } + if (imageIndices.length > 0) { + cacheMisses++; + const model = await resolveVisionModel(); + const response = await model.sendRequest(buildWholeConversationRequest(messages, visionPrompt), {}, token); + let fullDescription = ""; + for await (const part of response.text) { + fullDescription += part; + } + if (fullDescription) { + storeImageDescriptions(allHashes, fullDescription); + for (const index of imageIndices) { + descriptions.set(index, fullDescription); + } + } + } + return { descriptions, cacheHits, cacheMisses }; + } + + for (let index = 0; index < messages.length; index++) { + const msg = messages[index]; + const imageParts = msg.content.filter( + (part): part is vscode.LanguageModelDataPart => + part instanceof vscode.LanguageModelDataPart && part.mimeType.startsWith("image/"), + ); + if (imageParts.length === 0) { + continue; + } + const hashes = imageParts.map((part) => imageDescriptionKey(dataPartToBase64(part.data))); + + // All images already described → reuse the cached text, no model request. + const cachedDescription = lookupImageDescriptions(hashes); + if (cachedDescription !== undefined) { + cacheHits++; + descriptions.set(index, cachedDescription); + continue; + } + + // At least one new image → describe only this message and cache the result. + cacheMisses++; + const model = await resolveVisionModel(); + const response = await model.sendRequest(buildVisionRequestMessage(msg, visionPrompt), {}, token); + let fullDescription = ""; + for await (const part of response.text) { + fullDescription += part; + } + if (!fullDescription) { + continue; + } + storeImageDescriptions(hashes, fullDescription); + descriptions.set(index, fullDescription); } - return fullDescription.length > 0 ? fullDescription : undefined; + + return { descriptions, cacheHits, cacheMisses }; } // --------------------------------------------------------------------------- diff --git a/src/test/visionProxy.test.ts b/src/test/visionProxy.test.ts index 339b711..de135f0 100644 --- a/src/test/visionProxy.test.ts +++ b/src/test/visionProxy.test.ts @@ -1,6 +1,14 @@ import { describe, it } from "node:test"; import assert from "node:assert/strict"; import { buildStableModelCapabilities } from "../modelCapabilities"; +import { + clearImageDescriptionCache, + IMAGE_DESCRIPTION_CACHE_LIMIT, + imageDescriptionCache, + imageDescriptionKey, + lookupImageDescriptions, + storeImageDescriptions, +} from "../visionProxyCache"; /** * Vision proxy condition tests. @@ -89,3 +97,77 @@ describe("modelCapabilities vision proxy flag", () => { assert.equal("editTools" in capabilities, false); }); }); + +describe("vision proxy image description cache", () => { + it("imageDescriptionKey is a stable sha-256 hash of the base64 bytes", () => { + const key = imageDescriptionKey("aGVsbG8="); + + assert.equal(imageDescriptionKey("aGVsbG8="), key, "same bytes produce the same key"); + assert.match(key, /^[0-9a-f]{64}$/, "key is a 64-char hex sha-256 digest"); + assert.notEqual(imageDescriptionKey("aGVsbG8="), imageDescriptionKey("d29ybGQ="), "different bytes produce different keys"); + }); + + it("lookupImageDescriptions returns undefined when nothing is cached", () => { + clearImageDescriptionCache(); + assert.equal(lookupImageDescriptions([imageDescriptionKey("aGVsbG8=")]), undefined); + }); + + it("stores and looks up a description under every image hash", () => { + clearImageDescriptionCache(); + const h1 = imageDescriptionKey("aGVsbG8="); + const h2 = imageDescriptionKey("d29ybGQ="); + const description = "A red circle on a blue background."; + + storeImageDescriptions([h1, h2], description); + + assert.equal(lookupImageDescriptions([h1]), description); + assert.equal(lookupImageDescriptions([h2]), description); + assert.equal(lookupImageDescriptions([h1, h2]), description); + }); + + it("lookupImageDescriptions returns undefined when only some hashes are cached", () => { + clearImageDescriptionCache(); + const h1 = imageDescriptionKey("aGVsbG8="); + const h2 = imageDescriptionKey("d29ybGQ="); + + storeImageDescriptions([h1], "Only one image was described."); + + assert.equal(lookupImageDescriptions([h1, h2]), undefined); + }); + + it("reuses the cached description instead of re-describing (same image twice)", () => { + clearImageDescriptionCache(); + const hash = imageDescriptionKey("cmV1c2UtbWU="); + const description = "Description cached on the first turn."; + + // Simulates turn 1: not cached → described and stored. + assert.equal(lookupImageDescriptions([hash]), undefined); + storeImageDescriptions([hash], description); + + // Simulates turn 2: same image → cached, so no model request is needed. + assert.equal(lookupImageDescriptions([hash]), description); + assert.equal(imageDescriptionCache.size, 1); + }); + + it("evicts the oldest entries once the cache exceeds its limit", () => { + clearImageDescriptionCache(); + const firstKey = imageDescriptionKey("Zmlyc3Q="); + + for (let i = 0; i <= IMAGE_DESCRIPTION_CACHE_LIMIT; i++) { + storeImageDescriptions([imageDescriptionKey(`aW1hZ2UtaW5kZXgt${i}`)], `description-${i}`); + } + + assert.ok(imageDescriptionCache.size <= IMAGE_DESCRIPTION_CACHE_LIMIT, "cache size stays within the limit"); + assert.equal(imageDescriptionCache.has(firstKey), false, "oldest entry is evicted"); + assert.equal(imageDescriptionCache.has(imageDescriptionKey(`aW1hZ2UtaW5kZXgt${IMAGE_DESCRIPTION_CACHE_LIMIT}`)), true, "recent entry is kept"); + }); + + it("clearImageDescriptionCache empties the cache", () => { + clearImageDescriptionCache(); + storeImageDescriptions([imageDescriptionKey("aGVsbG8=")], "hello description"); + assert.equal(imageDescriptionCache.size, 1); + + clearImageDescriptionCache(); + assert.equal(imageDescriptionCache.size, 0); + }); +}); diff --git a/src/visionProxyCache.ts b/src/visionProxyCache.ts new file mode 100644 index 0000000..3207583 --- /dev/null +++ b/src/visionProxyCache.ts @@ -0,0 +1,79 @@ +import { createHash } from "node:crypto"; + +/** + * Cache mapping an image's content hash to the text description produced by + * the vision proxy. + * + * WHY: text-only models receive the same image attachments on EVERY turn of a + * conversation. Without a cache, `proxyVision()` would call the vision model + * via `model.sendRequest()` on each turn to describe the same bytes again — + * wasting Copilot quota, adding latency, and returning a different description + * every time. With this cache, once an image has been described, future turns + * reuse that description and never re-call the vision model. + * + * Key = SHA-256 of the image's base64 bytes (produced by `dataPartToBase64`), + * so the cache stays small even for large images. + * Value = the text description returned by the vision proxy. + * + * When a message contains several images, the combined description is stored + * under EVERY image hash, mirroring how `proxyVision()` describes a message as + * one unit. + */ +export const imageDescriptionCache = new Map(); + +/** Cap on cache entries to prevent unbounded memory growth across sessions. */ +export const IMAGE_DESCRIPTION_CACHE_LIMIT = 200; + +/** + * Stable key for an image's bytes. We hash the base64 string (as produced by + * `dataPartToBase64`) rather than keeping the string itself, which keeps memory + * usage small for large images. + */ +export function imageDescriptionKey(base64: string): string { + return createHash("sha256").update(base64).digest("hex"); +} + +/** + * Return the cached description for a set of image hashes, or `undefined` when + * ANY hash is missing. When multiple images were described together, the same + * combined description is stored under every hash, so all stored values are + * identical and we return the first one. + */ +export function lookupImageDescriptions(hashes: readonly string[]): string | undefined { + if (hashes.length === 0) { + return undefined; + } + const first = imageDescriptionCache.get(hashes[0]); + if (first === undefined) { + return undefined; + } + for (let index = 1; index < hashes.length; index++) { + if (!imageDescriptionCache.has(hashes[index])) { + return undefined; + } + } + return first; +} + +/** + * Store a description under every image hash. Once the cache exceeds + * {@link IMAGE_DESCRIPTION_CACHE_LIMIT}, the oldest entries are evicted (FIFO), + * mirroring the reasoning-content cache's eviction strategy. + */ +export function storeImageDescriptions(hashes: readonly string[], description: string): void { + for (const hash of hashes) { + imageDescriptionCache.set(hash, description); + } + while (imageDescriptionCache.size > IMAGE_DESCRIPTION_CACHE_LIMIT) { + const oldest = imageDescriptionCache.keys().next().value; + if (oldest === undefined) { + break; + } + imageDescriptionCache.delete(oldest); + } +} + +/** Test helper: drop all cached entries. */ +export function clearImageDescriptionCache(): void { + imageDescriptionCache.clear(); +}