From e43c01b31606ff78e9e770880e85ea0bab703094 Mon Sep 17 00:00:00 2001 From: Wallacy Date: Mon, 3 Aug 2026 10:18:50 -0300 Subject: [PATCH 1/2] fix(provider): improve Kimi context, model names, and image attachments (#87, #92, #94) --- .vscodeignore | 3 + CHANGELOG.md | 11 ++ README.md | 1 + ...44-20260803-issue94-image-normalization.md | 16 +++ .../45-20260803-issue92-provider-prefix.md | 9 ++ .../46-20260803-issue87-kimi-context-size.md | 9 ++ package-lock.json | 9 ++ package.json | 8 ++ src/extension.ts | 76 +++++++----- src/imageNormalizer.ts | 108 ++++++++++++++++++ src/metadata.ts | 40 ++++++- src/modelNames.ts | 35 ++++++ src/test/imageNormalizer.test.ts | 37 ++++++ src/test/metadata.test.ts | 46 +++++++- src/test/modelNames.test.ts | 23 ++++ 15 files changed, 399 insertions(+), 32 deletions(-) create mode 100644 docs/issues/44-20260803-issue94-image-normalization.md create mode 100644 docs/issues/45-20260803-issue92-provider-prefix.md create mode 100644 docs/issues/46-20260803-issue87-kimi-context-size.md create mode 100644 src/imageNormalizer.ts create mode 100644 src/modelNames.ts create mode 100644 src/test/imageNormalizer.test.ts create mode 100644 src/test/modelNames.test.ts diff --git a/.vscodeignore b/.vscodeignore index 3288a92..7fbc504 100644 --- a/.vscodeignore +++ b/.vscodeignore @@ -2,6 +2,9 @@ .vscode/** .vscode-test/** node_modules/** +!node_modules/@silvia-odwyer/ +!node_modules/@silvia-odwyer/photon-node/ +!node_modules/@silvia-odwyer/photon-node/** src/** *.vsix tsconfig.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fcfd5c..f5ee894 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ All notable changes to the **OpenCode Go BYOK Provider** extension are documented here. +## [Unreleased] + +### Added + +- **`[Model Picker]` Optional provider prefixes (#92).** Added `opencodego.showProviderPrefix` (default `true`) to hide `OpenCode Go` / `OpenCode Zen` prefixes in narrow model pickers when desired. Changes refresh the registered model names immediately. +- **`[Model Picker]` Kimi context-size selector (#87).** Kimi models with a context window larger than 256K now expose `256K` and the full window in the per-model configuration schema, with the smaller tier selected by default. + +### Fixed + +- **`[Vision]` Normalize image attachments before OpenCode Go requests (#94).** Image data URLs are resized and re-encoded using the same 2000x2000 / 5 MB base64 limits used by OpenCode's CLI, while preserving the original image when normalization is unavailable. + ## [0.4.5] — 2026-08-03 ### Fixed diff --git a/README.md b/README.md index 891d707..77935be 100644 --- a/README.md +++ b/README.md @@ -326,6 +326,7 @@ To manage agent API keys separately or see agent vendors in the Manage panel, en | `opencodego.requestTimeoutSeconds` | `600` | Total request timeout | | `opencodego.streamIdleTimeoutSeconds` | `120` | Cancel if stream goes idle | | `opencodego.showUsageStatusBar` | `true` | Show usage summary in status bar | +| `opencodego.showProviderPrefix` | `true` | Include `OpenCode Go` / `OpenCode Zen` in model names | | `opencodego.freeOnly` | `true` | Zen: free models only. `false` = include paid | | `opencodego.agentsWindow` | `true` | Expose agent-host model variants (`targetChatSessionType`) for the Agents window | | `opencodego.showAgentModelsInManagePanel` | `false` | Show agent vendors in Manage Language Models panel | diff --git a/docs/issues/44-20260803-issue94-image-normalization.md b/docs/issues/44-20260803-issue94-image-normalization.md new file mode 100644 index 0000000..52b00da --- /dev/null +++ b/docs/issues/44-20260803-issue94-image-normalization.md @@ -0,0 +1,16 @@ +**Status:** Implemented on `fix/open-issues-87-92-94` + +# Image Attachment Normalization (#94) + +OpenCode Go rejected some image attachments even though the same model and +image worked in the OpenCode CLI. The extension sent the original bytes as an +OpenAI `image_url` data URI, while the CLI normalizes images before sending. + +The request path now normalizes image data URLs with the same practical limits +used by OpenCode: a maximum `2000x2000` image size and a `5 MB` base64 payload. +It tries PNG first, then JPEG quality levels. If decoding or the optional +normalizer fails, the original data URI is preserved so the provider behavior +does not regress just because normalization is unavailable. + +The existing top-level and tool-result byte guards remain in place as a second +line of defense against oversized conversation payloads. diff --git a/docs/issues/45-20260803-issue92-provider-prefix.md b/docs/issues/45-20260803-issue92-provider-prefix.md new file mode 100644 index 0000000..40a1a3e --- /dev/null +++ b/docs/issues/45-20260803-issue92-provider-prefix.md @@ -0,0 +1,9 @@ +**Status:** Implemented on `fix/open-issues-87-92-94` + +# Optional Provider Prefixes (#92) + +Model names in the picker now respect the `opencodego.showProviderPrefix` +setting. It defaults to `true`, preserving names such as +`OpenCode Go / DeepSeek V4 Flash`; setting it to `false` displays only the +formatted model name. Changing the setting refreshes Go, Zen, and agent-host +model registrations without requiring a reload. diff --git a/docs/issues/46-20260803-issue87-kimi-context-size.md b/docs/issues/46-20260803-issue87-kimi-context-size.md new file mode 100644 index 0000000..9c6fa03 --- /dev/null +++ b/docs/issues/46-20260803-issue87-kimi-context-size.md @@ -0,0 +1,9 @@ +**Status:** Implemented on `fix/open-issues-87-92-94` + +# Kimi Context Size Picker (#87) + +The model configuration schema now exposes `256K` and the full context window +for Kimi models whose resolved metadata advertises more than `256K` (for +example, Kimi K3). The `256K` choice is the default/cheaper tier. Explicit +context tiers from `models.dev` still take precedence, and fixed-size Kimi +models do not receive a redundant selector. diff --git a/package-lock.json b/package-lock.json index db7d8fc..80d41e9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,9 @@ "name": "opencode-copilot-chat", "version": "0.4.5", "license": "MIT", + "dependencies": { + "@silvia-odwyer/photon-node": "^0.3.4" + }, "devDependencies": { "@types/node": "^26.1.0", "@types/vscode": "^1.125.0", @@ -440,6 +443,12 @@ "node": ">=20.0.0" } }, + "node_modules/@silvia-odwyer/photon-node": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/@silvia-odwyer/photon-node/-/photon-node-0.3.4.tgz", + "integrity": "sha512-bnly4BKB3KDTFxrUIcgCLbaeVVS8lrAkri1pEzskpmxu9MdfGQTy8b8EgcD83ywD3RPMsIulY8xJH5Awa+t9fA==", + "license": "Apache-2.0" + }, "node_modules/@sindresorhus/merge-streams": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/@sindresorhus/merge-streams/-/merge-streams-2.3.0.tgz", diff --git a/package.json b/package.json index e4aa4ba..3357449 100644 --- a/package.json +++ b/package.json @@ -167,6 +167,11 @@ "default": true, "description": "Show the latest OpenCode prompt/output/cache usage summary in the VS Code status bar after each response." }, + "opencodego.showProviderPrefix": { + "type": "boolean", + "default": true, + "description": "Show the OpenCode Go or OpenCode Zen provider prefix in model names." + }, "opencodego.freeOnly": { "type": "boolean", "default": true, @@ -331,5 +336,8 @@ "@types/vscode": "^1.125.0", "@vscode/vsce": "^3.9.2", "typescript": "^6.0.3" + }, + "dependencies": { + "@silvia-odwyer/photon-node": "^0.3.4" } } diff --git a/src/extension.ts b/src/extension.ts index f289573..8a8ee10 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -8,7 +8,7 @@ import { MODELS_DEV_API_URL, bundledModelMetadataSnapshot, fallbackModelMetadata, - getContextSizeOptions, + getContextSizeOptionsForModel, hasExplicitModelLimits, isFreshModelMetadata, normalizeLiveModelMetadata, @@ -46,6 +46,8 @@ import { } from "./streaming"; import { GO_VENDOR, ZEN_VENDOR, AGENT_GO_VENDOR, AGENT_ZEN_VENDOR, resolveBaseVendor, type AllProviderVendor, type ProviderVendor } from "./providerTypes"; import { isInternalDataPart } from "./chatParts"; +import { normalizeImageDataUrl } from "./imageNormalizer"; +import { providerModelDisplayName } from "./modelNames"; import { formatCacheHitRatio, @@ -735,6 +737,7 @@ export function activate(context: vscode.ExtensionContext) { ensureGoUsageStatusBar(context); const goProvider = new OpenCodeProvider(context, PROVIDERS[GO_VENDOR]); const zenProvider = new OpenCodeProvider(context, PROVIDERS[ZEN_VENDOR]); + const modelInfoProviders: OpenCodeProvider[] = [goProvider, zenProvider]; const subscriptions: vscode.Disposable[] = [ vscode.lm.registerLanguageModelChatProvider(GO_VENDOR, goProvider), @@ -900,6 +903,7 @@ export function activate(context: vscode.ExtensionContext) { if (enableAgents) { const agentGoProvider = new OpenCodeProvider(context, PROVIDERS[AGENT_GO_VENDOR]); const agentZenProvider = new OpenCodeProvider(context, PROVIDERS[AGENT_ZEN_VENDOR]); + modelInfoProviders.push(agentGoProvider, agentZenProvider); subscriptions.push( vscode.lm.registerLanguageModelChatProvider(AGENT_GO_VENDOR, agentGoProvider), vscode.lm.registerLanguageModelChatProvider(AGENT_ZEN_VENDOR, agentZenProvider), @@ -913,6 +917,11 @@ export function activate(context: vscode.ExtensionContext) { if (event.affectsConfiguration("opencodego.showUsageStatusBar")) { resetUsageStatusBar(); } + if (event.affectsConfiguration("opencodego.showProviderPrefix")) { + for (const provider of modelInfoProviders) { + provider.notifyModelInfoChanged(); + } + } }), ); @@ -1929,6 +1938,9 @@ class OpenCodeProvider implements vscode.LanguageModelChatProvider("showProviderPrefix", true); // CONTRACT: VS Code calls provideLanguageModelChatInformation frequently // (every ~300ms during UI refresh). Per-model logging produces thousands @@ -1962,7 +1974,7 @@ class OpenCodeProvider implements vscode.LanguageModelChatProvider = { rawModelId: modelId, - name: `${this.definition.modelNamePrefix} / ${formatModelName(modelId)}`, + name: providerModelDisplayName(this.definition.modelNamePrefix, modelId, showProviderPrefix), family: `${this.definition.isAgentVariant && this.definition.baseVendor ? this.definition.baseVendor : this.definition.vendor}-${modelId}-${MODEL_METADATA_REVISION}`, // Include effective limits in version so VS Code invalidates stale // picker metadata after limit changes (eg. 2M -> 262K corrections). @@ -2047,6 +2059,10 @@ class OpenCodeProvider implements vscode.LanguageModelChatProvider convertMessage(message, this.reasoningContentByToolCallId, rawModelId))); + const normalizedImageCount = await normalizeImagePartsInPlace(apiMessages); + if (normalizedImageCount > 0) { + this.log(`[vision] Normalized ${normalizedImageCount} image attachment(s) to provider-safe dimensions/encoding.`); + } const baseSettings = getSettings(); // Apply per-request Thinking selection (from Copilot Chat submenu) on top // of the workspace default. The override only affects the current model @@ -3541,6 +3557,31 @@ function dataPartToBase64(data: Uint8Array): string { return output; } +async function normalizeImagePartsInPlace(messages: ApiMessage[]): Promise { + let normalizedCount = 0; + + for (const message of messages) { + if (!Array.isArray(message.content)) { + continue; + } + + for (const part of message.content) { + const url = part.type === "image_url" ? part.image_url?.url : undefined; + if (!url) { + continue; + } + + const normalizedUrl = await normalizeImageDataUrl(url); + if (normalizedUrl !== url && part.image_url) { + part.image_url.url = normalizedUrl; + normalizedCount += 1; + } + } + } + + return normalizedCount; +} + function reasoningForToolCalls( toolCalls: OpenAiToolCall[], reasoningContentByToolCallId: ReadonlyMap @@ -3818,7 +3859,9 @@ function modelConfigurationSchema( } // --- Context Size (tiered pricing) --- - const contextSizeOptions = metadata?.cost ? getContextSizeOptions(metadata.cost, metadata.contextWindow) : undefined; + const contextSizeOptions = metadata + ? getContextSizeOptionsForModel(modelId, metadata.cost, metadata.contextWindow) + : undefined; if (contextSizeOptions && contextSizeOptions.length > 0) { properties.contextSize = { type: "number", @@ -4361,33 +4404,6 @@ function costCategory(cost: { input: number; output: number }): string { return "very_high"; } -function formatModelName(modelId: string): string { - const parts = modelId.split("-"); - const displayParts: string[] = []; - - for (let index = 0; index < parts.length; index += 1) { - const part = parts[index]; - - if (/^\d+$/.test(part) && /^\d+$/.test(parts[index + 1] ?? "")) { - const versionParts = [part]; - - while (/^\d+$/.test(parts[index + 1] ?? "")) { - versionParts.push(parts[index + 1]); - index += 1; - } - - displayParts.push(versionParts.join(".")); - continue; - } - - displayParts.push(part); - } - - return displayParts - .map((part) => part.toUpperCase() === part ? part : part.charAt(0).toUpperCase() + part.slice(1)) - .join(" "); -} - function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null; } diff --git a/src/imageNormalizer.ts b/src/imageNormalizer.ts new file mode 100644 index 0000000..f20f488 --- /dev/null +++ b/src/imageNormalizer.ts @@ -0,0 +1,108 @@ +import type * as Photon from "@silvia-odwyer/photon-node"; + +const MAX_IMAGE_WIDTH = 2_000; +const MAX_IMAGE_HEIGHT = 2_000; +const MAX_BASE64_BYTES = 5 * 1024 * 1024; +const JPEG_QUALITIES = [80, 85, 70, 55, 40] as const; + +type PhotonModule = typeof Photon; + +let photonModulePromise: Promise | undefined; + +function loadPhoton(): Promise { + photonModulePromise ??= import("@silvia-odwyer/photon-node"); + return photonModulePromise; +} + +function parseBase64DataUrl(url: string): { mime: string; base64: string } | undefined { + const match = /^data:([^;,]+);base64,(.+)$/is.exec(url); + if (!match) { + return undefined; + } + + return { mime: match[1], base64: match[2] }; +} + +function candidateSizes(width: number, height: number): Array<{ width: number; height: number }> { + const scale = Math.min(1, MAX_IMAGE_WIDTH / width, MAX_IMAGE_HEIGHT / height); + return Array.from({ length: 32 }).reduce>((sizes) => { + const previous = sizes.at(-1) ?? { + width: Math.max(1, Math.round(width * scale)), + height: Math.max(1, Math.round(height * scale)), + }; + const next = sizes.length === 0 + ? previous + : { + width: previous.width === 1 ? 1 : Math.max(1, Math.floor(previous.width * 0.75)), + height: previous.height === 1 ? 1 : Math.max(1, Math.floor(previous.height * 0.75)), + }; + + return sizes.some((size) => size.width === next.width && size.height === next.height) + ? sizes + : [...sizes, next]; + }, []); +} + +/** + * Mirrors OpenCode's image normalization before an image reaches a provider: + * resize oversized dimensions and try PNG/JPEG encodings until the base64 + * payload fits. Unsupported or malformed images are passed through unchanged. + */ +export async function normalizeImageDataUrl(url: string): Promise { + const parsed = parseBase64DataUrl(url); + if (!parsed) { + return url; + } + + let photon: PhotonModule; + try { + photon = await loadPhoton(); + } catch { + return url; + } + + let decoded: Photon.PhotonImage; + try { + decoded = photon.PhotonImage.new_from_byteslice(Buffer.from(parsed.base64, "base64")); + } catch { + return url; + } + + try { + const width = decoded.get_width(); + const height = decoded.get_height(); + const base64Bytes = Buffer.byteLength(parsed.base64, "utf8"); + + if (width <= MAX_IMAGE_WIDTH && height <= MAX_IMAGE_HEIGHT && base64Bytes <= MAX_BASE64_BYTES) { + return url; + } + + for (const size of candidateSizes(width, height)) { + const resized = photon.resize(decoded, size.width, size.height, photon.SamplingFilter.Lanczos3); + try { + const candidates: Array<{ mime: string; bytes: Uint8Array }> = [ + { mime: "image/png", bytes: resized.get_bytes() }, + ...JPEG_QUALITIES.map((quality) => ({ + mime: "image/jpeg", + bytes: resized.get_bytes_jpeg(quality), + })), + ]; + + for (const candidate of candidates) { + const base64 = Buffer.from(candidate.bytes).toString("base64"); + if (Buffer.byteLength(base64, "utf8") <= MAX_BASE64_BYTES) { + return `data:${candidate.mime};base64,${base64}`; + } + } + } finally { + resized.free(); + } + } + } catch { + return url; + } finally { + decoded.free(); + } + + return url; +} diff --git a/src/metadata.ts b/src/metadata.ts index bae1458..f468c18 100644 --- a/src/metadata.ts +++ b/src/metadata.ts @@ -671,6 +671,44 @@ export function getContextSizeOptions( return options; } +/** + * Resolve context-size options for a concrete model. Kimi K3 exposes a + * cheaper 256K context mode alongside its larger window, but models.dev does + * not consistently publish that distinction as a pricing tier. + */ +export function getContextSizeOptionsForModel( + modelId: string, + cost: ModelCost | undefined, + fullContextWindow: number, +): ContextSizeOption[] | undefined { + const metadataOptions = getContextSizeOptions(cost, fullContextWindow); + if (metadataOptions?.length) { + return metadataOptions; + } + + const kimiBaseContext = 256_000; + // 262,144 is the binary representation commonly used for a 256K window; + // do not expose a fake second tier for models whose whole window is 256K. + if (!/^(?:kimi-|k3(?:-|$))/i.test(modelId) || fullContextWindow <= 262_144) { + return undefined; + } + + return [ + { + value: kimiBaseContext, + label: formatContextSize(kimiBaseContext), + description: "Default pricing", + isDefault: true, + }, + { + value: fullContextWindow, + label: formatContextSize(fullContextWindow), + description: "Higher pricing", + isDefault: false, + }, + ]; +} + function formatContextSize(size: number): string { if (size >= 1_000_000) { const m = size / 1_000_000; @@ -681,4 +719,4 @@ function formatContextSize(size: number): string { return k === Math.floor(k) ? `${k}K` : `${k.toFixed(1)}K`; } return String(size); -} \ No newline at end of file +} diff --git a/src/modelNames.ts b/src/modelNames.ts new file mode 100644 index 0000000..80961c4 --- /dev/null +++ b/src/modelNames.ts @@ -0,0 +1,35 @@ +export function formatModelName(modelId: string): string { + const parts = modelId.split("-"); + const displayParts: string[] = []; + + for (let index = 0; index < parts.length; index += 1) { + const part = parts[index]; + + if (/^\d+$/.test(part) && /^\d+$/.test(parts[index + 1] ?? "")) { + const versionParts = [part]; + + while (/^\d+$/.test(parts[index + 1] ?? "")) { + versionParts.push(parts[index + 1]); + index += 1; + } + + displayParts.push(versionParts.join(".")); + continue; + } + + displayParts.push(part); + } + + return displayParts + .map((part) => part.toUpperCase() === part ? part : part.charAt(0).toUpperCase() + part.slice(1)) + .join(" "); +} + +export function providerModelDisplayName( + providerPrefix: string, + modelId: string, + showProviderPrefix = true, +): string { + const modelName = formatModelName(modelId); + return showProviderPrefix ? `${providerPrefix} / ${modelName}` : modelName; +} diff --git a/src/test/imageNormalizer.test.ts b/src/test/imageNormalizer.test.ts new file mode 100644 index 0000000..bee6ba4 --- /dev/null +++ b/src/test/imageNormalizer.test.ts @@ -0,0 +1,37 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { PhotonImage } from "@silvia-odwyer/photon-node"; +import { normalizeImageDataUrl } from "../imageNormalizer.js"; + +const ONE_PIXEL_PNG = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="; + +describe("normalizeImageDataUrl", () => { + it("keeps a small image unchanged", async () => { + const url = `data:image/png;base64,${ONE_PIXEL_PNG}`; + assert.equal(await normalizeImageDataUrl(url), url); + }); + + it("resizes an image that exceeds the CLI dimension limit", async () => { + const image = new PhotonImage(new Uint8Array(2_001 * 4).fill(255), 2_001, 1); + try { + const url = `data:image/png;base64,${Buffer.from(image.get_bytes()).toString("base64")}`; + const normalized = await normalizeImageDataUrl(url); + + assert.notEqual(normalized, url); + assert.match(normalized, /^data:image\/(png|jpeg);base64,/); + } finally { + image.free(); + } + }); + + it("passes non-data URLs through unchanged", async () => { + const url = "https://example.com/image.png"; + assert.equal(await normalizeImageDataUrl(url), url); + }); + + it("passes malformed image data through unchanged", async () => { + const url = "data:image/png;base64,not-an-image"; + assert.equal(await normalizeImageDataUrl(url), url); + }); +}); diff --git a/src/test/metadata.test.ts b/src/test/metadata.test.ts index bc96f39..44545d0 100644 --- a/src/test/metadata.test.ts +++ b/src/test/metadata.test.ts @@ -1,6 +1,10 @@ import { describe, it } from "node:test"; import assert from "node:assert/strict"; -import { fallbackModelMetadata, VISION_CAPABLE_MODELS } from "../metadata.js"; +import { + fallbackModelMetadata, + getContextSizeOptionsForModel, + VISION_CAPABLE_MODELS, +} from "../metadata.js"; import { GO_VENDOR, ZEN_VENDOR } from "../providerTypes.js"; /** @@ -94,3 +98,43 @@ describe("VISION_CAPABLE_MODELS", () => { assert.ok(VISION_CAPABLE_MODELS.size > 10); }); }); + +describe("getContextSizeOptionsForModel — Kimi context tiers (issue #87)", () => { + it("offers 256K and the full window when Kimi has a larger context", () => { + const options = getContextSizeOptionsForModel( + "kimi-k3", + { input: 3, output: 15 }, + 1_048_576, + ); + + assert.deepEqual(options?.map((option) => option.value), [256_000, 1_048_576]); + assert.equal(options?.[0].isDefault, true); + assert.equal(options?.[1].description, "Higher pricing"); + }); + + it("recognizes the official short K3 model id", () => { + const options = getContextSizeOptionsForModel("k3", undefined, 1_000_000); + assert.deepEqual(options?.map((option) => option.value), [256_000, 1_000_000]); + }); + + it("does not add a redundant tier to a 256K Kimi model", () => { + assert.equal( + getContextSizeOptionsForModel("kimi-k2.6", { input: 0.95, output: 4 }, 262_144), + undefined, + ); + }); + + it("prefers explicit models.dev pricing tiers", () => { + const options = getContextSizeOptionsForModel( + "kimi-k3", + { + input: 3, + output: 15, + tiers: [{ input: 3, output: 15, tier: { type: "context", size: 200_000 } }], + }, + 1_048_576, + ); + + assert.deepEqual(options?.map((option) => option.value), [200_000, 1_048_576]); + }); +}); diff --git a/src/test/modelNames.test.ts b/src/test/modelNames.test.ts new file mode 100644 index 0000000..66b84d3 --- /dev/null +++ b/src/test/modelNames.test.ts @@ -0,0 +1,23 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { formatModelName, providerModelDisplayName } from "../modelNames.js"; + +describe("provider model display names", () => { + it("formats numeric model versions like the existing picker", () => { + assert.equal(formatModelName("gpt-5-6-luna"), "Gpt 5.6 Luna"); + }); + + it("includes the provider prefix by default", () => { + assert.equal( + providerModelDisplayName("OpenCode Go", "kimi-k3"), + "OpenCode Go / Kimi K3", + ); + }); + + it("can hide the provider prefix without changing the model name", () => { + assert.equal( + providerModelDisplayName("OpenCode Zen", "kimi-k3", false), + "Kimi K3", + ); + }); +}); From 4572a9f4878b606c26fca5c50475b6bee6cf52ac Mon Sep 17 00:00:00 2001 From: Wallacy Date: Tue, 4 Aug 2026 11:26:10 -0300 Subject: [PATCH 2/2] fix(vision): normalize images before the payload guard (#94) --- ...44-20260803-issue94-image-normalization.md | 10 +- src/extension.ts | 122 +++++++----------- src/imageNormalizer.ts | 51 +++++--- src/test/imageNormalizer.test.ts | 30 ++++- 4 files changed, 116 insertions(+), 97 deletions(-) diff --git a/docs/issues/44-20260803-issue94-image-normalization.md b/docs/issues/44-20260803-issue94-image-normalization.md index 52b00da..3e97344 100644 --- a/docs/issues/44-20260803-issue94-image-normalization.md +++ b/docs/issues/44-20260803-issue94-image-normalization.md @@ -9,8 +9,10 @@ OpenAI `image_url` data URI, while the CLI normalizes images before sending. The request path now normalizes image data URLs with the same practical limits used by OpenCode: a maximum `2000x2000` image size and a `5 MB` base64 payload. It tries PNG first, then JPEG quality levels. If decoding or the optional -normalizer fails, the original data URI is preserved so the provider behavior -does not regress just because normalization is unavailable. +normalizer fails, the original data URI is preserved and the final base64 guard +decides whether it is still safe to send. -The existing top-level and tool-result byte guards remain in place as a second -line of defense against oversized conversation payloads. +Top-level images are normalized before the final `5 MB` base64 guard, so an +image larger than the old `2 MB` raw-byte threshold can still be sent when it +can be reduced successfully. The separate `1 MB` raw-byte guard for tool +results remains in place to bound cumulative MCP screenshot history. diff --git a/src/extension.ts b/src/extension.ts index 8a8ee10..2c71047 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -46,7 +46,11 @@ import { } from "./streaming"; import { GO_VENDOR, ZEN_VENDOR, AGENT_GO_VENDOR, AGENT_ZEN_VENDOR, resolveBaseVendor, type AllProviderVendor, type ProviderVendor } from "./providerTypes"; import { isInternalDataPart } from "./chatParts"; -import { normalizeImageDataUrl } from "./imageNormalizer"; +import { + getImageDataUrlBase64Bytes, + MAX_IMAGE_BASE64_BYTES, + normalizeImageDataUrl, +} from "./imageNormalizer"; import { providerModelDisplayName } from "./modelNames"; import { @@ -485,6 +489,11 @@ interface OpenAiToolCall { }; } +interface ConvertedMessageResult { + messages: ApiMessage[]; + normalizedImageCount: number; +} + interface PendingToolCall { id: string; name: string; @@ -591,28 +600,6 @@ const MAX_TOOL_RESULT_IMAGE_BYTES = 1_000_000; */ const MAX_HISTORY_IMAGES_KEPT = 2; -/** - * Hard upper limit (in bytes of raw image data) for a single top-level image - * attachment pasted or dropped into the chat by the user. Top-level images - * (screenshots, photos) are typically larger than MCP tool-result screenshots, - * so this threshold is intentionally more liberal than the tool-result guard. - * - * Rationale (evidence-based): - * - Anthropic API hard limit: 10 MB per image base64 (5 MB on Bedrock/Vertex). - * - OpenAI API: 512 MB total payload, but upstream models auto-resize to a - * patch budget (1568–2576 px long-edge) so anything larger is wasted. - * - OpenCode Go gateway: limit not published, but verified to reject a - * 3.18 MB payload with HTTP 400 "Upstream request failed" (issue #38). - * - 2 MB raw → ~2.7 MB base64, comfortably under observed rejection point - * while allowing typical user screenshots/photos without false positives. - * - * Larger images are replaced with a placeholder text part so the model still - * knows an image was attached and the user gets an actionable hint to resize. - * Vision-capable models auto-downsample upstream anyway, so there is no value - * in forwarding multi-MB raw image data. - */ -const MAX_TOP_LEVEL_IMAGE_BYTES = 2_000_000; - type CopilotCompatibleCapabilities = vscode.LanguageModelChatCapabilities & { supportsToolCalling: boolean; supportsImageToText: boolean; @@ -2058,8 +2045,13 @@ class OpenCodeProvider implements vscode.LanguageModelChatProvider convertMessage(message, this.reasoningContentByToolCallId, rawModelId))); - const normalizedImageCount = await normalizeImagePartsInPlace(apiMessages); + const convertedMessages = await Promise.all( + messages.map((message) => convertMessage(message, this.reasoningContentByToolCallId, rawModelId)) + ); + const apiMessages = normalizeMessages(convertedMessages.flatMap((result) => result.messages)); + const normalizedImageCount = convertedMessages + .map((result) => result.normalizedImageCount) + .reduce((total, count) => total + count, 0); if (normalizedImageCount > 0) { this.log(`[vision] Normalized ${normalizedImageCount} image attachment(s) to provider-safe dimensions/encoding.`); } @@ -3347,16 +3339,31 @@ function anthropicToolChoice(mode: vscode.LanguageModelChatToolMode): { type: "a return { type: mode === vscode.LanguageModelChatToolMode.Required ? "any" : "auto" }; } -function convertMessage( +async function convertMessage( message: vscode.LanguageModelChatRequestMessage, reasoningContentByToolCallId: ReadonlyMap, rawModelId?: string, -): ApiMessage[] { +): Promise { const role = message.role === vscode.LanguageModelChatMessageRole.Assistant ? "assistant" : "user"; const textParts: string[] = []; const imageParts: OpenAiContentPart[] = []; const toolCalls: OpenAiToolCall[] = []; const toolResults: ApiMessage[] = []; + let normalizedImageCount = 0; + + const normalizeImagePart = async (part: vscode.LanguageModelDataPart): Promise => { + const originalUrl = `data:${part.mimeType};base64,${dataPartToBase64(part.data)}`; + const normalizedUrl = await normalizeImageDataUrl(originalUrl); + if (normalizedUrl !== originalUrl) { + normalizedImageCount += 1; + } + return normalizedUrl; + }; + + const finish = (messages: ApiMessage[]): ConvertedMessageResult => ({ + messages, + normalizedImageCount, + }); for (const part of message.content) { if (part instanceof vscode.LanguageModelToolCallPart) { @@ -3398,10 +3405,10 @@ function convertMessage( ); continue; } - const base64 = dataPartToBase64(resultPart.data); + const imageUrl = await normalizeImagePart(resultPart); toolImageParts.push({ type: "image_url", - image_url: { url: `data:${resultPart.mimeType};base64,${base64}` }, + image_url: { url: imageUrl }, }); continue; } @@ -3458,26 +3465,22 @@ function convertMessage( } if (part instanceof vscode.LanguageModelDataPart && part.mimeType.startsWith("image/")) { - // SIZE GUARD: Top-level images larger than MAX_TOP_LEVEL_IMAGE_BYTES are - // replaced with a placeholder text part. This prevents a single oversized - // pasted image (e.g. 4K screenshot, high-res phone photo) from producing - // a multi-MB base64 payload that triggers upstream 400 "Upstream request - // failed" rejections from OpenCode Go. Vision-capable models auto-resize - // upstream to a patch budget anyway, so there is no fidelity loss in - // practice — the model would have downscaled it regardless. The user - // gets an actionable hint so they can resize and re-attach. - if (part.data.byteLength > MAX_TOP_LEVEL_IMAGE_BYTES) { + // Normalize before the final payload guard. The previous raw-byte guard + // ran first and dropped images that could have been resized or compressed + // into a provider-safe representation. + const imageUrl = await normalizeImagePart(part); + const base64Bytes = getImageDataUrlBase64Bytes(imageUrl); + if (base64Bytes === undefined || base64Bytes > MAX_IMAGE_BASE64_BYTES) { textParts.push( - `[Image attachment omitted: ${part.data.byteLength} bytes exceeds the ` - + `${MAX_TOP_LEVEL_IMAGE_BYTES}-byte limit for top-level attachments. ` - + `Resize or compress the image to under ${Math.floor(MAX_TOP_LEVEL_IMAGE_BYTES / 1_000_000)} MB and re-attach it.]` + `[Image attachment omitted: normalized payload exceeds the ` + + `${Math.floor(MAX_IMAGE_BASE64_BYTES / (1024 * 1024))} MB base64 limit. ` + + `Resize or compress the image and re-attach it.]` ); continue; } - const base64 = dataPartToBase64(part.data); imageParts.push({ type: "image_url", - image_url: { url: `data:${part.mimeType};base64,${base64}` } + image_url: { url: imageUrl } }); continue; } @@ -3522,21 +3525,21 @@ function convertMessage( // Other families (DeepSeek, Kimi, GLM, Qwen, MiniMax) tolerate the echo // and keep it for cross-turn reasoning continuity. const shouldOmitReasoningEcho = rawModelId !== undefined && /^mimo-/i.test(rawModelId); - return [{ + return finish([{ role, content: typeof content === "string" ? content || null : content, reasoning_content: shouldOmitReasoningEcho ? undefined : reasoningForToolCalls(toolCalls, reasoningContentByToolCallId), tool_calls: toolCalls - }]; + }]); } if (toolResults.length) { - return content ? [{ role, content }, ...toolResults] : toolResults; + return finish(content ? [{ role, content }, ...toolResults] : toolResults); } - return [{ role, content }]; + return finish([{ role, content }]); } function dataPartToBase64(data: Uint8Array): string { @@ -3557,31 +3560,6 @@ function dataPartToBase64(data: Uint8Array): string { return output; } -async function normalizeImagePartsInPlace(messages: ApiMessage[]): Promise { - let normalizedCount = 0; - - for (const message of messages) { - if (!Array.isArray(message.content)) { - continue; - } - - for (const part of message.content) { - const url = part.type === "image_url" ? part.image_url?.url : undefined; - if (!url) { - continue; - } - - const normalizedUrl = await normalizeImageDataUrl(url); - if (normalizedUrl !== url && part.image_url) { - part.image_url.url = normalizedUrl; - normalizedCount += 1; - } - } - } - - return normalizedCount; -} - function reasoningForToolCalls( toolCalls: OpenAiToolCall[], reasoningContentByToolCallId: ReadonlyMap diff --git a/src/imageNormalizer.ts b/src/imageNormalizer.ts index f20f488..54f7ceb 100644 --- a/src/imageNormalizer.ts +++ b/src/imageNormalizer.ts @@ -2,7 +2,7 @@ import type * as Photon from "@silvia-odwyer/photon-node"; const MAX_IMAGE_WIDTH = 2_000; const MAX_IMAGE_HEIGHT = 2_000; -const MAX_BASE64_BYTES = 5 * 1024 * 1024; +export const MAX_IMAGE_BASE64_BYTES = 5 * 1024 * 1024; const JPEG_QUALITIES = [80, 85, 70, 55, 40] as const; type PhotonModule = typeof Photon; @@ -23,24 +23,35 @@ function parseBase64DataUrl(url: string): { mime: string; base64: string } | und return { mime: match[1], base64: match[2] }; } +export function getImageDataUrlBase64Bytes(url: string): number | undefined { + const parsed = parseBase64DataUrl(url); + return parsed ? Buffer.byteLength(parsed.base64, "utf8") : undefined; +} + function candidateSizes(width: number, height: number): Array<{ width: number; height: number }> { const scale = Math.min(1, MAX_IMAGE_WIDTH / width, MAX_IMAGE_HEIGHT / height); - return Array.from({ length: 32 }).reduce>((sizes) => { - const previous = sizes.at(-1) ?? { - width: Math.max(1, Math.round(width * scale)), - height: Math.max(1, Math.round(height * scale)), - }; - const next = sizes.length === 0 - ? previous - : { - width: previous.width === 1 ? 1 : Math.max(1, Math.floor(previous.width * 0.75)), - height: previous.height === 1 ? 1 : Math.max(1, Math.floor(previous.height * 0.75)), - }; - - return sizes.some((size) => size.width === next.width && size.height === next.height) - ? sizes - : [...sizes, next]; - }, []); + let nextWidth = Math.max(1, Math.round(width * scale)); + let nextHeight = Math.max(1, Math.round(height * scale)); + const sizes: Array<{ width: number; height: number }> = []; + + while (sizes.length < 32) { + if (sizes.some((size) => size.width === nextWidth && size.height === nextHeight)) { + break; + } + + sizes.push({ width: nextWidth, height: nextHeight }); + + const reducedWidth = nextWidth === 1 ? 1 : Math.max(1, Math.floor(nextWidth * 0.75)); + const reducedHeight = nextHeight === 1 ? 1 : Math.max(1, Math.floor(nextHeight * 0.75)); + if (reducedWidth === nextWidth && reducedHeight === nextHeight) { + break; + } + + nextWidth = reducedWidth; + nextHeight = reducedHeight; + } + + return sizes; } /** @@ -71,9 +82,9 @@ export async function normalizeImageDataUrl(url: string): Promise { try { const width = decoded.get_width(); const height = decoded.get_height(); - const base64Bytes = Buffer.byteLength(parsed.base64, "utf8"); + const base64Bytes = getImageDataUrlBase64Bytes(url) ?? Number.POSITIVE_INFINITY; - if (width <= MAX_IMAGE_WIDTH && height <= MAX_IMAGE_HEIGHT && base64Bytes <= MAX_BASE64_BYTES) { + if (width <= MAX_IMAGE_WIDTH && height <= MAX_IMAGE_HEIGHT && base64Bytes <= MAX_IMAGE_BASE64_BYTES) { return url; } @@ -90,7 +101,7 @@ export async function normalizeImageDataUrl(url: string): Promise { for (const candidate of candidates) { const base64 = Buffer.from(candidate.bytes).toString("base64"); - if (Buffer.byteLength(base64, "utf8") <= MAX_BASE64_BYTES) { + if (Buffer.byteLength(base64, "utf8") <= MAX_IMAGE_BASE64_BYTES) { return `data:${candidate.mime};base64,${base64}`; } } diff --git a/src/test/imageNormalizer.test.ts b/src/test/imageNormalizer.test.ts index bee6ba4..c859ccf 100644 --- a/src/test/imageNormalizer.test.ts +++ b/src/test/imageNormalizer.test.ts @@ -1,7 +1,11 @@ import { describe, it } from "node:test"; import assert from "node:assert/strict"; import { PhotonImage } from "@silvia-odwyer/photon-node"; -import { normalizeImageDataUrl } from "../imageNormalizer.js"; +import { + getImageDataUrlBase64Bytes, + MAX_IMAGE_BASE64_BYTES, + normalizeImageDataUrl, +} from "../imageNormalizer.js"; const ONE_PIXEL_PNG = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="; @@ -25,6 +29,30 @@ describe("normalizeImageDataUrl", () => { } }); + it("does not reject a large raw image when its normalized base64 payload fits", async () => { + const width = 750; + const height = 1_000; + const pixels = new Uint8Array(width * height * 4); + let seed = 0x12345678; + for (let index = 0; index < pixels.length; index += 1) { + seed = (seed * 1_664_525 + 1_013_904_223) >>> 0; + pixels[index] = seed >>> 24; + } + + const image = new PhotonImage(pixels, width, height); + try { + const bytes = image.get_bytes(); + assert.ok(bytes.byteLength > 2_000_000); + const url = `data:image/png;base64,${Buffer.from(bytes).toString("base64")}`; + const normalized = await normalizeImageDataUrl(url); + + assert.equal(normalized, url); + assert.ok(getImageDataUrlBase64Bytes(normalized)! <= MAX_IMAGE_BASE64_BYTES); + } finally { + image.free(); + } + }); + it("passes non-data URLs through unchanged", async () => { const url = "https://example.com/image.png"; assert.equal(await normalizeImageDataUrl(url), url);