diff --git a/.changeset/preserve-image-output-capacity.md b/.changeset/preserve-image-output-capacity.md new file mode 100644 index 00000000000..e3c196e64c5 --- /dev/null +++ b/.changeset/preserve-image-output-capacity.md @@ -0,0 +1,5 @@ +--- +"@kilocode/cli": patch +--- + +Preserve model output capacity when requests contain encoded images. The output token cap now uses the provider-reported context size from the previous turn, so image and vision input is measured by the provider instead of by encoded payload size. diff --git a/bun.lock b/bun.lock index aaae54f60e9..cdded3646b6 100644 --- a/bun.lock +++ b/bun.lock @@ -721,16 +721,16 @@ }, }, "trustedDependencies": [ - "esbuild", "tree-sitter-powershell", - "protobufjs", "web-tree-sitter", "tree-sitter-bash", + "esbuild", + "protobufjs", ], "patchedDependencies": { - "mammoth@1.12.0": "patches/mammoth@1.12.0.patch", - "@standard-community/standard-openapi@0.2.9": "patches/@standard-community%2Fstandard-openapi@0.2.9.patch", "@silvia-odwyer/photon-node@0.3.4": "patches/@silvia-odwyer%2Fphoton-node@0.3.4.patch", + "@standard-community/standard-openapi@0.2.9": "patches/@standard-community%2Fstandard-openapi@0.2.9.patch", + "mammoth@1.12.0": "patches/mammoth@1.12.0.patch", }, "overrides": { "@effect/platform-node-shared": "4.0.0-beta.46", diff --git a/packages/opencode/src/kilocode/session/llm.ts b/packages/opencode/src/kilocode/session/llm.ts index 2a40816f872..c6627a28b9f 100644 --- a/packages/opencode/src/kilocode/session/llm.ts +++ b/packages/opencode/src/kilocode/session/llm.ts @@ -39,7 +39,15 @@ export namespace KiloLLM { /** * Caps `maxOutputTokens` to fit within the model's context window after - * accounting for the actual estimated input tokens (messages + tool schemas). + * accounting for the context the outgoing request will consume. + * + * Like opencode, the provider is the source of truth: when the last finished + * turn reported usage, `reported` carries that provider-tokenized context size + * (input + output + cache), which already accounts for image/vision input the + * client cannot see. The client-side media-normalized estimate (encoded bytes + * excluded) is used as a floor so newly added text or tool schemas still cap + * output, and as the sole basis on the first turn before any usage is reported. + * The larger of the two is used so the cap never under-counts. * * Many small models (e.g. qwen 7B, 32K context) ship with a default * max_output of 32K, leaving no room for input once tools are included. @@ -51,14 +59,18 @@ export namespace KiloLLM { messages: ModelMessage[] tools: Record configured: number | undefined - tokens?: number + usage?: ReturnType + reported?: number }): number | undefined { if (input.configured == null) return input.configured if (input.configured <= 0) return undefined const { context } = input.model.limit if (!context) return input.configured - const tokens = input.tokens ?? KiloSessionOverflow.measure({ messages: input.messages, tools: input.tools }).raw + const estimated = + input.usage?.normalized ?? + KiloSessionOverflow.measure({ messages: input.messages, tools: input.tools }).normalized + const tokens = Math.max(input.reported ?? 0, estimated) const available = context - tokens - SAFETY // If available is ≤0 the input alone exceeds context — return the original // value so the provider returns a natural overflow error which triggers diff --git a/packages/opencode/src/session/llm.ts b/packages/opencode/src/session/llm.ts index bfacd5524e2..917fc6f43ec 100644 --- a/packages/opencode/src/session/llm.ts +++ b/packages/opencode/src/session/llm.ts @@ -52,6 +52,7 @@ export type StreamInput = { retries?: number toolChoice?: "auto" | "required" | "none" preflight?: boolean // kilocode_change - enable proactive threshold compaction for normal session turns + reportedContextTokens?: number // kilocode_change - provider-reported context size from the last finished turn, source of truth for the output cap } export type StreamRequest = StreamInput & { @@ -141,7 +142,8 @@ const live: Layer.Layer< messages: estimated, tools: base.tools, configured: base.params.maxOutputTokens, - tokens: usage?.raw, + usage, + reported: input.reportedContextTokens, }) if ( preflight && diff --git a/packages/opencode/src/session/prompt.ts b/packages/opencode/src/session/prompt.ts index ee6e8b7b220..12c8f5bb738 100644 --- a/packages/opencode/src/session/prompt.ts +++ b/packages/opencode/src/session/prompt.ts @@ -6,6 +6,7 @@ import { KiloSessionPromptQueue } from "@/kilocode/session/prompt-queue" // kilo import { KiloSession } from "@/kilocode/session" // kilocode_change import { KiloCostPropagation } from "@/kilocode/session/cost-propagation" // kilocode_change import { KiloSessionProcessor } from "@/kilocode/session/processor" // kilocode_change +import { KiloSessionOverflow } from "@/kilocode/session/overflow" // kilocode_change import { CommandTimeout } from "@/kilocode/command-timeout" // kilocode_change import { Suggestion } from "@/kilocode/suggestion" // kilocode_change import { Question } from "@/question" // kilocode_change @@ -1684,6 +1685,16 @@ export const layer = Layer.effect( tools, model, toolChoice: format.type === "json_schema" ? "required" : undefined, + // kilocode_change start - feed the provider-reported context size from the last finished + // turn into the output-token cap, so image/vision input is measured by the provider + // rather than by encoded payload bytes (see KiloLLM.capOutputTokens). Summary messages + // are skipped like in the isOverflow check above: their reported input reflects the + // pre-compaction history, not the trimmed context of the next request. + reportedContextTokens: + lastFinished && lastFinished.summary !== true + ? KiloSessionOverflow.count(lastFinished.tokens) + : undefined, + // kilocode_change end }) if (structured !== undefined) { diff --git a/packages/opencode/test/kilocode/session-overflow.test.ts b/packages/opencode/test/kilocode/session-overflow.test.ts index f2a9df0434c..1c5075c0085 100644 --- a/packages/opencode/test/kilocode/session-overflow.test.ts +++ b/packages/opencode/test/kilocode/session-overflow.test.ts @@ -134,6 +134,65 @@ describe("Kilo request estimation", () => { expect(KiloLLM.needsEstimate({ model: model({ context: 0, output: 32_000 }), configured: 32_000 })).toBe(false) expect(KiloLLM.needsEstimate({ model: mdl, configured: 32_000 })).toBe(true) }) + + test("does not reduce output for encoded media payload size", () => { + const mdl = model({ context: 200_000, output: 32_000 }) + const messages = [ + { + role: "user", + content: [{ type: "image", image: `data:image/png;base64,${"x".repeat(600_000)}` }], + }, + ] satisfies ModelMessage[] + const usage = KiloSessionOverflow.measure({ messages, tools: {} }) + + expect(usage.raw).toBeGreaterThan(usage.normalized) + expect(KiloLLM.capOutputTokens({ model: mdl, messages, tools: {}, configured: 32_000, usage })).toBe(32_000) + }) + + test("still reduces output for oversized text", () => { + const mdl = model({ context: 200_000, output: 32_000 }) + const messages = [{ role: "user" as const, content: "x".repeat(600_000) }] + + const cap = KiloLLM.capOutputTokens({ model: mdl, messages, tools: {}, configured: 32_000 }) + expect(cap).toBeGreaterThanOrEqual(1_024) + expect(cap).toBeLessThan(32_000) + }) + + test("prefers provider-reported context over the client estimate for images", () => { + // The client cannot price encoded image bytes, but the provider reported a + // large vision-token cost for the last turn. + const mdl = model({ context: 300_000, output: 32_000 }) + const messages = [ + { + role: "user", + content: [{ type: "image", image: `data:image/png;base64,${"x".repeat(600_000)}` }], + }, + ] satisfies ModelMessage[] + + // Without reported usage the media-normalized estimate leaves output untouched. + expect(KiloLLM.capOutputTokens({ model: mdl, messages, tools: {}, configured: 32_000 })).toBe(32_000) + + // With the provider-reported context size, output is capped to fit real usage. + expect(KiloLLM.capOutputTokens({ model: mdl, messages, tools: {}, configured: 32_000, reported: 280_000 })).toBe( + 17_952, + ) + }) + + test("uses the media-normalized floor when reported usage is smaller", () => { + const mdl = model({ context: 200_000, output: 32_000 }) + const messages = [{ role: "user" as const, content: "x".repeat(600_000) }] + + const withoutReported = KiloLLM.capOutputTokens({ model: mdl, messages, tools: {}, configured: 32_000 }) + const withStaleReported = KiloLLM.capOutputTokens({ + model: mdl, + messages, + tools: {}, + configured: 32_000, + reported: 1_000, + }) + expect(withStaleReported).toBe(withoutReported) + expect(withStaleReported).toBeLessThan(32_000) + }) }) describe("Kilo preflight compaction", () => {