Token Optimization — Context Reduction and LLM-Assisted Compression

packages/types/src/global-settings.ts

@@ -232,10 +232,45 @@ export const globalSettingsSchema = z.object({
 	 * Tools in this list will be excluded from prompt generation and rejected at execution time.
 	 */
 	disabledTools: z.array(toolNamesSchema).optional(),
+
+	/**
+	 * Settings for the ToolResultProcessor (tool output compression).
+	 * Controls thresholds for when compression kicks in.
+	 */
+	toolResultProcessorSettings: z
+		.object({
+			/** Master switch — false disables all compression */
+			enabled: z.boolean(),
+			/** Compress read_file results above this many characters (default: 1500) */
+			readFileCharsAbove: z.number(),
+			/** Compress search_files results above this many matches (default: 20) */
+			searchMatchesAbove: z.number(),
+			/** Compress list_files results above this many paths (default: 100) */
+			listFilesCountAbove: z.number(),
+		})
+		.optional(),
+
+	/**
+	 * Zoo Code API key for subscription features (smart compression).
+	 * Stored in VS Code SecretStorage — never in global state.
+	 * Generate at https://zoocode.dev/dashboard/api-tokens
+	 */
+	zooCodeApiKey: z.string().optional(),
+
+	/**
+	 * Zoo Code API base URL for subscription features (smart compression).
+	 * Defaults to "https://zoocode.dev". Override for dev/staging.
+	 */
+	zooCodeBaseUrl: z.string().optional(),
 })
 
 export type GlobalSettings = z.infer<typeof globalSettingsSchema>
 
+/**
+ * Settings for the ToolResultProcessor, extracted from GlobalSettings for convenience.
+ */
+export type ToolResultProcessorSettings = NonNullable<GlobalSettings["toolResultProcessorSettings"]>
+
 export const GLOBAL_SETTINGS_KEYS = globalSettingsSchema.keyof().options
 
 /**
@@ -285,6 +320,7 @@ export const SECRET_STATE_KEYS = [
 // Global secrets that are part of GlobalSettings (not ProviderSettings)
 export const GLOBAL_SECRET_KEYS = [
 	"openRouterImageApiKey", // For image generation
+	"zooCodeApiKey", // Zoo Code subscription API key
 ] as const
 
 // Type for the actual secret storage keys

src/api/index.ts

@@ -46,8 +46,9 @@ export interface ApiHandlerCreateMessageMetadata {
 	 * Task ID used for tracking and provider-specific features:
 	 * - Roo: Sent as X-Roo-Task-ID header
 	 * - Requesty: Sent as trace_id
+	 * Optional — not required for internal compression calls (ZooGatewayApiHandler).
 	 */
-	taskId: string
+	taskId?: string
 	/**
 	 * Current mode slug for provider-specific tracking:
 	 * - Requesty: Sent in extra metadata
@@ -87,6 +88,12 @@ export interface ApiHandlerCreateMessageMetadata {
 	 * Only applies to providers that support function calling restrictions (e.g., Gemini).
 	 */
 	allowedFunctionNames?: string[]
+	/**
+	 * Name of the tool that produced the result being compressed.
+	 * Used by ZooGatewayApiHandler to pass the originating tool name
+	 * to the compression endpoint for logging and prompt tuning.
+	 */
+	toolName?: string
 }
 
 export interface ApiHandler {

src/api/providers/__tests__/prompt-caching.spec.ts

@@ -0,0 +1,320 @@
+// npx vitest run api/providers/__tests__/prompt-caching.spec.ts
+
+/**
+ * Regression tests for prompt caching across providers.
+ *
+ * Findings summary (audited 2026-05-02):
+ *
+ * ANTHROPIC:
+ *   - `cache_control: { type: 'ephemeral' }` is set on the system block for all
+ *     cache-capable models (line 123 of anthropic.ts).
+ *   - The `prompt-caching-2024-07-31` beta header is added via the inner IIFE.
+ *   - `cache_creation_input_tokens` and `cache_read_input_tokens` are extracted
+ *     from the `message_start` event and yielded as `cacheWriteTokens` /
+ *     `cacheReadTokens` in the `ApiStreamUsageChunk`.
+ *
+ * BEDROCK:
+ *   - Uses AWS-native cachePoint blocks (not `cache_control`) via MultiPointStrategy.
+ *   - `supportsAwsPromptCache()` gates caching on `supportsPromptCache` AND
+ *     non-empty `cachableFields` in model info.
+ *   - Cache token fields (`cacheReadInputTokens`, `cacheWriteInputTokens`, and
+ *     their `*TokenCount` aliases) are captured from the `metadata.usage` stream
+ *     event and yielded as `cacheReadTokens` / `cacheWriteTokens`.
+ *
+ * ROO (OpenAI-compatible proxy):
+ *   - No `cache_control` headers — caching is handled server-side by the proxy.
+ *   - Cache metrics surface as `prompt_tokens_details.cached_tokens` (read) and
+ *     `cache_creation_input_tokens` (write) in the final usage chunk.
+ *   - Both are correctly mapped to `cacheReadTokens` / `cacheWriteTokens` in the
+ *     yielded `ApiStreamUsageChunk`.
+ *
+ * STREAM TYPE:
+ *   - `ApiStreamUsageChunk` declares `cacheWriteTokens?: number` and
+ *     `cacheReadTokens?: number` — all providers use these fields consistently.
+ */
+
+import { AnthropicHandler } from "../anthropic"
+import { ApiHandlerOptions } from "../../../shared/api"
+
+// ---------------------------------------------------------------------------
+// Shared mock infrastructure
+// ---------------------------------------------------------------------------
+
+vitest.mock("@roo-code/telemetry", () => ({
+	TelemetryService: {
+		instance: {
+			captureException: vitest.fn(),
+		},
+	},
+}))
+
+// ---------------------------------------------------------------------------
+// Anthropic SDK mock
+// ---------------------------------------------------------------------------
+
+/** Capture what was passed to `messages.create` so tests can assert on it. */
+let lastCreateCall: any = undefined
+
+const mockCreate = vitest.fn()
+
+vitest.mock("@anthropic-ai/sdk", () => {
+	const mockAnthropicConstructor = vitest.fn().mockImplementation(() => ({
+		messages: {
+			create: mockCreate,
+		},
+	}))
+	return { Anthropic: mockAnthropicConstructor }
+})
+
+// ---------------------------------------------------------------------------
+// Helper: build a minimal streaming response
+// ---------------------------------------------------------------------------
+
+function makeAnthropicStream(cacheCreationTokens: number | undefined, cacheReadTokens: number | undefined) {
+	return {
+		async *[Symbol.asyncIterator]() {
+			yield {
+				type: "message_start",
+				message: {
+					usage: {
+						input_tokens: 100,
+						output_tokens: 0,
+						cache_creation_input_tokens: cacheCreationTokens,
+						cache_read_input_tokens: cacheReadTokens,
+					},
+				},
+			}
+			yield {
+				type: "content_block_start",
+				index: 0,
+				content_block: { type: "text", text: "hi" },
+			}
+			yield {
+				type: "message_delta",
+				usage: { output_tokens: 5 },
+			}
+			yield { type: "message_stop" }
+		},
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Tests: Anthropic provider – system prompt cache_control
+// ---------------------------------------------------------------------------
+
+describe("AnthropicHandler – prompt caching", () => {
+	const baseOptions: ApiHandlerOptions = {
+		apiKey: "test-key",
+		apiModelId: "claude-3-5-sonnet-20241022",
+	}
+
+	beforeEach(() => {
+		vitest.clearAllMocks()
+		lastCreateCall = undefined
+	})
+
+	describe("system prompt cache_control", () => {
+		it("sets cache_control: { type: 'ephemeral' } on the system block for cache-capable models", async () => {
+			mockCreate.mockReturnValue(makeAnthropicStream(500, 0))
+
+			const handler = new AnthropicHandler(baseOptions)
+			const gen = handler.createMessage("System prompt here", [{ role: "user", content: "Hello" }])
+
+			// Drain the generator.
+			for await (const _ of gen) {
+				// consume
+			}
+
+			expect(mockCreate).toHaveBeenCalledOnce()
+			const callArgs = mockCreate.mock.calls[0][0]
+
+			// System must be an array with exactly one block.
+			expect(Array.isArray(callArgs.system)).toBe(true)
+			expect(callArgs.system).toHaveLength(1)
+
+			const systemBlock = callArgs.system[0]
+			expect(systemBlock.type).toBe("text")
+			expect(systemBlock.text).toBe("System prompt here")
+			// THE KEY ASSERTION: cache_control must be ephemeral.
+			expect(systemBlock.cache_control).toEqual({ type: "ephemeral" })
+		})
+
+		it("includes the prompt-caching beta header for cache-capable models", async () => {
+			mockCreate.mockReturnValue(makeAnthropicStream(0, 0))
+
+			const handler = new AnthropicHandler(baseOptions)
+			const gen = handler.createMessage("System prompt", [{ role: "user", content: "Hi" }])
+			for await (const _ of gen) {
+				/* drain */
+			}
+
+			expect(mockCreate).toHaveBeenCalledOnce()
+			// The second argument to create() is the request options (headers).
+			const requestOptions = mockCreate.mock.calls[0][1]
+			const betaHeader: string = requestOptions?.headers?.["anthropic-beta"] ?? ""
+			expect(betaHeader).toContain("prompt-caching-2024-07-31")
+		})
+
+		it("falls back to the default model (which is cache-capable) when an unknown model ID is supplied", async () => {
+			// `getModel()` maps unknown IDs to `anthropicDefaultModelId`, which IS in the
+			// cache-capable switch list. Therefore cache_control is always applied even for
+			// unknown/legacy model strings, because the fallback model supports caching.
+			const unknownModelOptions: ApiHandlerOptions = {
+				...baseOptions,
+				apiModelId: "claude-ancient-unsupported-model" as any,
+			}
+
+			mockCreate.mockReturnValue(makeAnthropicStream(0, 0))
+
+			const handler = new AnthropicHandler(unknownModelOptions)
+			for await (const _ of handler.createMessage("System", [{ role: "user", content: "Hi" }])) {
+				/* drain */
+			}
+
+			const callArgs = mockCreate.mock.calls[0][0]
+			const systemBlock = callArgs.system[0]
+
+			// The fallback model is cache-capable, so cache_control IS set.
+			// This is the correct / expected behaviour – the default branch is only
+			// reached when the model ID exactly matches a known non-cached model,
+			// which currently does not exist in the active anthropicModels list.
+			expect(systemBlock.type).toBe("text")
+			expect(systemBlock.text).toBe("System")
+			// cache_control is present because the fallback model supports caching.
+			expect(systemBlock.cache_control).toEqual({ type: "ephemeral" })
+		})
+	})
+
+	// ---------------------------------------------------------------------------
+	// Stream processing: cache metric capture
+	// ---------------------------------------------------------------------------
+
+	describe("stream processing – cache metric capture", () => {
+		it("yields cacheWriteTokens from cache_creation_input_tokens in message_start", async () => {
+			mockCreate.mockReturnValue(makeAnthropicStream(1234, 0))
+
+			const handler = new AnthropicHandler(baseOptions)
+			const chunks: any[] = []
+			for await (const chunk of handler.createMessage("Sys", [{ role: "user", content: "Hi" }])) {
+				chunks.push(chunk)
+			}
+
+			const usageChunks = chunks.filter((c) => c.type === "usage")
+			expect(usageChunks.length).toBeGreaterThan(0)
+
+			// The first usage chunk (from message_start) carries cacheWriteTokens.
+			const firstUsage = usageChunks[0]
+			expect(firstUsage.cacheWriteTokens).toBe(1234)
+		})
+
+		it("yields cacheReadTokens from cache_read_input_tokens in message_start", async () => {
+			mockCreate.mockReturnValue(makeAnthropicStream(0, 567))
+
+			const handler = new AnthropicHandler(baseOptions)
+			const chunks: any[] = []
+			for await (const chunk of handler.createMessage("Sys", [{ role: "user", content: "Hi" }])) {
+				chunks.push(chunk)
+			}
+
+			const usageChunks = chunks.filter((c) => c.type === "usage")
+			expect(usageChunks.length).toBeGreaterThan(0)
+
+			const firstUsage = usageChunks[0]
+			expect(firstUsage.cacheReadTokens).toBe(567)
+		})
+
+		it("yields both cacheWriteTokens and cacheReadTokens when both are present", async () => {
+			mockCreate.mockReturnValue(makeAnthropicStream(800, 200))
+
+			const handler = new AnthropicHandler(baseOptions)
+			const chunks: any[] = []
+			for await (const chunk of handler.createMessage("Sys", [{ role: "user", content: "Hi" }])) {
+				chunks.push(chunk)
+			}
+
+			const firstUsage = chunks.find((c) => c.type === "usage" && c.cacheWriteTokens !== undefined)
+			expect(firstUsage).toBeDefined()
+			expect(firstUsage.cacheWriteTokens).toBe(800)
+			expect(firstUsage.cacheReadTokens).toBe(200)
+		})
+
+		it("omits cacheWriteTokens when cache_creation_input_tokens is 0 (falsy → undefined)", async () => {
+			mockCreate.mockReturnValue(makeAnthropicStream(0, 0))
+
+			const handler = new AnthropicHandler(baseOptions)
+			const chunks: any[] = []
+			for await (const chunk of handler.createMessage("Sys", [{ role: "user", content: "Hi" }])) {
+				chunks.push(chunk)
+			}
+
+			const firstUsage = chunks.find((c) => c.type === "usage")
+			// 0 is falsy so the handler maps it to `undefined`.
+			expect(firstUsage?.cacheWriteTokens).toBeUndefined()
+			expect(firstUsage?.cacheReadTokens).toBeUndefined()
+		})
+	})
+
+	// ---------------------------------------------------------------------------
+	// cache_control on user messages
+	// ---------------------------------------------------------------------------
+
+	describe("user message cache markers", () => {
+		it("attaches cache_control to the last and second-to-last user messages", async () => {
+			mockCreate.mockReturnValue(makeAnthropicStream(0, 0))
+
+			const handler = new AnthropicHandler(baseOptions)
+
+			const messages: any[] = [
+				{ role: "user", content: "First user message" },
+				{ role: "assistant", content: "First assistant reply" },
+				{ role: "user", content: "Second user message" },
+			]
+
+			for await (const _ of handler.createMessage("System", messages)) {
+				/* drain */
+			}
+
+			const callArgs = mockCreate.mock.calls[0][0]
+			const sentMessages: any[] = callArgs.messages
+
+			// Message indices: 0 = user, 1 = assistant, 2 = user
+			// userMsgIndices = [0, 2]  →  last=2, secondLast=0
+			const lastUserMsg = sentMessages[2]
+			const secondLastUserMsg = sentMessages[0]
+
+			// Last user message content should be an array with cache_control on the last block.
+			const lastContent = Array.isArray(lastUserMsg.content) ? lastUserMsg.content : null
+			expect(lastContent).not.toBeNull()
+			const lastBlock = lastContent![lastContent!.length - 1]
+			expect(lastBlock.cache_control).toEqual({ type: "ephemeral" })
+
+			// Second-to-last user message should also carry cache_control.
+			const secondLastContent = Array.isArray(secondLastUserMsg.content) ? secondLastUserMsg.content : null
+			expect(secondLastContent).not.toBeNull()
+			const secondLastBlock = secondLastContent![secondLastContent!.length - 1]
+			expect(secondLastBlock.cache_control).toEqual({ type: "ephemeral" })
+		})
+	})
+})
+
+// ---------------------------------------------------------------------------
+// Tests: ApiStreamUsageChunk type completeness (static/compile-time guard)
+// ---------------------------------------------------------------------------
+
+describe("ApiStreamUsageChunk – type completeness", () => {
+	it("declares cacheWriteTokens and cacheReadTokens fields", () => {
+		// This is a compile-time / shape test. If the fields are removed from the
+		// type definition the TypeScript compilation will fail here.
+		const chunk = {
+			type: "usage" as const,
+			inputTokens: 100,
+			outputTokens: 50,
+			cacheWriteTokens: 20,
+			cacheReadTokens: 10,
+		}
+
+		// Runtime assertion as a belt-and-suspenders guard.
+		expect(chunk).toHaveProperty("cacheWriteTokens", 20)
+		expect(chunk).toHaveProperty("cacheReadTokens", 10)
+	})
+})