diff --git a/packages/ai/.changes/fix-anthropic-tool-cache.md b/packages/ai/.changes/fix-anthropic-tool-cache.md new file mode 100644 index 0000000000..b2c2041e03 --- /dev/null +++ b/packages/ai/.changes/fix-anthropic-tool-cache.md @@ -0,0 +1 @@ +- Fixed Anthropic-compatible prompt caching so the rolling cache marker advances to the latest tool result. diff --git a/packages/ai/src/providers/openai-completions.ts b/packages/ai/src/providers/openai-completions.ts index 00f945f17b..e39d959178 100644 --- a/packages/ai/src/providers/openai-completions.ts +++ b/packages/ai/src/providers/openai-completions.ts @@ -724,7 +724,7 @@ function addCacheControlToLastConversationMessage( ): void { for (let i = messages.length - 1; i >= 0; i--) { const message = messages[i]; - if (message.role === "user" || message.role === "assistant") { + if (message.role === "user" || message.role === "assistant" || message.role === "tool") { if (addCacheControlToMessage(message, cacheControl)) { return; } @@ -755,7 +755,7 @@ function addCacheControlToMessage( message: ChatCompletionMessageParam, cacheControl: OpenAICompatCacheControl, ): boolean { - if (message.role === "user" || message.role === "assistant") { + if (message.role === "user" || message.role === "assistant" || message.role === "tool") { return addCacheControlToTextContent(message, cacheControl); } return false; @@ -765,6 +765,7 @@ function addCacheControlToTextContent( message: | ChatCompletionInstructionMessageParam | ChatCompletionAssistantMessageParam + | ChatCompletionToolMessageParam | Extract, cacheControl: OpenAICompatCacheControl, ): boolean { diff --git a/packages/ai/test/openai-completions-cache-control-format.test.ts b/packages/ai/test/openai-completions-cache-control-format.test.ts index f1ff9a5c7b..9b6f66c46a 100644 --- a/packages/ai/test/openai-completions-cache-control-format.test.ts +++ b/packages/ai/test/openai-completions-cache-control-format.test.ts @@ -2,7 +2,7 @@ import { Type } from "typebox"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { getModel } from "../src/models.js"; import { streamOpenAICompletions } from "../src/providers/openai-completions.js"; -import type { AssistantMessage, Model } from "../src/types.js"; +import type { AssistantMessage, Context, Model, Usage } from "../src/types.js"; interface CacheControl { type: "ephemeral"; @@ -32,6 +32,15 @@ const mockState = vi.hoisted(() => ({ lastParams: undefined as CapturedParams | undefined, })); +const emptyUsage: Usage = { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, +}; + vi.mock("openai", () => { class FakeOpenAI { chat = { @@ -78,6 +87,7 @@ vi.mock("openai", () => { async function runCompletion( model: Model<"openai-completions">, options?: { cacheRetention?: "none" | "short" | "long" }, + messages?: Context["messages"], ): Promise<{ params: CapturedParams; result: AssistantMessage }> { const timestamp = Date.now(); @@ -85,7 +95,7 @@ async function runCompletion( model, { systemPrompt: "System prompt", - messages: [{ role: "user", content: "Hello", timestamp }], + messages: messages ?? [{ role: "user", content: "Hello", timestamp }], tools: [ { name: "read", @@ -169,6 +179,38 @@ describe("openai-completions cacheControlFormat", () => { expect(result.usage.cost.cacheWrite).toBeCloseTo((80 * model.cost.cacheWrite) / 1_000_000); }); + it("advances the Anthropic cache marker to a tool result", async () => { + const model = getModel("prime-inference", "anthropic/claude-haiku-4.5"); + const now = Date.now(); + const messages: Context["messages"] = [ + { role: "user", content: "Read the file", timestamp: now }, + { + role: "assistant", + content: [{ type: "toolCall", id: "tool-1", name: "read", arguments: { path: "file.txt" } }], + api: model.api, + provider: model.provider, + model: model.id, + usage: emptyUsage, + stopReason: "toolUse", + timestamp: now + 1, + }, + { + role: "toolResult", + toolCallId: "tool-1", + toolName: "read", + content: [{ type: "text", text: "file contents" }], + isError: false, + timestamp: now + 2, + }, + ]; + + const { params } = await runCompletion(model, undefined, messages); + expect(params.messages.at(-1)).toMatchObject({ + role: "tool", + content: [{ type: "text", text: "file contents", cache_control: { type: "ephemeral" } }], + }); + }); + it("preserves Anthropic-style cache markers for OpenRouter Anthropic models", async () => { const model = getModel("openrouter", "anthropic/claude-sonnet-4"); const params = await capturePayload(model);