From 87865a29e5f3ce7a9b780db05f7980fc164ca96b Mon Sep 17 00:00:00 2001 From: Seth Date: Mon, 31 Aug 2026 19:34:48 -0400 Subject: [PATCH 1/4] fix(ai): cache Anthropic tool results --- .../ai/src/providers/openai-completions.ts | 5 +- ...i-completions-cache-control-format.test.ts | 69 ++++++++++++++++++- 2 files changed, 70 insertions(+), 4 deletions(-) diff --git a/packages/ai/src/providers/openai-completions.ts b/packages/ai/src/providers/openai-completions.ts index 00f945f17b..e39d959178 100644 --- a/packages/ai/src/providers/openai-completions.ts +++ b/packages/ai/src/providers/openai-completions.ts @@ -724,7 +724,7 @@ function addCacheControlToLastConversationMessage( ): void { for (let i = messages.length - 1; i >= 0; i--) { const message = messages[i]; - if (message.role === "user" || message.role === "assistant") { + if (message.role === "user" || message.role === "assistant" || message.role === "tool") { if (addCacheControlToMessage(message, cacheControl)) { return; } @@ -755,7 +755,7 @@ function addCacheControlToMessage( message: ChatCompletionMessageParam, cacheControl: OpenAICompatCacheControl, ): boolean { - if (message.role === "user" || message.role === "assistant") { + if (message.role === "user" || message.role === "assistant" || message.role === "tool") { return addCacheControlToTextContent(message, cacheControl); } return false; @@ -765,6 +765,7 @@ function addCacheControlToTextContent( message: | ChatCompletionInstructionMessageParam | ChatCompletionAssistantMessageParam + | ChatCompletionToolMessageParam | Extract, cacheControl: OpenAICompatCacheControl, ): boolean { diff --git a/packages/ai/test/openai-completions-cache-control-format.test.ts b/packages/ai/test/openai-completions-cache-control-format.test.ts index f1ff9a5c7b..c4d3e4be23 100644 --- a/packages/ai/test/openai-completions-cache-control-format.test.ts +++ b/packages/ai/test/openai-completions-cache-control-format.test.ts @@ -2,7 +2,7 @@ import { Type } from "typebox"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { getModel } from "../src/models.js"; import { streamOpenAICompletions } from "../src/providers/openai-completions.js"; -import type { AssistantMessage, Model } from "../src/types.js"; +import type { AssistantMessage, Context, Model, Usage } from "../src/types.js"; interface CacheControl { type: "ephemeral"; @@ -32,6 +32,15 @@ const mockState = vi.hoisted(() => ({ lastParams: undefined as CapturedParams | undefined, })); +const emptyUsage: Usage = { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, +}; + vi.mock("openai", () => { class FakeOpenAI { chat = { @@ -78,6 +87,7 @@ vi.mock("openai", () => { async function runCompletion( model: Model<"openai-completions">, options?: { cacheRetention?: "none" | "short" | "long" }, + messages?: Context["messages"], ): Promise<{ params: CapturedParams; result: AssistantMessage }> { const timestamp = Date.now(); @@ -85,7 +95,7 @@ async function runCompletion( model, { systemPrompt: "System prompt", - messages: [{ role: "user", content: "Hello", timestamp }], + messages: messages ?? [{ role: "user", content: "Hello", timestamp }], tools: [ { name: "read", @@ -169,6 +179,61 @@ describe("openai-completions cacheControlFormat", () => { expect(result.usage.cost.cacheWrite).toBeCloseTo((80 * model.cost.cacheWrite) / 1_000_000); }); + it("advances the Anthropic cache marker to the latest tool result", async () => { + const model = getModel("prime-inference", "anthropic/claude-fable-5"); + const now = Date.now(); + const firstAssistant: AssistantMessage = { + role: "assistant", + content: [ + { type: "text", text: "Starting the first read." }, + { type: "toolCall", id: "tool-1", name: "read", arguments: { path: "first.txt" } }, + ], + api: model.api, + provider: model.provider, + model: model.id, + usage: emptyUsage, + stopReason: "toolUse", + timestamp: now + 1, + }; + const secondAssistant: AssistantMessage = { + ...firstAssistant, + content: [{ type: "toolCall", id: "tool-2", name: "read", arguments: { path: "second.txt" } }], + timestamp: now + 3, + }; + const messages: Context["messages"] = [ + { role: "user", content: "Read both files", timestamp: now }, + firstAssistant, + { + role: "toolResult", + toolCallId: "tool-1", + toolName: "read", + content: [{ type: "text", text: "first result" }], + isError: false, + timestamp: now + 2, + }, + secondAssistant, + { + role: "toolResult", + toolCallId: "tool-2", + toolName: "read", + content: [{ type: "text", text: "second result" }], + isError: false, + timestamp: now + 4, + }, + ]; + + const { params } = await runCompletion(model, undefined, messages); + const lastMessage = params.messages[params.messages.length - 1]; + expect(lastMessage.role).toBe("tool"); + expect(Array.isArray(lastMessage.content)).toBe(true); + expect((lastMessage.content as TextPart[])[0]?.cache_control).toEqual({ type: "ephemeral" }); + + const firstAssistantPayload = params.messages.find( + (message) => message.role === "assistant" && message.content !== null, + ); + expect(typeof firstAssistantPayload?.content).toBe("string"); + }); + it("preserves Anthropic-style cache markers for OpenRouter Anthropic models", async () => { const model = getModel("openrouter", "anthropic/claude-sonnet-4"); const params = await capturePayload(model); From d370d417587a48609b655b4f6e24cb59c2574b11 Mon Sep 17 00:00:00 2001 From: Seth Date: Mon, 31 Aug 2026 19:36:05 -0400 Subject: [PATCH 2/4] docs(ai): add cache fix changelog fragment --- packages/ai/.changes/fix-anthropic-tool-cache.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 packages/ai/.changes/fix-anthropic-tool-cache.md diff --git a/packages/ai/.changes/fix-anthropic-tool-cache.md b/packages/ai/.changes/fix-anthropic-tool-cache.md new file mode 100644 index 0000000000..b2c2041e03 --- /dev/null +++ b/packages/ai/.changes/fix-anthropic-tool-cache.md @@ -0,0 +1 @@ +- Fixed Anthropic-compatible prompt caching so the rolling cache marker advances to the latest tool result. From be91ea9b32214102527a3aa86840587d78da17c1 Mon Sep 17 00:00:00 2001 From: Seth Date: Mon, 31 Aug 2026 19:53:24 -0400 Subject: [PATCH 3/4] test(ai): trim Anthropic cache regression --- ...i-completions-cache-control-format.test.ts | 57 ++++++------------- 1 file changed, 17 insertions(+), 40 deletions(-) diff --git a/packages/ai/test/openai-completions-cache-control-format.test.ts b/packages/ai/test/openai-completions-cache-control-format.test.ts index c4d3e4be23..0b5333e36c 100644 --- a/packages/ai/test/openai-completions-cache-control-format.test.ts +++ b/packages/ai/test/openai-completions-cache-control-format.test.ts @@ -179,59 +179,36 @@ describe("openai-completions cacheControlFormat", () => { expect(result.usage.cost.cacheWrite).toBeCloseTo((80 * model.cost.cacheWrite) / 1_000_000); }); - it("advances the Anthropic cache marker to the latest tool result", async () => { + it("advances the Anthropic cache marker to a tool result", async () => { const model = getModel("prime-inference", "anthropic/claude-fable-5"); const now = Date.now(); - const firstAssistant: AssistantMessage = { - role: "assistant", - content: [ - { type: "text", text: "Starting the first read." }, - { type: "toolCall", id: "tool-1", name: "read", arguments: { path: "first.txt" } }, - ], - api: model.api, - provider: model.provider, - model: model.id, - usage: emptyUsage, - stopReason: "toolUse", - timestamp: now + 1, - }; - const secondAssistant: AssistantMessage = { - ...firstAssistant, - content: [{ type: "toolCall", id: "tool-2", name: "read", arguments: { path: "second.txt" } }], - timestamp: now + 3, - }; const messages: Context["messages"] = [ - { role: "user", content: "Read both files", timestamp: now }, - firstAssistant, + { role: "user", content: "Read the file", timestamp: now }, { - role: "toolResult", - toolCallId: "tool-1", - toolName: "read", - content: [{ type: "text", text: "first result" }], - isError: false, - timestamp: now + 2, + role: "assistant", + content: [{ type: "toolCall", id: "tool-1", name: "read", arguments: { path: "file.txt" } }], + api: model.api, + provider: model.provider, + model: model.id, + usage: emptyUsage, + stopReason: "toolUse", + timestamp: now + 1, }, - secondAssistant, { role: "toolResult", - toolCallId: "tool-2", + toolCallId: "tool-1", toolName: "read", - content: [{ type: "text", text: "second result" }], + content: [{ type: "text", text: "file contents" }], isError: false, - timestamp: now + 4, + timestamp: now + 2, }, ]; const { params } = await runCompletion(model, undefined, messages); - const lastMessage = params.messages[params.messages.length - 1]; - expect(lastMessage.role).toBe("tool"); - expect(Array.isArray(lastMessage.content)).toBe(true); - expect((lastMessage.content as TextPart[])[0]?.cache_control).toEqual({ type: "ephemeral" }); - - const firstAssistantPayload = params.messages.find( - (message) => message.role === "assistant" && message.content !== null, - ); - expect(typeof firstAssistantPayload?.content).toBe("string"); + expect(params.messages.at(-1)).toMatchObject({ + role: "tool", + content: [{ type: "text", text: "file contents", cache_control: { type: "ephemeral" } }], + }); }); it("preserves Anthropic-style cache markers for OpenRouter Anthropic models", async () => { From eb645563cd8c8d386239861e3bdbfe82e5045fa0 Mon Sep 17 00:00:00 2001 From: Seth Date: Mon, 31 Aug 2026 19:58:42 -0400 Subject: [PATCH 4/4] test(ai): use Haiku cache fixture --- .../ai/test/openai-completions-cache-control-format.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ai/test/openai-completions-cache-control-format.test.ts b/packages/ai/test/openai-completions-cache-control-format.test.ts index 0b5333e36c..9b6f66c46a 100644 --- a/packages/ai/test/openai-completions-cache-control-format.test.ts +++ b/packages/ai/test/openai-completions-cache-control-format.test.ts @@ -180,7 +180,7 @@ describe("openai-completions cacheControlFormat", () => { }); it("advances the Anthropic cache marker to a tool result", async () => { - const model = getModel("prime-inference", "anthropic/claude-fable-5"); + const model = getModel("prime-inference", "anthropic/claude-haiku-4.5"); const now = Date.now(); const messages: Context["messages"] = [ { role: "user", content: "Read the file", timestamp: now },