Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/ai/.changes/fix-anthropic-tool-cache.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Fixed Anthropic-compatible prompt caching so the rolling cache marker advances to the latest tool result.
5 changes: 3 additions & 2 deletions packages/ai/src/providers/openai-completions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -724,7 +724,7 @@ function addCacheControlToLastConversationMessage(
): void {
for (let i = messages.length - 1; i >= 0; i--) {
const message = messages[i];
if (message.role === "user" || message.role === "assistant") {
if (message.role === "user" || message.role === "assistant" || message.role === "tool") {
if (addCacheControlToMessage(message, cacheControl)) {
return;
}
Expand Down Expand Up @@ -755,7 +755,7 @@ function addCacheControlToMessage(
message: ChatCompletionMessageParam,
cacheControl: OpenAICompatCacheControl,
): boolean {
if (message.role === "user" || message.role === "assistant") {
if (message.role === "user" || message.role === "assistant" || message.role === "tool") {
return addCacheControlToTextContent(message, cacheControl);
}
return false;
Expand All @@ -765,6 +765,7 @@ function addCacheControlToTextContent(
message:
| ChatCompletionInstructionMessageParam
| ChatCompletionAssistantMessageParam
| ChatCompletionToolMessageParam
| Extract<ChatCompletionMessageParam, { role: "user" }>,
cacheControl: OpenAICompatCacheControl,
): boolean {
Expand Down
46 changes: 44 additions & 2 deletions packages/ai/test/openai-completions-cache-control-format.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { Type } from "typebox";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { getModel } from "../src/models.js";
import { streamOpenAICompletions } from "../src/providers/openai-completions.js";
import type { AssistantMessage, Model } from "../src/types.js";
import type { AssistantMessage, Context, Model, Usage } from "../src/types.js";

interface CacheControl {
type: "ephemeral";
Expand Down Expand Up @@ -32,6 +32,15 @@ const mockState = vi.hoisted(() => ({
lastParams: undefined as CapturedParams | undefined,
}));

const emptyUsage: Usage = {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
};

vi.mock("openai", () => {
class FakeOpenAI {
chat = {
Expand Down Expand Up @@ -78,14 +87,15 @@ vi.mock("openai", () => {
async function runCompletion(
model: Model<"openai-completions">,
options?: { cacheRetention?: "none" | "short" | "long" },
messages?: Context["messages"],
): Promise<{ params: CapturedParams; result: AssistantMessage }> {
const timestamp = Date.now();

const result = await streamOpenAICompletions(
model,
{
systemPrompt: "System prompt",
messages: [{ role: "user", content: "Hello", timestamp }],
messages: messages ?? [{ role: "user", content: "Hello", timestamp }],
tools: [
{
name: "read",
Expand Down Expand Up @@ -169,6 +179,38 @@ describe("openai-completions cacheControlFormat", () => {
expect(result.usage.cost.cacheWrite).toBeCloseTo((80 * model.cost.cacheWrite) / 1_000_000);
});

it("advances the Anthropic cache marker to a tool result", async () => {
const model = getModel("prime-inference", "anthropic/claude-fable-5");
const now = Date.now();
const messages: Context["messages"] = [
{ role: "user", content: "Read the file", timestamp: now },
{
role: "assistant",
content: [{ type: "toolCall", id: "tool-1", name: "read", arguments: { path: "file.txt" } }],
api: model.api,
provider: model.provider,
model: model.id,
usage: emptyUsage,
stopReason: "toolUse",
timestamp: now + 1,
},
{
role: "toolResult",
toolCallId: "tool-1",
toolName: "read",
content: [{ type: "text", text: "file contents" }],
isError: false,
timestamp: now + 2,
},
];

const { params } = await runCompletion(model, undefined, messages);
expect(params.messages.at(-1)).toMatchObject({
role: "tool",
content: [{ type: "text", text: "file contents", cache_control: { type: "ephemeral" } }],
});
});

it("preserves Anthropic-style cache markers for OpenRouter Anthropic models", async () => {
const model = getModel("openrouter", "anthropic/claude-sonnet-4");
const params = await capturePayload(model);
Expand Down
Loading