diff --git a/apps/edge-api/internal/anthropic/cache_control_test.go b/apps/edge-api/internal/anthropic/cache_control_test.go new file mode 100644 index 000000000..8bf87e932 --- /dev/null +++ b/apps/edge-api/internal/anthropic/cache_control_test.go @@ -0,0 +1,579 @@ +package anthropic_test + +import ( + "bytes" + "encoding/json" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + "unicode/utf8" + + "github.com/sakibsadmanshajib/hive/apps/edge-api/internal/anthropic" +) + +// -------------------------------------------------------------------------- +// Request side: cache_control on every documented placement, plus the +// regression guard (no cache_control anywhere must change nothing) and the +// flattening trap the coordinator's research specifically called out: a +// typed content-block array carrying a cache breakpoint must survive +// translate_request.go as an array of typed blocks, never collapsed to a +// plain string (a plain string cannot carry a per-block cache_control, so +// collapsing it silently destroys the breakpoint the caller asked for). +// -------------------------------------------------------------------------- + +func TestToOAIRequest_CacheControl_Placements(t *testing.T) { + cases := []struct { + name string + raw string + check func(t *testing.T, got anthropic.OAIRequest) + }{ + { + name: "system block", + raw: `{"model":"m","max_tokens":5, + "system":[{"type":"text","text":"long instructions"},{"type":"text","text":"tail","cache_control":{"type":"ephemeral"}}], + "messages":[{"role":"user","content":"hi"}]}`, + check: func(t *testing.T, got anthropic.OAIRequest) { + if got.Messages[0].Role != "system" { + t.Fatalf("want first message role=system got %q", got.Messages[0].Role) + } + b, _ := json.Marshal(got.Messages[0].Content) + var parts []map[string]interface{} + if err := json.Unmarshal(b, &parts); err != nil { + t.Fatalf("system content did not serialize as a block array: %s (err %v)", b, err) + } + if len(parts) != 2 { + t.Fatalf("want 2 system blocks got %d: %s", len(parts), b) + } + if _, ok := parts[0]["cache_control"]; ok { + t.Errorf("first system block must carry no cache_control, got %v", parts[0]) + } + cc, ok := parts[1]["cache_control"].(map[string]interface{}) + if !ok { + t.Fatalf("second system block missing cache_control: %v", parts[1]) + } + if cc["type"] != "ephemeral" { + t.Errorf("cache_control.type: want ephemeral got %v", cc["type"]) + } + }, + }, + { + name: "message content block, ttl 1h", + raw: `{"model":"m","max_tokens":5,"messages":[{"role":"user","content":[ + {"type":"text","text":"a huge cached document"}, + {"type":"text","text":"question","cache_control":{"type":"ephemeral","ttl":"1h"}} + ]}]}`, + check: func(t *testing.T, got anthropic.OAIRequest) { + b, _ := json.Marshal(got.Messages[0].Content) + var parts []map[string]interface{} + if err := json.Unmarshal(b, &parts); err != nil { + t.Fatalf("message content did not serialize as a block array: %s (err %v)", b, err) + } + if len(parts) != 2 { + t.Fatalf("want 2 message blocks got %d: %s", len(parts), b) + } + cc, ok := parts[1]["cache_control"].(map[string]interface{}) + if !ok { + t.Fatalf("second message block missing cache_control: %v", parts[1]) + } + if cc["type"] != "ephemeral" || cc["ttl"] != "1h" { + t.Errorf("cache_control: want {ephemeral 1h} got %v", cc) + } + }, + }, + { + name: "tool definition", + raw: `{"model":"m","max_tokens":5,"messages":[{"role":"user","content":"weather?"}], + "tools":[{"name":"get_weather","input_schema":{},"cache_control":{"type":"ephemeral"}}]}`, + check: func(t *testing.T, got anthropic.OAIRequest) { + if len(got.Tools) != 1 { + t.Fatalf("want 1 tool got %d", len(got.Tools)) + } + if got.Tools[0].CacheControl == nil || got.Tools[0].CacheControl.Type != "ephemeral" { + t.Errorf("tool cache_control: got %+v", got.Tools[0].CacheControl) + } + b, _ := json.Marshal(got.Tools[0]) + var m map[string]interface{} + _ = json.Unmarshal(b, &m) + if _, ok := m["cache_control"]; !ok { + t.Errorf("tool JSON missing cache_control key entirely: %s", b) + } + }, + }, + { + name: "request root", + raw: `{"model":"m","max_tokens":5,"messages":[{"role":"user","content":"hi"}], + "cache_control":{"type":"ephemeral"}}`, + check: func(t *testing.T, got anthropic.OAIRequest) { + if got.CacheControl == nil || got.CacheControl.Type != "ephemeral" { + t.Fatalf("root cache_control: got %+v", got.CacheControl) + } + b, _ := json.Marshal(got) + var m map[string]interface{} + _ = json.Unmarshal(b, &m) + if _, ok := m["cache_control"]; !ok { + t.Errorf("request JSON missing root cache_control key: %s", b) + } + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var req anthropic.MessagesRequest + if err := json.Unmarshal([]byte(tc.raw), &req); err != nil { + t.Fatalf("unmarshal request: %v", err) + } + got, err := anthropic.ToOAIRequest(req) + if err != nil { + t.Fatalf("ToOAIRequest: %v", err) + } + tc.check(t, got) + }) + } +} + +// TestToOAIRequest_CacheControl_SingleTextBlock_NotFlattenedToString is the +// flattening regression test: a message whose content is exactly one text +// block, which is the specific shape convertMessage collapses to a bare +// string for the common case, must NOT collapse when that one block carries +// a cache_control -- a string has nowhere to hang a breakpoint on, so +// flattening here would silently destroy the client's cache and then bill +// them full price for it. +func TestToOAIRequest_CacheControl_SingleTextBlock_NotFlattenedToString(t *testing.T) { + raw := `{"model":"m","max_tokens":5,"messages":[{"role":"user","content":[ + {"type":"text","text":"cache me","cache_control":{"type":"ephemeral"}} + ]}]}` + var req anthropic.MessagesRequest + if err := json.Unmarshal([]byte(raw), &req); err != nil { + t.Fatalf("unmarshal: %v", err) + } + got, err := anthropic.ToOAIRequest(req) + if err != nil { + t.Fatalf("ToOAIRequest: %v", err) + } + b, err := json.Marshal(got.Messages[0].Content) + if err != nil { + t.Fatalf("marshal content: %v", err) + } + if strings.HasPrefix(strings.TrimSpace(string(b)), `"`) { + t.Fatalf("content flattened to a bare string, cache_control lost: %s", b) + } + var parts []map[string]interface{} + if err := json.Unmarshal(b, &parts); err != nil { + t.Fatalf("content is not a block array: %s (err %v)", b, err) + } + if len(parts) != 1 || parts[0]["cache_control"] == nil { + t.Fatalf("expected the sole block to keep its cache_control: %v", parts) + } +} + +// TestToOAIRequest_NoCacheControl_Unchanged is the regression guard: a +// request that carries no cache_control anywhere must serialize exactly as +// it did before cache_control support existed -- plain string content, no +// cache_control key anywhere in the output. +func TestToOAIRequest_NoCacheControl_Unchanged(t *testing.T) { + req := anthropic.MessagesRequest{ + Model: "claude-3-haiku", + System: anthropic.SystemField{Text: "Be concise."}, + Messages: []anthropic.Message{{Role: "user", Content: anthropic.MessageContent{Text: "Hello"}}}, + MaxTokens: 100, + Tools: []anthropic.Tool{ + {Name: "get_weather", InputSchema: json.RawMessage(`{}`)}, + }, + } + got, err := anthropic.ToOAIRequest(req) + if err != nil { + t.Fatalf("ToOAIRequest: %v", err) + } + b, err := json.Marshal(got) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if strings.Contains(string(b), "cache_control") { + t.Errorf("cache_control key leaked into a request that never set one: %s", b) + } + sysContent, _ := json.Marshal(got.Messages[0].Content) + if string(sysContent) != `"Be concise."` { + t.Errorf("system content: want plain string got %s", sysContent) + } +} + +func TestToOAIRequest_CacheControl_ToolResultBlock(t *testing.T) { + raw := `{"model":"m","max_tokens":5,"messages":[{"role":"user","content":[ + {"type":"tool_result","tool_use_id":"tu_01","content":"Sunny","cache_control":{"type":"ephemeral"}} + ]}]}` + var req anthropic.MessagesRequest + if err := json.Unmarshal([]byte(raw), &req); err != nil { + t.Fatalf("unmarshal: %v", err) + } + got, err := anthropic.ToOAIRequest(req) + if err != nil { + t.Fatalf("ToOAIRequest: %v", err) + } + msg := got.Messages[0] + if msg.CacheControl == nil || msg.CacheControl.Type != "ephemeral" { + t.Errorf("tool_result message cache_control: got %+v", msg.CacheControl) + } +} + +func TestToOAIRequest_CacheControl_ToolUseBlock(t *testing.T) { + raw := `{"model":"m","max_tokens":5,"messages":[{"role":"assistant","content":[ + {"type":"tool_use","id":"tu_01","name":"search","input":{"query":"q"},"cache_control":{"type":"ephemeral"}} + ]}]}` + var req anthropic.MessagesRequest + if err := json.Unmarshal([]byte(raw), &req); err != nil { + t.Fatalf("unmarshal: %v", err) + } + got, err := anthropic.ToOAIRequest(req) + if err != nil { + t.Fatalf("ToOAIRequest: %v", err) + } + if len(got.Messages[0].ToolCalls) != 1 { + t.Fatalf("want 1 tool call got %d", len(got.Messages[0].ToolCalls)) + } + cc := got.Messages[0].ToolCalls[0].CacheControl + if cc == nil || cc.Type != "ephemeral" { + t.Errorf("tool_use cache_control: got %+v", cc) + } +} + +func TestToOAIRequest_SessionID_Passthrough(t *testing.T) { + req := anthropic.MessagesRequest{ + Model: "m", + Messages: []anthropic.Message{{Role: "user", Content: anthropic.MessageContent{Text: "hi"}}}, + MaxTokens: 5, + SessionID: "conv-abc-123", + } + got, err := anthropic.ToOAIRequest(req) + if err != nil { + t.Fatalf("ToOAIRequest: %v", err) + } + if got.SessionID != "conv-abc-123" { + t.Errorf("session_id: want conv-abc-123 got %q", got.SessionID) + } +} + +func TestToOAIRequest_SessionID_TruncatedAt256(t *testing.T) { + long := strings.Repeat("a", 300) + req := anthropic.MessagesRequest{ + Model: "m", + Messages: []anthropic.Message{{Role: "user", Content: anthropic.MessageContent{Text: "hi"}}}, + MaxTokens: 5, + SessionID: long, + } + got, err := anthropic.ToOAIRequest(req) + if err != nil { + t.Fatalf("ToOAIRequest: %v", err) + } + if len(got.SessionID) != 256 { + t.Errorf("session_id length: want 256 got %d", len(got.SessionID)) + } + if got.SessionID != long[:256] { + t.Errorf("session_id: truncated value does not match the prefix of the original") + } +} + +// -------------------------------------------------------------------------- +// Response side: the exclusive-shape echo contract Anthropic clients +// (Claude Code included) read cache savings from. OpenRouter's usage is +// inclusive (prompt_tokens already contains the cached and written tokens), +// so the fresh/uncached input count is a subtraction, never an addition. +// -------------------------------------------------------------------------- + +func TestFromOAIResponse_CacheTokens_NonStreaming(t *testing.T) { + oai := anthropic.OAIResponse{ + ID: "chatcmpl-cache", + Choices: []anthropic.OAIChoice{ + {FinishReason: "stop", Message: anthropic.OAIMsg{Role: "assistant", Content: "hi"}}, + }, + Usage: anthropic.OAIUsage{ + PromptTokens: 1000, + CompletionTokens: 20, + TotalTokens: 1020, + PromptTokensDetails: &anthropic.OAIPromptTokensDetails{ + CachedTokens: 800, + CacheWriteTokens: 50, + }, + }, + } + + got := anthropic.FromOAIResponse(oai, "claude-3-haiku") + + if got.Usage.InputTokens != 150 { + t.Errorf("input_tokens (fresh remainder): want 150 got %d", got.Usage.InputTokens) + } + if got.Usage.CacheReadInputTokens != 800 { + t.Errorf("cache_read_input_tokens: want 800 got %d", got.Usage.CacheReadInputTokens) + } + if got.Usage.CacheCreationInputTokens != 50 { + t.Errorf("cache_creation_input_tokens: want 50 got %d", got.Usage.CacheCreationInputTokens) + } + if got.Usage.OutputTokens != 20 { + t.Errorf("output_tokens: want 20 got %d", got.Usage.OutputTokens) + } + + b, err := json.Marshal(got.Usage) + if err != nil { + t.Fatalf("marshal usage: %v", err) + } + var m map[string]interface{} + if err := json.Unmarshal(b, &m); err != nil { + t.Fatalf("unmarshal usage: %v", err) + } + if _, ok := m["cache_read_input_tokens"]; !ok { + t.Errorf("wire usage missing cache_read_input_tokens key: %s", b) + } + if _, ok := m["cache_creation_input_tokens"]; !ok { + t.Errorf("wire usage missing cache_creation_input_tokens key: %s", b) + } +} + +// TestFromOAIResponse_CacheTokens_ClampedNeverNegative guards the failure +// mode the research doc calls out explicitly: if cached+written tokens ever +// exceed the inclusive prompt_tokens total (a sign the upstream shape +// assumption broke), the client-facing input_tokens must clamp to zero, never +// go negative. +func TestFromOAIResponse_CacheTokens_ClampedNeverNegative(t *testing.T) { + oai := anthropic.OAIResponse{ + Usage: anthropic.OAIUsage{ + PromptTokens: 100, + PromptTokensDetails: &anthropic.OAIPromptTokensDetails{ + CachedTokens: 90, + CacheWriteTokens: 50, + }, + }, + } + got := anthropic.FromOAIResponse(oai, "m") + if got.Usage.InputTokens != 0 { + t.Errorf("input_tokens: want clamped 0 got %d", got.Usage.InputTokens) + } +} + +func TestFromOAIResponse_NoCacheDetails_InputTokensUnchanged(t *testing.T) { + oai := anthropic.OAIResponse{ + Usage: anthropic.OAIUsage{PromptTokens: 10, CompletionTokens: 5}, + } + got := anthropic.FromOAIResponse(oai, "m") + if got.Usage.InputTokens != 10 { + t.Errorf("input_tokens with no cache details: want unchanged 10 got %d", got.Usage.InputTokens) + } + if got.Usage.CacheReadInputTokens != 0 || got.Usage.CacheCreationInputTokens != 0 { + t.Errorf("cache fields should be zero with no PromptTokensDetails: %+v", got.Usage) + } + b, _ := json.Marshal(got.Usage) + if strings.Contains(string(b), "cache_read_input_tokens") || strings.Contains(string(b), "cache_creation_input_tokens") { + t.Errorf("omitempty cache fields leaked into a response with no cache activity: %s", b) + } +} + +// -------------------------------------------------------------------------- +// Streaming: the same exclusive-shape numbers, carried through the SSE +// translator's message_delta (the only point in this relay architecture +// where the terminal, authoritative usage is known -- see the doc comment on +// SSETranslator.emitMessageStart for why message_start cannot carry them). +// -------------------------------------------------------------------------- + +func TestSSETranslator_CacheTokensInMessageDelta(t *testing.T) { + stream := buildOAIStream( + `{"id":"chatcmpl-c","model":"m","choices":[{"index":0,"delta":{"content":"hi"},"finish_reason":null}]}`, + `{"id":"chatcmpl-c","model":"m","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],`+ + `"usage":{"prompt_tokens":1000,"completion_tokens":20,"total_tokens":1020,`+ + `"prompt_tokens_details":{"cached_tokens":800,"cache_write_tokens":50}}}`, + ) + rec := httptest.NewRecorder() + tr := anthropic.NewSSETranslator(rec, "m") + if err := tr.Translate(strings.NewReader(stream)); err != nil { + t.Fatalf("translate error: %v", err) + } + events := parseSSEEvents(t, rec.Body.String()) + found := false + for _, ev := range events { + if ev["type"] != "message_delta" { + continue + } + found = true + usage, ok := ev["usage"].(map[string]interface{}) + if !ok { + t.Fatalf("message_delta has no usage object") + } + if usage["cache_read_input_tokens"].(float64) != 800 { + t.Errorf("cache_read_input_tokens: want 800 got %v", usage["cache_read_input_tokens"]) + } + if usage["cache_creation_input_tokens"].(float64) != 50 { + t.Errorf("cache_creation_input_tokens: want 50 got %v", usage["cache_creation_input_tokens"]) + } + if usage["output_tokens"].(float64) != 20 { + t.Errorf("output_tokens: want 20 got %v", usage["output_tokens"]) + } + } + if !found { + t.Fatal("no message_delta event observed") + } +} + +// -------------------------------------------------------------------------- +// Adversarial-review follow-ups (PR #1152 threads): +// MEDIUM 1 - freshInputTokens must alarm, not just clamp, on a negative +// result (the exact signature of the upstream inclusive/exclusive shape +// assumption breaking). +// LOW 1 - cache_control is validated at the trust boundary: type must be +// "ephemeral", ttl must be "5m"/"1h" if set, at most 4 breakpoints total. +// LOW 2 - session_id truncation is rune-boundary-safe, not a raw byte cut. +// -------------------------------------------------------------------------- + +func TestFromOAIResponse_CacheTokens_NegativeClamp_LogsWarn(t *testing.T) { + var buf bytes.Buffer + prev := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&buf, nil))) + defer slog.SetDefault(prev) + + oai := anthropic.OAIResponse{ + Model: "openrouter/anthropic/claude-3-haiku", + Usage: anthropic.OAIUsage{ + PromptTokens: 100, + PromptTokensDetails: &anthropic.OAIPromptTokensDetails{ + CachedTokens: 90, + CacheWriteTokens: 50, // 90+50 > 100: upstream shape assumption broke + }, + }, + } + got := anthropic.FromOAIResponse(oai, "claude-3-haiku") + + if got.Usage.InputTokens != 0 { + t.Errorf("input_tokens: want clamped 0 got %d", got.Usage.InputTokens) + } + logged := buf.String() + if !strings.Contains(logged, "fresh input tokens went negative") { + t.Errorf("expected a WARN alarm on the negative clamp, got log: %s", logged) + } + if !strings.Contains(logged, "claude-3-haiku") { + t.Errorf("expected the alarm to name the client alias, got log: %s", logged) + } + if !strings.Contains(logged, "openrouter/anthropic/claude-3-haiku") { + t.Errorf("expected the alarm to name the upstream model, got log: %s", logged) + } +} + +func TestSSETranslator_CacheTokens_NegativeClamp_LogsWarn(t *testing.T) { + var buf bytes.Buffer + prev := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&buf, nil))) + defer slog.SetDefault(prev) + + stream := buildOAIStream( + `{"id":"chatcmpl-c","model":"openrouter/anthropic/claude-3-haiku","choices":[{"index":0,"delta":{"content":"hi"},"finish_reason":null}]}`, + `{"id":"chatcmpl-c","model":"openrouter/anthropic/claude-3-haiku","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],`+ + `"usage":{"prompt_tokens":100,"completion_tokens":5,"total_tokens":105,`+ + `"prompt_tokens_details":{"cached_tokens":90,"cache_write_tokens":50}}}`, + ) + rec := httptest.NewRecorder() + tr := anthropic.NewSSETranslator(rec, "claude-3-haiku") + if err := tr.Translate(strings.NewReader(stream)); err != nil { + t.Fatalf("translate error: %v", err) + } + if !strings.Contains(buf.String(), "fresh input tokens went negative") { + t.Errorf("expected a WARN alarm on the negative clamp, got log: %s", buf.String()) + } +} + +func TestToOAIRequest_SessionID_TruncatedAtRuneBoundary(t *testing.T) { + // 255 ASCII bytes followed by a 3-byte rune (e.g. "€", U+20AC) straddles + // byte 256 mid-rune under a plain byte-index cut. + long := strings.Repeat("a", 255) + "€€€€" + req := anthropic.MessagesRequest{ + Model: "m", + Messages: []anthropic.Message{{Role: "user", Content: anthropic.MessageContent{Text: "hi"}}}, + MaxTokens: 5, + SessionID: long, + } + got, err := anthropic.ToOAIRequest(req) + if err != nil { + t.Fatalf("ToOAIRequest: %v", err) + } + if len(got.SessionID) > 256 { + t.Fatalf("session_id exceeds the 256 byte ceiling: %d bytes", len(got.SessionID)) + } + if !utf8.ValidString(got.SessionID) { + t.Fatalf("session_id is not valid UTF-8 after truncation: %q", got.SessionID) + } + // json.Marshal must round-trip cleanly with no substituted U+FFFD. + b, err := json.Marshal(got.SessionID) + if err != nil { + t.Fatalf("marshal session_id: %v", err) + } + if strings.Contains(string(b), "\\ufffd") { + t.Errorf("session_id marshaled with a replacement character: %s", b) + } +} + +func TestValidateCacheControl(t *testing.T) { + cases := []struct { + name string + raw string + wantErr bool + }{ + { + name: "valid ephemeral, no ttl", + raw: `{"model":"m","max_tokens":5,"messages":[{"role":"user","content":[{"type":"text","text":"a","cache_control":{"type":"ephemeral"}}]}]}`, + wantErr: false, + }, + { + name: "valid ephemeral, 1h ttl", + raw: `{"model":"m","max_tokens":5,"messages":[{"role":"user","content":[{"type":"text","text":"a","cache_control":{"type":"ephemeral","ttl":"1h"}}]}]}`, + wantErr: false, + }, + { + name: "invalid type rejected", + raw: `{"model":"m","max_tokens":5,"messages":[{"role":"user","content":[{"type":"text","text":"a","cache_control":{"type":"persistent"}}]}]}`, + wantErr: true, + }, + { + name: "invalid ttl rejected", + raw: `{"model":"m","max_tokens":5,"messages":[{"role":"user","content":[{"type":"text","text":"a","cache_control":{"type":"ephemeral","ttl":"1d"}}]}]}`, + wantErr: true, + }, + { + name: "exactly 4 breakpoints accepted", + raw: `{"model":"m","max_tokens":5, + "cache_control":{"type":"ephemeral"}, + "system":[{"type":"text","text":"s","cache_control":{"type":"ephemeral"}}], + "tools":[{"name":"t","input_schema":{},"cache_control":{"type":"ephemeral"}}], + "messages":[{"role":"user","content":[{"type":"text","text":"a","cache_control":{"type":"ephemeral"}}]}]}`, + wantErr: false, + }, + { + name: "5 breakpoints rejected", + raw: `{"model":"m","max_tokens":5, + "cache_control":{"type":"ephemeral"}, + "system":[{"type":"text","text":"s","cache_control":{"type":"ephemeral"}}], + "tools":[{"name":"t","input_schema":{},"cache_control":{"type":"ephemeral"}}], + "messages":[{"role":"user","content":[ + {"type":"text","text":"a","cache_control":{"type":"ephemeral"}}, + {"type":"text","text":"b","cache_control":{"type":"ephemeral"}} + ]}]}`, + wantErr: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var req anthropic.MessagesRequest + if err := json.Unmarshal([]byte(tc.raw), &req); err != nil { + t.Fatalf("unmarshal: %v", err) + } + h := anthropic.NewHandler(anthropic.Deps{OpenAIChat: &fakeChat{}}) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, newAuthedRequest(t, tc.raw)) + + if tc.wantErr { + if rec.Code != http.StatusBadRequest { + t.Fatalf("status: want 400 got %d, body=%s", rec.Code, rec.Body.String()) + } + } else if rec.Code == http.StatusBadRequest { + t.Fatalf("unexpected 400 for a valid request: %s", rec.Body.String()) + } + }) + } +} diff --git a/apps/edge-api/internal/anthropic/handler.go b/apps/edge-api/internal/anthropic/handler.go index dfd625b20..efea2f3cc 100644 --- a/apps/edge-api/internal/anthropic/handler.go +++ b/apps/edge-api/internal/anthropic/handler.go @@ -108,6 +108,10 @@ func (h *Handler) handleMessages(w http.ResponseWriter, r *http.Request) { writeAnthropicError(w, http.StatusBadRequest, "max_tokens is required and must be greater than 0", "") return } + if err := validateCacheControl(req); err != nil { + writeAnthropicError(w, http.StatusBadRequest, err.Error(), "") + return + } // Capture the client alias before translation so we can echo it back // in the response. We must never return an upstream route identifier. diff --git a/apps/edge-api/internal/anthropic/stream.go b/apps/edge-api/internal/anthropic/stream.go index 3d6c40839..3483aa412 100644 --- a/apps/edge-api/internal/anthropic/stream.go +++ b/apps/edge-api/internal/anthropic/stream.go @@ -38,10 +38,12 @@ type SSETranslator struct { clientAlias string // state - messageID string - inputTokens int - outputTokens int - stopReason string + messageID string + inputTokens int + outputTokens int + cacheCreationTokens int + cacheReadTokens int + stopReason string openBlockIndex int // index of the currently open content block hasOpenBlock bool // whether a content_block_start has been emitted without stop @@ -158,8 +160,17 @@ func (t *SSETranslator) FeedLine(line []byte) bool { } if chunk.Usage != nil { - t.inputTokens = chunk.Usage.PromptTokens + cacheRead, cacheWrite := 0, 0 + if chunk.Usage.PromptTokensDetails != nil { + cacheRead = chunk.Usage.PromptTokensDetails.CachedTokens + cacheWrite = chunk.Usage.PromptTokensDetails.CacheWriteTokens + } + // Overwrite, never accumulate: a usage-bearing chunk reports its own + // totals to date, not a delta to add to the last one. + t.inputTokens = freshInputTokens(chunk.Usage.PromptTokens, cacheRead, cacheWrite, t.clientAlias, chunk.Model) t.outputTokens = chunk.Usage.CompletionTokens + t.cacheCreationTokens = cacheWrite + t.cacheReadTokens = cacheRead } if len(chunk.Choices) == 0 { @@ -208,6 +219,15 @@ func (t *SSETranslator) WriteErr() error { return t.writeErr } // --- emitters --- +// emitMessageStart fires on the first upstream chunk, which is always before +// any usage-bearing chunk arrives in this gateway's relay (usage is a +// terminal SSE frame from stream_options.include_usage). So unlike the real +// Anthropic API, which knows input and cache counts before generation even +// starts, t.inputTokens/cacheCreationTokens/cacheReadTokens here are always +// still zero at this point -- a pre-existing limitation of relaying rather +// than natively serving the stream, not something introduced by adding the +// two cache fields. The authoritative values land in message_delta below, +// same as output_tokens already only becomes accurate there. func (t *SSETranslator) emitMessageStart() { ev := StreamEvent{ Type: "message_start", @@ -219,7 +239,11 @@ func (t *SSETranslator) emitMessageStart() { Content: []string{}, StopReason: nil, StopSequence: nil, - Usage: StreamUsage{InputTokens: t.inputTokens}, + Usage: StreamUsage{ + InputTokens: t.inputTokens, + CacheCreationInputTokens: t.cacheCreationTokens, + CacheReadInputTokens: t.cacheReadTokens, + }, }, } t.writeEvent("message_start", ev) @@ -330,7 +354,13 @@ func (t *SSETranslator) emitMessageDelta() { Type: "message_delta", StopReason: t.stopReason, }, - Usage: &StreamUsage{OutputTokens: t.outputTokens}, + // Cumulative final totals -- see the overwrite-never-accumulate note + // in FeedLine's usage capture above. + Usage: &StreamUsage{ + OutputTokens: t.outputTokens, + CacheCreationInputTokens: t.cacheCreationTokens, + CacheReadInputTokens: t.cacheReadTokens, + }, }) } diff --git a/apps/edge-api/internal/anthropic/translate_request.go b/apps/edge-api/internal/anthropic/translate_request.go index 47caa5706..6f84f7a1e 100644 --- a/apps/edge-api/internal/anthropic/translate_request.go +++ b/apps/edge-api/internal/anthropic/translate_request.go @@ -3,9 +3,16 @@ package anthropic import ( "encoding/json" "fmt" + "log/slog" "strings" + "unicode/utf8" ) +// maxSessionIDLen is OpenRouter's documented ceiling for the sticky-routing +// session_id passthrough (see MessagesRequest.SessionID). Truncated rather +// than rejected: it is a routing hint, not a correctness-bearing field. +const maxSessionIDLen = 256 + // ToOAIRequest lowers an Anthropic MessagesRequest to an internal OpenAI-shaped // OAIRequest that can be forwarded through the existing LiteLLM dispatch path. // It never leaks provider names; the model alias is passed through as-is so the @@ -13,10 +20,10 @@ import ( func ToOAIRequest(req MessagesRequest) (OAIRequest, error) { var msgs []OAIMessage - if req.System.Text != "" { + if req.System.Text != "" || len(req.System.Blocks) > 0 { msgs = append(msgs, OAIMessage{ Role: "system", - Content: OAIMessageContent{Text: req.System.Text}, + Content: systemContent(req.System), }) } @@ -29,11 +36,21 @@ func ToOAIRequest(req MessagesRequest) (OAIRequest, error) { } out := OAIRequest{ - Model: req.Model, - Messages: msgs, - Temperature: req.Temperature, - TopP: req.TopP, - Stream: req.Stream, + Model: req.Model, + Messages: msgs, + Temperature: req.Temperature, + TopP: req.TopP, + Stream: req.Stream, + CacheControl: req.CacheControl, + } + + if req.SessionID != "" { + id, truncated := truncateSessionID(req.SessionID) + if truncated { + slog.Warn("anthropic session_id truncated to the OpenRouter ceiling", + "original_len", len(req.SessionID), "truncated_len", len(id)) + } + out.SessionID = id } if req.MaxTokens > 0 { @@ -60,6 +77,43 @@ func ToOAIRequest(req MessagesRequest) (OAIRequest, error) { return out, nil } +// systemContent lowers a SystemField to an OAIMessageContent. When no block +// carries a cache breakpoint it reproduces the exact plain-string output this +// function always emitted (the regression guard: an uncached system prompt +// must serialize identically to before this change), because collapsing to a +// flat string cannot represent a per-block cache_control -- a request that +// does use one gets the block-array form instead, each block's own +// CacheControl carried onto its OAIContentPart. +func systemContent(sf SystemField) OAIMessageContent { + needsBlocks := false + for _, bl := range sf.Blocks { + if bl.CacheControl != nil { + needsBlocks = true + break + } + } + if !needsBlocks { + return OAIMessageContent{Text: sf.Text} + } + + parts := make([]OAIContentPart, 0, len(sf.Blocks)) + for _, bl := range sf.Blocks { + if bl.Type != "text" { + // Anthropic's system field only ever carries text blocks; a + // non-text block here would already have been silently ignored + // by SystemField.UnmarshalJSON's Text concatenation, so dropping + // it here too is not a new gap this change introduces. + continue + } + parts = append(parts, OAIContentPart{ + Type: "text", + Text: bl.Text, + CacheControl: bl.CacheControl, + }) + } + return OAIMessageContent{Parts: parts} +} + // convertMessage converts a single Anthropic message to one or more OAIMessages. // Parallel tool calls send multiple tool_result blocks in one Anthropic message; // each must become a separate OAI "tool" message so the LLM receives all results. @@ -89,9 +143,10 @@ func convertMessage(m Message) ([]OAIMessage, error) { } content := toolResultText(bl) toolResults = append(toolResults, OAIMessage{ - Role: "tool", - Content: OAIMessageContent{Text: content}, - ToolCallID: bl.ToolUseID, + Role: "tool", + Content: OAIMessageContent{Text: content}, + ToolCallID: bl.ToolUseID, + CacheControl: bl.CacheControl, }) } else if len(toolResults) > 0 { // tool_result blocks were already seen; a non-tool_result block after them is invalid. @@ -113,7 +168,7 @@ func convertMessage(m Message) ([]OAIMessage, error) { for _, bl := range m.Content.Blocks { switch bl.Type { case "text": - parts = append(parts, OAIContentPart{Type: "text", Text: bl.Text}) + parts = append(parts, OAIContentPart{Type: "text", Text: bl.Text, CacheControl: bl.CacheControl}) case "image": if bl.Source == nil { @@ -126,8 +181,9 @@ func convertMessage(m Message) ([]OAIMessage, error) { dataURI = bl.Source.URL } parts = append(parts, OAIContentPart{ - Type: "image_url", - ImageURL: &OAIImageURL{URL: dataURI}, + Type: "image_url", + ImageURL: &OAIImageURL{URL: dataURI}, + CacheControl: bl.CacheControl, }) case "tool_use": @@ -142,6 +198,7 @@ func convertMessage(m Message) ([]OAIMessage, error) { Name: bl.Name, Arguments: args, }, + CacheControl: bl.CacheControl, }) } } @@ -150,8 +207,22 @@ func convertMessage(m Message) ([]OAIMessage, error) { if len(toolCalls) > 0 && len(parts) == 0 { return []OAIMessage{{Role: m.Role, ToolCalls: toolCalls}}, nil } - // Tool calls plus text content. + // Tool calls plus text/image content. A part with a cache breakpoint + // keeps the block-array form so it is not lost; otherwise this + // reproduces the exact flat-string concatenation this function always + // emitted (including its pre-existing wart of silently dropping any + // image part mixed with tool calls -- see the PR body's dropped-field + // audit; fixing that is a separate, non-cache_control change and out of + // this fix's surgical scope), which is the regression guard for the + // overwhelmingly common no-cache-control case. if len(toolCalls) > 0 { + if partsNeedArrayForm(parts) { + return []OAIMessage{{ + Role: m.Role, + Content: OAIMessageContent{Parts: parts}, + ToolCalls: toolCalls, + }}, nil + } var contentStr string for _, p := range parts { if p.Type == "text" { @@ -165,13 +236,26 @@ func convertMessage(m Message) ([]OAIMessage, error) { }}, nil } - // Pure text/image parts: use array form for vision, string for text-only. - if len(parts) == 1 && parts[0].Type == "text" { + // Pure text/image parts: use array form for vision or a cache breakpoint, + // string for a lone uncached text block. + if len(parts) == 1 && parts[0].Type == "text" && parts[0].CacheControl == nil { return []OAIMessage{{Role: m.Role, Content: OAIMessageContent{Text: parts[0].Text}}}, nil } return []OAIMessage{{Role: m.Role, Content: OAIMessageContent{Parts: parts}}}, nil } +// partsNeedArrayForm reports whether a set of content parts must keep the +// block-array form rather than collapse to a flat string: true if any part +// carries a cache breakpoint, which a flat string cannot address. +func partsNeedArrayForm(parts []OAIContentPart) bool { + for _, p := range parts { + if p.CacheControl != nil { + return true + } + } + return false +} + // toolResultText extracts the text content from a tool_result block. func toolResultText(bl ContentBlock) string { if bl.Content == nil { @@ -204,6 +288,7 @@ func convertTools(tools []Tool) ([]OAITool, error) { Description: t.Description, Parameters: params, }, + CacheControl: t.CacheControl, }) } return out, nil @@ -223,7 +308,7 @@ func convertToolChoice(tc *ToolChoice) OAIToolChoice { case "tool": return OAIToolChoice{ Named: &OAINamedToolChoice{ - Type: "function", + Type: "function", Function: OAINamedToolChoiceFunction{Name: tc.Name}, }, } @@ -231,3 +316,78 @@ func convertToolChoice(tc *ToolChoice) OAIToolChoice { return OAIToolChoice{Sentinel: "auto"} } } + +// truncateSessionID cuts id to at most maxSessionIDLen bytes at a rune +// boundary. A plain byte-index slice (id[:maxSessionIDLen]) can land inside +// a multibyte UTF-8 rune; json.Marshal would then silently replace the +// corrupted tail byte with U+FFFD, so the value forwarded to OpenRouter +// would differ from any prefix the caller actually sent. Reports whether it +// actually cut anything, so the caller can log it. +func truncateSessionID(id string) (string, bool) { + if len(id) <= maxSessionIDLen { + return id, false + } + cut := maxSessionIDLen + for cut > 0 && !utf8.RuneStart(id[cut]) { + cut-- + } + return id[:cut], true +} + +// maxCacheControlBreakpoints is Anthropic's documented per-request cap. +// Enforced locally so an unbounded number of breakpoints cannot make Hive +// forward unbounded upstream work on a client's behalf. +const maxCacheControlBreakpoints = 4 + +// validateCacheControl checks every cache_control breakpoint in the request +// (root, system blocks, message content blocks, tool_result content +// nested blocks are deliberately excluded -- Anthropic's cache_control lives +// on a tool_result block itself, not inside its nested content, so nothing +// there can validly carry one -- and tool definitions) against Anthropic's +// documented shape: type must be "ephemeral", ttl if set must be "5m" or +// "1h", and no more than maxCacheControlBreakpoints total. Anthropic's real +// API already rejects a violation, so this is a pure fail-fast: catching a +// client mistake (or a script fuzzing this endpoint) before a full round +// trip to a paid upstream, not a new restriction on anything valid. +func validateCacheControl(req MessagesRequest) error { + count := 0 + check := func(cc *CacheControl) error { + if cc == nil { + return nil + } + count++ + if cc.Type != "ephemeral" { + return fmt.Errorf("cache_control.type must be %q, got %q", "ephemeral", cc.Type) + } + if cc.TTL != "" && cc.TTL != "5m" && cc.TTL != "1h" { + return fmt.Errorf("cache_control.ttl must be %q or %q, got %q", "5m", "1h", cc.TTL) + } + return nil + } + + if err := check(req.CacheControl); err != nil { + return err + } + for _, bl := range req.System.Blocks { + if err := check(bl.CacheControl); err != nil { + return err + } + } + for _, m := range req.Messages { + for _, bl := range m.Content.Blocks { + if err := check(bl.CacheControl); err != nil { + return err + } + } + } + for _, t := range req.Tools { + if err := check(t.CacheControl); err != nil { + return err + } + } + + if count > maxCacheControlBreakpoints { + return fmt.Errorf("at most %d cache_control breakpoints per request, got %d", maxCacheControlBreakpoints, count) + } + return nil +} diff --git a/apps/edge-api/internal/anthropic/translate_response.go b/apps/edge-api/internal/anthropic/translate_response.go index 8fca967ba..5c376df88 100644 --- a/apps/edge-api/internal/anthropic/translate_response.go +++ b/apps/edge-api/internal/anthropic/translate_response.go @@ -3,6 +3,7 @@ package anthropic import ( "encoding/json" "fmt" + "log/slog" "strings" ) @@ -25,10 +26,7 @@ func FromOAIResponse(resp OAIResponse, clientAlias string) MessagesResponse { Type: "message", Role: "assistant", Model: model, - Usage: ResponseUsage{ - InputTokens: resp.Usage.PromptTokens, - OutputTokens: resp.Usage.CompletionTokens, - }, + Usage: anthropicUsage(resp.Usage, model, resp.Model), } if len(resp.Choices) == 0 { @@ -65,6 +63,54 @@ func FromOAIResponse(resp OAIResponse, clientAlias string) MessagesResponse { return out } +// anthropicUsage converts an inclusive OpenAI/OpenRouter usage object into the +// exclusive shape Anthropic clients (Claude Code included) read cache savings +// from. clientAlias and upstreamModel exist only to name the alarm in +// freshInputTokens; see its doc comment for why this is a subtraction, never +// an addition, and ResponseUsage's doc comment for the shape itself. +func anthropicUsage(u OAIUsage, clientAlias, upstreamModel string) ResponseUsage { + cacheRead, cacheWrite := 0, 0 + if u.PromptTokensDetails != nil { + cacheRead = u.PromptTokensDetails.CachedTokens + cacheWrite = u.PromptTokensDetails.CacheWriteTokens + } + return ResponseUsage{ + InputTokens: freshInputTokens(u.PromptTokens, cacheRead, cacheWrite, clientAlias, upstreamModel), + OutputTokens: u.CompletionTokens, + CacheCreationInputTokens: cacheWrite, + CacheReadInputTokens: cacheRead, + } +} + +// freshInputTokens recovers the Anthropic-exclusive "fresh, uncached" input +// count from OpenRouter's inclusive prompt_tokens, which already contains +// both the cache read and cache write tokens: fresh = prompt - read - write, +// SUBTRACT never ADD (Anthropic-native reporting, which this gateway never +// actually receives since every route goes through OpenRouter, is the +// opposite ADD convention -- getting these swapped either nearly doubles +// billed input on every warm turn or, subtracting on top of an +// already-exclusive number, drives it negative). +// +// A negative result is CLAMPED to zero for the client-facing wire (never a +// negative token count) AND ALARMED via slog.Warn naming clientAlias and +// upstreamModel: per the cache-pricing research doc's section 5, a negative +// here means the upstream inclusive/exclusive shape assumption broke, and +// this is the one place in the request lifecycle that already has the +// numbers in hand to catch it. The equivalent alarm for the actual BILLED +// amount belongs to inference/pricing.go's CreditsForTokens +// (feat/cache-aware-billing, out of this package's scope) -- this function +// only ever decides what the client sees, never what gets billed. +func freshInputTokens(promptTokens, cacheRead, cacheWrite int, clientAlias, upstreamModel string) int { + fresh := promptTokens - cacheRead - cacheWrite + if fresh < 0 { + slog.Warn("anthropic cache usage: fresh input tokens went negative, clamped to zero", + "alias", clientAlias, "upstream_model", upstreamModel, + "prompt_tokens", promptTokens, "cache_read_tokens", cacheRead, "cache_write_tokens", cacheWrite) + return 0 + } + return fresh +} + // mapFinishReason converts an OpenAI finish_reason to an Anthropic stop_reason. func mapFinishReason(reason string) string { switch reason { diff --git a/apps/edge-api/internal/anthropic/types.go b/apps/edge-api/internal/anthropic/types.go index 4be47f050..84366f96e 100644 --- a/apps/edge-api/internal/anthropic/types.go +++ b/apps/edge-api/internal/anthropic/types.go @@ -25,6 +25,19 @@ type MessagesRequest struct { StopSequences []string `json:"stop_sequences,omitempty"` Stream bool `json:"stream,omitempty"` Metadata *RequestMetadata `json:"metadata,omitempty"` + // CacheControl, at request root, is a valid alternative to a per-block + // breakpoint: it applies a single ephemeral cache marker to the whole + // request instead of the caller placing one on each block by hand. Kept + // as a pointer so an absent key stays absent (see CacheControl doc). + CacheControl *CacheControl `json:"cache_control,omitempty"` + // SessionID is not part of Anthropic's own /v1/messages schema; it exists + // here purely as passthrough for a client or proxy that injects it to ask + // OpenRouter for sticky provider routing, which is what keeps a warm + // cache addressed at the same upstream instance across turns. Preserved + // verbatim rather than dropped, capped at the 256 chars OpenRouter + // documents; a longer value is truncated rather than rejected, since this + // is a routing hint, not a correctness-bearing field. + SessionID string `json:"session_id,omitempty"` } // RequestMetadata carries optional caller metadata (unused in routing). @@ -32,10 +45,33 @@ type RequestMetadata struct { UserID string `json:"user_id,omitempty"` } +// CacheControl marks an Anthropic prompt-caching breakpoint. It is valid on a +// content block, a system prompt block, a tool definition, and the request +// root. It is always a pointer field so an absent cache_control stays absent +// on the wire: an empty {} object is a materially different request from no +// key at all (Anthropic's own API rejects certain shapes with an empty +// cache_control differently from a request that never mentions it), so a +// plain (non-pointer) struct with omitempty would be wrong here -- Go's +// encoding/json does not treat a zero-value struct as empty for omitempty +// purposes, meaning a bare CacheControl{} field would always serialize. +type CacheControl struct { + Type string `json:"type"` // "ephemeral" is the only type Anthropic defines today. + // TTL is "5m" (the default when omitted) or "1h". Omitted rather than + // forced to "5m" so a caller that never set it gets back exactly the + // request it sent, and so the 1h/5m write-price split (2x vs 1.25x) can + // be recovered downstream from what the caller actually asked for. + TTL string `json:"ttl,omitempty"` +} + // SystemField can be a plain string or an array of TextBlocks. // We use a custom unmarshaller so neither form needs an any field. type SystemField struct { Text string + // Blocks preserves the original array-form blocks (nil for a plain-string + // system field). Text alone collapses every block to one concatenated + // string, which loses each block's cache_control breakpoint; Blocks is + // what ToOAIRequest consults to keep that data when it is present. + Blocks []ContentBlock } func (s *SystemField) UnmarshalJSON(b []byte) error { @@ -55,6 +91,7 @@ func (s *SystemField) UnmarshalJSON(b []byte) error { s.Text += bl.Text } } + s.Blocks = blocks return nil } @@ -103,6 +140,10 @@ type ContentBlock struct { // type=tool_result ToolUseID string `json:"tool_use_id,omitempty"` Content *ToolResultContent `json:"content,omitempty"` + + // CacheControl is valid on any block type above: text, image, tool_use, + // and tool_result all support a cache breakpoint. + CacheControl *CacheControl `json:"cache_control,omitempty"` } // ImageSource describes an image payload (base64 or URL). @@ -138,6 +179,10 @@ type Tool struct { Name string `json:"name"` Description string `json:"description,omitempty"` InputSchema json.RawMessage `json:"input_schema"` + // CacheControl on any one tool caches the whole tools array up to and + // including that entry -- Anthropic has no separate "cache the tools as + // a group" field, this per-tool placement is the group mechanism. + CacheControl *CacheControl `json:"cache_control,omitempty"` } // ToolChoice controls how the model selects tools. @@ -172,9 +217,18 @@ type ResponseBlock struct { } // ResponseUsage carries token counts in the Anthropic response envelope. +// This is the EXCLUSIVE shape real Anthropic clients (Claude Code included) +// read cache savings from: InputTokens is the fresh/uncached remainder only, +// never the total prompt size, and CacheCreationInputTokens / +// CacheReadInputTokens are reported alongside it rather than folded in. See +// freshInputTokens in translate_response.go for the arithmetic that gets an +// inclusive upstream number (OpenRouter's prompt_tokens, which already +// contains the cached and written tokens) into this exclusive shape. type ResponseUsage struct { - InputTokens int `json:"input_tokens"` - OutputTokens int `json:"output_tokens"` + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` + CacheCreationInputTokens int `json:"cache_creation_input_tokens,omitempty"` + CacheReadInputTokens int `json:"cache_read_input_tokens,omitempty"` } // CountTokensResponse is the /v1/messages/count_tokens response. @@ -188,7 +242,7 @@ type CountTokensResponse struct { // StreamEvent is a typed SSE event emitted in an Anthropic streaming response. type StreamEvent struct { - Type string `json:"type"` + Type string `json:"type"` // Index is a pointer because 0 is a valid, common index (the first and // often only content block) that must still serialize, while message_start // and message_delta (which reuse this same struct and never set Index) must @@ -196,7 +250,7 @@ type StreamEvent struct { // field on those two event types. A plain int with `omitempty` dropped the // key for index 0 as readily as for "unset", which is exactly the bug this // type used to have (see TestSSETranslator_FirstBlockIndexIsPresentAndZero). - Index *int `json:"index,omitempty"` + Index *int `json:"index,omitempty"` // message_start Message *StreamMessage `json:"message,omitempty"` @@ -242,10 +296,14 @@ type StreamDelta struct { StopSequence *string `json:"stop_sequence,omitempty"` } -// StreamUsage carries token counts in streaming events. +// StreamUsage carries token counts in streaming events. Same exclusive shape +// as ResponseUsage (see its doc comment): InputTokens excludes both cache +// fields below, it is never the raw upstream prompt_tokens total. type StreamUsage struct { - InputTokens int `json:"input_tokens,omitempty"` - OutputTokens int `json:"output_tokens,omitempty"` + InputTokens int `json:"input_tokens,omitempty"` + OutputTokens int `json:"output_tokens,omitempty"` + CacheCreationInputTokens int `json:"cache_creation_input_tokens,omitempty"` + CacheReadInputTokens int `json:"cache_read_input_tokens,omitempty"` } // -------------------------------------------------------------------------- @@ -265,7 +323,7 @@ type OAIToolChoice struct { // OAINamedToolChoice is the structured form of a named function selector. type OAINamedToolChoice struct { - Type string `json:"type"` // always "function" + Type string `json:"type"` // always "function" Function OAINamedToolChoiceFunction `json:"function"` } @@ -294,6 +352,14 @@ type OAIRequest struct { TopP *float64 `json:"top_p,omitempty"` Stream bool `json:"stream,omitempty"` StreamOptions *OAIStreamOptions `json:"stream_options,omitempty"` + // CacheControl carries a request-root cache breakpoint straight through. + // This body is forwarded to LiteLLM (and from there to OpenRouter) + // essentially byte-for-byte -- see litellm_client.go's rewriteModel, + // which only ever touches the "model" key -- so this field surviving the + // marshal here is what makes it survive the whole hop. + CacheControl *CacheControl `json:"cache_control,omitempty"` + // SessionID: see MessagesRequest.SessionID. + SessionID string `json:"session_id,omitempty"` } // OAIStreamOptions mirrors OpenAI's stream_options object. IncludeUsage asks @@ -327,6 +393,11 @@ type OAIMessage struct { Content OAIMessageContent `json:"content,omitempty"` ToolCallID string `json:"tool_call_id,omitempty"` ToolCalls []OAIToolCall `json:"tool_calls,omitempty"` + // CacheControl carries a breakpoint that collapsed onto this whole + // message rather than one of its content parts -- the tool_result case, + // where Content is always a single flattened string with no parts array + // to hang a per-part cache_control on. + CacheControl *CacheControl `json:"cache_control,omitempty"` } // OAIToolCall is an assistant tool invocation. @@ -334,6 +405,9 @@ type OAIToolCall struct { ID string `json:"id"` Type string `json:"type"` // "function" Function OAIFunctionCall `json:"function"` + // CacheControl carries a breakpoint from an Anthropic tool_use content + // block onto its lowered tool-call form. + CacheControl *CacheControl `json:"cache_control,omitempty"` } // OAIFunctionCall holds the function name and JSON-stringified arguments. @@ -346,6 +420,9 @@ type OAIFunctionCall struct { type OAITool struct { Type string `json:"type"` // "function" Function OAIFunction `json:"function"` + // CacheControl sits at the top level, sibling of Type and Function, + // mirroring where Anthropic places it on its own Tool object. + CacheControl *CacheControl `json:"cache_control,omitempty"` } // OAIFunction holds a function definition. @@ -365,6 +442,9 @@ type OAIContentPart struct { Type string `json:"type"` // "text" | "image_url" Text string `json:"text,omitempty"` ImageURL *OAIImageURL `json:"image_url,omitempty"` + // CacheControl carries an Anthropic content-block breakpoint through + // unchanged: same placement (sibling of "type"), same shape. + CacheControl *CacheControl `json:"cache_control,omitempty"` } // OAIResponse is the OpenAI chat completions response shape. @@ -390,11 +470,40 @@ type OAIMsg struct { ToolCalls []OAIToolCall `json:"tool_calls,omitempty"` } -// OAIUsage holds token counts from the OpenAI response. +// OAIUsage holds token counts from the OpenAI response. PromptTokens here is +// INCLUSIVE, the OpenAI/OpenRouter convention: it already contains whatever +// PromptTokensDetails.CachedTokens and CacheWriteTokens report, unlike the +// Anthropic-native EXCLUSIVE convention ResponseUsage/StreamUsage use for the +// client-facing wire. freshInputTokens in translate_response.go does that +// conversion; nothing here does it, this type only has to capture the +// upstream number. +// +// This struct is populated by unmarshaling the bytes the inference package's +// own chat/completions handler already wrote (it does its own typed +// round-trip through inference.UsageResponse before this package ever sees a +// byte), so PromptTokensDetails only carries data end-to-end once that struct +// also declares matching fields with these same JSON names -- see the PR body +// merge-coordination note. type OAIUsage struct { - PromptTokens int `json:"prompt_tokens"` - CompletionTokens int `json:"completion_tokens"` - TotalTokens int `json:"total_tokens"` + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + PromptTokensDetails *OAIPromptTokensDetails `json:"prompt_tokens_details,omitempty"` +} + +// OAIPromptTokensDetails is the cache breakdown nested under usage in the +// OpenAI-compatible shape OpenRouter reports for a cached Anthropic request. +// This struct is only ever unmarshaled here, never marshaled out to a real +// OpenAI-compatible client (that outbound contract belongs to the +// inference package's own response types, not this one) -- it exists purely +// so this package can read the numbers back out of them for the Anthropic +// echo in translate_response.go and stream.go. +type OAIPromptTokensDetails struct { + CachedTokens int `json:"cached_tokens,omitempty"` + // CacheWriteTokens has no OpenAI-native counterpart (OpenAI itself never + // bills a cache write); OpenRouter adds it as an extension when the + // upstream model does. + CacheWriteTokens int `json:"cache_write_tokens,omitempty"` } // OAIDelta is a streaming delta chunk.