From 92aa3ab224edca8ffe99ec161df8b245f4585808 Mon Sep 17 00:00:00 2001 From: Edward Upton Date: Sun, 15 Mar 2026 00:50:00 -0700 Subject: [PATCH] fix(anthropic): deterministic tool schema serialization for prompt caching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Anthropic's prompt caching is prefix-based — when clients send tool definitions with non-deterministic JSON key ordering across turns, the cache key changes every turn, causing repeated cache writes instead of reads. Changes: - Add OrderedMap.SortKeys() with JSON Schema priority ordering (type, description, properties, required first, then alphabetically) so LLMs see type information before descriptions and constraints. - Add OrderedMap.SortedCopy() which returns a new tree with sorted keys sharing primitive values — avoids expensive Marshal/Unmarshal cloning. - Add ToolFunctionParameters.Normalized() that returns a shallow copy with all nested OrderedMaps sorted via SortedCopy, replacing the old Clone() + NormalizeKeyOrder() pattern. - Call Normalized() in both Chat Completions and Responses API Anthropic converters before serialization. - Replace map[string]interface{} roundtrip in excludeFields path with map[string]json.RawMessage to preserve nested byte ordering. - Passthrough mode uses MarshalSorted for deterministic map key output. Made-with: Cursor --- core/providers/anthropic/chat.go | 4 + core/providers/anthropic/responses.go | 10 ++ core/providers/anthropic/utils.go | 39 ++++---- core/schemas/chatcompletions.go | 53 +++++++++++ core/schemas/orderedmap.go | 127 ++++++++++++++++++++++++++ core/schemas/orderedmap_test.go | 80 ++++++++++++++++ 6 files changed, 297 insertions(+), 16 deletions(-) diff --git a/core/providers/anthropic/chat.go b/core/providers/anthropic/chat.go index a8e78a99fa4..75712d7587a 100644 --- a/core/providers/anthropic/chat.go +++ b/core/providers/anthropic/chat.go @@ -134,6 +134,10 @@ func ToAnthropicChatRequest(ctx *schemas.BifrostContext, bifrostReq *schemas.Bif } } + if anthropicTool.InputSchema != nil { + anthropicTool.InputSchema = anthropicTool.InputSchema.Normalized() + } + if tool.CacheControl != nil { anthropicTool.CacheControl = tool.CacheControl } diff --git a/core/providers/anthropic/responses.go b/core/providers/anthropic/responses.go index a3df52b4cfa..b54fe9b8a97 100644 --- a/core/providers/anthropic/responses.go +++ b/core/providers/anthropic/responses.go @@ -4539,6 +4539,16 @@ func convertBifrostToolToAnthropic(model string, tool *schemas.ResponsesTool) *A } } + // Normalize tool schema key ordering to ensure deterministic serialization. + // Clients (e.g. Claude Agent SDK) may send non-deterministic property orderings + // across turns, which breaks Anthropic's prefix-based prompt caching since tool + // definitions are part of the serialized request prefix. + // Normalized() returns a shallow copy with sorted key slices, so the + // caller-owned tool.ResponsesToolFunction.Parameters is never mutated. + if anthropicTool.InputSchema != nil { + anthropicTool.InputSchema = anthropicTool.InputSchema.Normalized() + } + if tool.CacheControl != nil { anthropicTool.CacheControl = tool.CacheControl } diff --git a/core/providers/anthropic/utils.go b/core/providers/anthropic/utils.go index 111df3c04a8..9f20d30fbfc 100644 --- a/core/providers/anthropic/utils.go +++ b/core/providers/anthropic/utils.go @@ -116,7 +116,7 @@ func getRequestBodyForResponses(ctx *schemas.BifrostContext, request *schemas.Bi delete(requestBody, field) } } - jsonBody, err = sonic.Marshal(requestBody) + jsonBody, err = providerUtils.MarshalSorted(requestBody) if err != nil { return nil, providerUtils.NewBifrostOperationError(schemas.ErrProviderRequestMarshal, err, providerName) } @@ -160,21 +160,13 @@ func getRequestBodyForResponses(ctx *schemas.BifrostContext, request *schemas.Bi return nil, providerUtils.NewBifrostOperationError(schemas.ErrProviderRequestMarshal, err, providerName) } } - } else { - // Remove excluded fields - if len(excludeFields) > 0 { - var jsonMap map[string]interface{} - if err := sonic.Unmarshal(jsonBody, &jsonMap); err != nil { - return nil, providerUtils.NewBifrostOperationError(schemas.ErrProviderRequestMarshal, err, providerName) - } - for _, field := range excludeFields { - delete(jsonMap, field) - } - // Re-marshal the map - jsonBody, err = sonic.Marshal(jsonMap) - if err != nil { - return nil, providerUtils.NewBifrostOperationError(schemas.ErrProviderRequestMarshal, err, providerName) - } + } else if len(excludeFields) > 0 { + // Remove top-level keys while preserving nested JSON byte ordering + // (avoids roundtrip through map[string]interface{} which loses key order + // in nested objects like tool input_schema). + jsonBody, err = excludeTopLevelJSONKeys(jsonBody, excludeFields) + if err != nil { + return nil, providerUtils.NewBifrostOperationError(schemas.ErrProviderRequestMarshal, err, providerName) } } } @@ -1646,3 +1638,18 @@ func IsClaudeCodeRequest(ctx *schemas.BifrostContext) bool { } return false } + +// excludeTopLevelJSONKeys removes the specified top-level keys from a JSON +// object without parsing nested values. Nested bytes are preserved as-is, +// which keeps the original key ordering within tool schemas and other +// deeply nested structures. +func excludeTopLevelJSONKeys(data []byte, keys []string) ([]byte, error) { + var raw map[string]json.RawMessage + if err := sonic.Unmarshal(data, &raw); err != nil { + return nil, fmt.Errorf("anthropic: unmarshalling for field exclusion: %w", err) + } + for _, k := range keys { + delete(raw, k) + } + return sonic.Marshal(raw) +} diff --git a/core/schemas/chatcompletions.go b/core/schemas/chatcompletions.go index 86537620a1d..2384f0a08dc 100644 --- a/core/schemas/chatcompletions.go +++ b/core/schemas/chatcompletions.go @@ -409,6 +409,59 @@ func (t *ToolFunctionParameters) UnmarshalJSON(data []byte) error { return nil } +// Normalized returns a shallow copy of the ToolFunctionParameters with all +// OrderedMap keys sorted using JSON Schema priority ordering (type, +// description, properties, required first, then alphabetically). The copy +// shares primitive values with the original but has independent key slices, +// so sorting does not mutate the caller's data. +// +// The captured keyOrder is cleared so the struct field declaration order is +// used for the top-level keys. This produces deterministic JSON serialization +// regardless of the client's original key ordering, which is critical for +// Anthropic's prefix-based prompt caching. +func (t *ToolFunctionParameters) Normalized() *ToolFunctionParameters { + if t == nil { + return nil + } + out := *t + out.keyOrder = JSONKeyOrder{} + out.Properties = t.Properties.SortedCopy() + out.Defs = t.Defs.SortedCopy() + out.Definitions = t.Definitions.SortedCopy() + out.Items = t.Items.SortedCopy() + if len(t.AnyOf) > 0 { + out.AnyOf = make([]OrderedMap, len(t.AnyOf)) + for i := range t.AnyOf { + if cp := t.AnyOf[i].SortedCopy(); cp != nil { + out.AnyOf[i] = *cp + } + } + } + if len(t.OneOf) > 0 { + out.OneOf = make([]OrderedMap, len(t.OneOf)) + for i := range t.OneOf { + if cp := t.OneOf[i].SortedCopy(); cp != nil { + out.OneOf[i] = *cp + } + } + } + if len(t.AllOf) > 0 { + out.AllOf = make([]OrderedMap, len(t.AllOf)) + for i := range t.AllOf { + if cp := t.AllOf[i].SortedCopy(); cp != nil { + out.AllOf[i] = *cp + } + } + } + if t.AdditionalProperties != nil && t.AdditionalProperties.AdditionalPropertiesMap != nil { + out.AdditionalProperties = &AdditionalPropertiesStruct{ + AdditionalPropertiesBool: t.AdditionalProperties.AdditionalPropertiesBool, + AdditionalPropertiesMap: t.AdditionalProperties.AdditionalPropertiesMap.SortedCopy(), + } + } + return &out +} + type AdditionalPropertiesStruct struct { AdditionalPropertiesBool *bool AdditionalPropertiesMap *OrderedMap diff --git a/core/schemas/orderedmap.go b/core/schemas/orderedmap.go index be066d67164..9ec2861e4b2 100644 --- a/core/schemas/orderedmap.go +++ b/core/schemas/orderedmap.go @@ -305,6 +305,133 @@ func (om *OrderedMap) UnmarshalJSON(data []byte) error { return nil } +// jsonSchemaPriority maps JSON Schema keywords to their preferred +// serialization position. Keys present in this map are emitted first +// (in the given order), followed by all remaining keys alphabetically. +// This matches the optimal ordering for LLM tool schemas: the model +// sees type and description before properties, constraints, etc. +var jsonSchemaPriority = map[string]int{ + "type": 0, + "description": 1, + "properties": 2, + "required": 3, +} + +// SortKeys sorts the keys of this OrderedMap using JSON Schema priority +// ordering (type, description, properties, required first), with remaining +// keys sorted alphabetically. Nested *OrderedMap values are also sorted +// recursively. +func (om *OrderedMap) SortKeys() { + if om == nil || len(om.keys) == 0 { + return + } + sort.Slice(om.keys, func(i, j int) bool { + pi, okI := jsonSchemaPriority[om.keys[i]] + pj, okJ := jsonSchemaPriority[om.keys[j]] + switch { + case okI && okJ: + return pi < pj + case okI: + return true + case okJ: + return false + default: + return om.keys[i] < om.keys[j] + } + }) + for k, v := range om.values { + switch nested := v.(type) { + case *OrderedMap: + nested.SortKeys() + case map[string]interface{}: + converted := OrderedMapFromMap(nested) + converted.SortKeys() + om.values[k] = converted + case []interface{}: + sortOrderedMapsInSlice(nested) + } + } +} + +func sortOrderedMapsInSlice(s []interface{}) { + for i, item := range s { + switch v := item.(type) { + case *OrderedMap: + v.SortKeys() + case map[string]interface{}: + converted := OrderedMapFromMap(v) + converted.SortKeys() + s[i] = converted + case []interface{}: + sortOrderedMapsInSlice(v) + } + } +} + +// SortedCopy returns a new OrderedMap with keys sorted using JSON Schema +// priority ordering. Nested *OrderedMap values are recursively copied and +// sorted. Primitive values (strings, numbers, bools) are shared, not cloned. +// This is much cheaper than a full JSON marshal/unmarshal Clone because it +// only allocates new key slices and value maps. +func (om *OrderedMap) SortedCopy() *OrderedMap { + if om == nil { + return nil + } + if len(om.keys) == 0 { + return &OrderedMap{values: make(map[string]interface{})} + } + + newKeys := make([]string, len(om.keys)) + copy(newKeys, om.keys) + sort.Slice(newKeys, func(i, j int) bool { + pi, okI := jsonSchemaPriority[newKeys[i]] + pj, okJ := jsonSchemaPriority[newKeys[j]] + switch { + case okI && okJ: + return pi < pj + case okI: + return true + case okJ: + return false + default: + return newKeys[i] < newKeys[j] + } + }) + + newValues := make(map[string]interface{}, len(om.values)) + for k, v := range om.values { + switch nested := v.(type) { + case *OrderedMap: + newValues[k] = nested.SortedCopy() + case map[string]interface{}: + newValues[k] = OrderedMapFromMap(nested).SortedCopy() + case []interface{}: + newValues[k] = sortedCopySlice(nested) + default: + newValues[k] = v + } + } + + return &OrderedMap{keys: newKeys, values: newValues} +} + +func sortedCopySlice(s []interface{}) []interface{} { + out := make([]interface{}, len(s)) + for i, item := range s { + switch v := item.(type) { + case *OrderedMap: + out[i] = v.SortedCopy() + case map[string]interface{}: + out[i] = OrderedMapFromMap(v).SortedCopy() + case []interface{}: + out[i] = sortedCopySlice(v) + default: + out[i] = item + } + } + return out +} + // decodeOrderedValue reads a single JSON value from the decoder. // Objects are decoded as *OrderedMap (preserving key order). // Arrays are decoded as []interface{} with each element recursively decoded. diff --git a/core/schemas/orderedmap_test.go b/core/schemas/orderedmap_test.go index 1fcad4ac29d..d606375e849 100644 --- a/core/schemas/orderedmap_test.go +++ b/core/schemas/orderedmap_test.go @@ -314,6 +314,86 @@ func TestOrderedMap_UnmarshalThenMarshalPreservesOrder(t *testing.T) { assert.Equal(t, input, string(output)) } +func TestOrderedMap_SortKeys_PlainMapValues(t *testing.T) { + om := NewOrderedMapFromPairs( + KV("properties", map[string]interface{}{ + "z_field": map[string]interface{}{ + "type": "string", + "description": "last field", + }, + "a_field": map[string]interface{}{ + "type": "number", + "description": "first field", + }, + }), + KV("type", "object"), + ) + + om.SortKeys() + + assert.Equal(t, []string{"type", "properties"}, om.Keys(), "top-level keys should be schema-sorted") + + props, ok := om.Get("properties") + require.True(t, ok) + propsOM, ok := props.(*OrderedMap) + require.True(t, ok, "plain map should be converted to *OrderedMap, got %T", props) + + assert.Equal(t, []string{"a_field", "z_field"}, propsOM.Keys(), "nested plain map keys should be sorted") + + zField, ok := propsOM.Get("z_field") + require.True(t, ok) + zFieldOM, ok := zField.(*OrderedMap) + require.True(t, ok, "deeply nested plain map should be converted to *OrderedMap, got %T", zField) + assert.Equal(t, []string{"type", "description"}, zFieldOM.Keys(), "deeply nested keys should be schema-sorted") +} + +func TestOrderedMap_SortKeys_PlainMapInSlice(t *testing.T) { + om := NewOrderedMapFromPairs( + KV("anyOf", []interface{}{ + map[string]interface{}{ + "description": "first option", + "type": "string", + }, + }), + ) + + om.SortKeys() + + anyOf, ok := om.Get("anyOf") + require.True(t, ok) + slice, ok := anyOf.([]interface{}) + require.True(t, ok) + require.Len(t, slice, 1) + + elemOM, ok := slice[0].(*OrderedMap) + require.True(t, ok, "plain map in slice should be converted to *OrderedMap, got %T", slice[0]) + assert.Equal(t, []string{"type", "description"}, elemOM.Keys()) +} + +func TestOrderedMap_SortedCopy_PlainMapValues(t *testing.T) { + om := NewOrderedMapFromPairs( + KV("properties", map[string]interface{}{ + "z_field": "string", + "a_field": "number", + }), + KV("type", "object"), + ) + + sorted := om.SortedCopy() + + assert.Equal(t, []string{"type", "properties"}, sorted.Keys()) + + props, ok := sorted.Get("properties") + require.True(t, ok) + propsOM, ok := props.(*OrderedMap) + require.True(t, ok, "plain map should be converted to *OrderedMap in SortedCopy, got %T", props) + assert.Equal(t, []string{"a_field", "z_field"}, propsOM.Keys()) + + origProps, _ := om.Get("properties") + _, isMap := origProps.(map[string]interface{}) + assert.True(t, isMap, "original should be unmodified (still a plain map)") +} + func TestOrderedMap_EmptyArray(t *testing.T) { input := `{"items":[],"name":"test"}`