diff --git a/core/providers/anthropic/anthropic.go b/core/providers/anthropic/anthropic.go index 56c62af4e54..21bf81db78d 100644 --- a/core/providers/anthropic/anthropic.go +++ b/core/providers/anthropic/anthropic.go @@ -726,6 +726,7 @@ func HandleAnthropicChatCompletionStreaming( // Check for structured output tool name and track state var structuredOutputToolName string var isAccumulatingStructuredOutput bool + var consumedStructuredOutput bool // true once the SO tool block has been fully streamed as content if toolName, ok := ctx.Value(schemas.BifrostContextKeyStructuredOutputToolName).(string); ok { structuredOutputToolName = toolName } @@ -825,9 +826,11 @@ func HandleAnthropicChatCompletionStreaming( mappedReason := ConvertAnthropicFinishReasonToBifrost(*event.Delta.StopReason) finishReason = &mappedReason - // Override finish reason for structured output - // When structured output is used, tool_use stop reason should appear as "stop" to the client - if structuredOutputToolName != "" && *finishReason == string(schemas.BifrostFinishReasonToolCalls) { + // Override finish reason for structured output only when the SO tool + // was consumed into content AND no real tool calls were also emitted. + // streamState.nextToolCallIndex > 0 means real tool_use blocks were seen. + if consumedStructuredOutput && streamState.nextToolCallIndex == 0 && + *finishReason == string(schemas.BifrostFinishReasonToolCalls) { stopReason := string(schemas.BifrostFinishReasonStop) finishReason = &stopReason } @@ -884,6 +887,7 @@ func HandleAnthropicChatCompletionStreaming( // Check for content block stop if event.Type == AnthropicStreamEventTypeContentBlockStop && isAccumulatingStructuredOutput { isAccumulatingStructuredOutput = false + consumedStructuredOutput = true continue } } diff --git a/core/providers/anthropic/chat.go b/core/providers/anthropic/chat.go index 43d75761f3d..28a709db592 100644 --- a/core/providers/anthropic/chat.go +++ b/core/providers/anthropic/chat.go @@ -409,10 +409,16 @@ func ToAnthropicChatRequest(ctx *schemas.BifrostContext, bifrostReq *schemas.Bif responseFormatTool := convertChatResponseFormatToTool(ctx, bifrostReq.Params) if responseFormatTool != nil { anthropicReq.Tools = append(anthropicReq.Tools, *responseFormatTool) - // Force the model to use this specific tool - anthropicReq.ToolChoice = &AnthropicToolChoice{ - Type: "tool", - Name: responseFormatTool.Name, + // Anthropic rejects forced tool_choice when extended thinking is active. + // Skip forcing tool_choice in that case; the model may still call the tool. + thinkingEnabled := bifrostReq.Params.Reasoning != nil && + (bifrostReq.Params.Reasoning.MaxTokens != nil || + (bifrostReq.Params.Reasoning.Effort != nil && *bifrostReq.Params.Reasoning.Effort != "none")) + if !thinkingEnabled { + anthropicReq.ToolChoice = &AnthropicToolChoice{ + Type: "tool", + Name: responseFormatTool.Name, + } } } } else { @@ -790,6 +796,7 @@ func (response *AnthropicMessageResponse) ToBifrostChatResponse(ctx *schemas.Bif structuredOutputToolName = toolName } } + var usedStructuredOutputTool bool // Collect all content and tool calls into a single message var toolCalls []schemas.ChatAssistantMessageToolCall @@ -825,6 +832,7 @@ func (response *AnthropicMessageResponse) ToBifrostChatResponse(ctx *schemas.Bif jsonStr = "{}" } contentStr = &jsonStr + usedStructuredOutputTool = true continue // Skip adding to toolCalls } @@ -914,6 +922,12 @@ func (response *AnthropicMessageResponse) ToBifrostChatResponse(ctx *schemas.Bif FinishReason: func() *string { if response.StopReason != "" { mapped := ConvertAnthropicFinishReasonToBifrost(response.StopReason) + // When the structured output tool was folded back into text content, the + // stop reason should be "stop", not "tool_calls". + if usedStructuredOutputTool && len(toolCalls) == 0 && + mapped == string(schemas.BifrostFinishReasonToolCalls) { + mapped = string(schemas.BifrostFinishReasonStop) + } return &mapped } return nil @@ -1186,26 +1200,8 @@ func (chunk *AnthropicStreamEvent) ToBifrostChatCompletionStream(ctx *schemas.Bi } case AnthropicStreamDeltaTypeInputJSON: - // Handle tool use streaming - accumulate partial JSON + // Handle tool use streaming - accumulate partial JSON. if chunk.Delta.PartialJSON != nil { - if structuredOutputToolName != "" { - // Structured output: stream JSON as content - streamResponse := &schemas.BifrostChatResponse{ - Object: "chat.completion.chunk", - Choices: []schemas.BifrostResponseChoice{ - { - Index: 0, - ChatStreamResponseChoice: &schemas.ChatStreamResponseChoice{ - Delta: &schemas.ChatStreamResponseChoiceDelta{ - Content: chunk.Delta.PartialJSON, - }, - }, - }, - }, - } - return streamResponse, nil, false - } - // Resolve which tool-call this delta belongs to via the content-block index. toolCallIdx := state.contentBlockToToolCallIdx[*chunk.Index] diff --git a/core/providers/anthropic/chat_test.go b/core/providers/anthropic/chat_test.go index 24accf77006..613cb896289 100644 --- a/core/providers/anthropic/chat_test.go +++ b/core/providers/anthropic/chat_test.go @@ -845,3 +845,267 @@ func TestToAnthropicChatRequest_NonOpus47_NoDefaultDisplay(t *testing.T) { t.Errorf("expected Display to be nil for non-Opus 4.7, got %q", *result.Thinking.Display) } } + +// --------------------------------------------------------------------------- +// Structured output (response_format: json_schema) round-trip tests +// --------------------------------------------------------------------------- + +// makeSOResponseFormat returns a response_format interface value in the +// OpenAI wire format expected by convertChatResponseFormatToTool. +func makeSOResponseFormat(schemaName string) interface{} { + return map[string]interface{}{ + "type": "json_schema", + "json_schema": map[string]interface{}{ + "name": schemaName, + "schema": map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "color": map[string]interface{}{"type": "string"}, + "animal": map[string]interface{}{"type": "string"}, + }, + "required": []interface{}{"color", "animal"}, + }, + }, + } +} + +// TestToAnthropicChatRequest_StructuredOutput_Vertex_NoThinking verifies that when +// response_format=json_schema is passed to a Vertex-targeted request without thinking, +// Bifrost adds a synthetic bf_so_* tool AND forces tool_choice to that tool. +func TestToAnthropicChatRequest_StructuredOutput_Vertex_NoThinking(t *testing.T) { + rf := makeSOResponseFormat("my_schema") + bifrostReq := &schemas.BifrostChatRequest{ + Provider: schemas.Vertex, + Model: "claude-opus-4-6", + Input: []schemas.ChatMessage{ + {Role: schemas.ChatMessageRoleUser, Content: &schemas.ChatMessageContent{ContentStr: schemas.Ptr("Hello")}}, + }, + Params: &schemas.ChatParameters{ + ResponseFormat: &rf, + }, + } + + ctx, cancel := schemas.NewBifrostContextWithCancel(context.Background()) + defer cancel() + result, err := ToAnthropicChatRequest(ctx, bifrostReq) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // A synthetic tool with the bf_so_ prefix must be present. + var soTool *AnthropicTool + for i := range result.Tools { + if len(result.Tools[i].Name) > 6 && result.Tools[i].Name[:6] == "bf_so_" { + soTool = &result.Tools[i] + break + } + } + if soTool == nil { + t.Fatal("expected a synthetic bf_so_* tool to be added for Vertex structured output") + } + + // ToolChoice must be set and must point at the SO tool. + if result.ToolChoice == nil { + t.Fatal("expected ToolChoice to be set when thinking is disabled") + } + if result.ToolChoice.Type != "tool" { + t.Errorf("expected ToolChoice.Type=tool, got %q", result.ToolChoice.Type) + } + if result.ToolChoice.Name != soTool.Name { + t.Errorf("expected ToolChoice.Name=%q, got %q", soTool.Name, result.ToolChoice.Name) + } +} + +// TestToAnthropicChatRequest_StructuredOutput_Vertex_ThinkingEffort verifies that when +// response_format=json_schema + reasoning_effort='medium' is sent to Vertex, Bifrost +// still adds the synthetic tool but does NOT set tool_choice (to avoid Anthropic's +// "Thinking may not be enabled when tool_choice forces tool use" 400 error). +func TestToAnthropicChatRequest_StructuredOutput_Vertex_ThinkingEffort(t *testing.T) { + rf := makeSOResponseFormat("my_schema") + effort := "medium" + bifrostReq := &schemas.BifrostChatRequest{ + Provider: schemas.Vertex, + Model: "claude-opus-4-6", + Input: []schemas.ChatMessage{ + {Role: schemas.ChatMessageRoleUser, Content: &schemas.ChatMessageContent{ContentStr: schemas.Ptr("Hello")}}, + }, + Params: &schemas.ChatParameters{ + MaxCompletionTokens: new(16000), + ResponseFormat: &rf, + Reasoning: &schemas.ChatReasoning{Effort: &effort}, + }, + } + + ctx, cancel := schemas.NewBifrostContextWithCancel(context.Background()) + defer cancel() + result, err := ToAnthropicChatRequest(ctx, bifrostReq) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Synthetic tool must still be present so the model knows the schema. + found := false + for _, tool := range result.Tools { + if len(tool.Name) > 6 && tool.Name[:6] == "bf_so_" { + found = true + break + } + } + if !found { + t.Fatal("expected synthetic bf_so_* tool to be present even with thinking enabled") + } + + // ToolChoice must NOT be set — forcing it would trigger a 400 from Anthropic. + if result.ToolChoice != nil { + t.Errorf("expected ToolChoice to be nil when thinking is enabled (effort=%q), got %+v", effort, result.ToolChoice) + } +} + +// TestToAnthropicChatRequest_StructuredOutput_Vertex_ThinkingMaxTokens is the same as +// the effort variant but uses explicit budget_tokens reasoning instead. +func TestToAnthropicChatRequest_StructuredOutput_Vertex_ThinkingMaxTokens(t *testing.T) { + rf := makeSOResponseFormat("my_schema") + maxTok := 4000 + bifrostReq := &schemas.BifrostChatRequest{ + Provider: schemas.Vertex, + Model: "claude-opus-4-6", + Input: []schemas.ChatMessage{ + {Role: schemas.ChatMessageRoleUser, Content: &schemas.ChatMessageContent{ContentStr: schemas.Ptr("Hello")}}, + }, + Params: &schemas.ChatParameters{ + MaxCompletionTokens: new(16000), + ResponseFormat: &rf, + Reasoning: &schemas.ChatReasoning{MaxTokens: &maxTok}, + }, + } + + ctx, cancel := schemas.NewBifrostContextWithCancel(context.Background()) + defer cancel() + result, err := ToAnthropicChatRequest(ctx, bifrostReq) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if result.ToolChoice != nil { + t.Errorf("expected ToolChoice to be nil when thinking (MaxTokens) is enabled, got %+v", result.ToolChoice) + } +} + +// TestToBifrostChatResponse_StructuredOutput_FinishReasonStop verifies that when +// the model responds with only the synthetic SO tool (no real tool calls), the +// finish_reason is mapped to "stop", not "tool_calls". +func TestToBifrostChatResponse_StructuredOutput_FinishReasonStop(t *testing.T) { + soToolName := "bf_so_my_schema" + jsonInput, err := json.Marshal(map[string]interface{}{"color": "blue", "animal": "fox"}) + if err != nil { + t.Fatalf("failed to marshal structured output input: %v", err) + } + + response := &AnthropicMessageResponse{ + ID: "msg_so_test", + Type: "message", + Role: "assistant", + Model: "claude-opus-4-6", + Content: []AnthropicContentBlock{ + { + Type: AnthropicContentBlockTypeToolUse, + ID: schemas.Ptr("toolu_001"), + Name: schemas.Ptr(soToolName), + Input: json.RawMessage(jsonInput), + }, + }, + StopReason: AnthropicStopReasonToolUse, + } + + ctx, cancel := schemas.NewBifrostContextWithCancel(context.Background()) + defer cancel() + ctx.SetValue(schemas.BifrostContextKeyStructuredOutputToolName, soToolName) + + result := response.ToBifrostChatResponse(ctx) + if result == nil { + t.Fatal("expected non-nil result") + } + + choice := result.Choices[0] + + // Content must be the JSON from the SO tool, not nil. + msg := choice.ChatNonStreamResponseChoice.Message + if msg.Content.ContentStr == nil { + t.Fatal("expected ContentStr to be set from the structured output tool input") + } + + // No real tool calls should be surfaced. + if msg.ChatAssistantMessage != nil && len(msg.ChatAssistantMessage.ToolCalls) > 0 { + t.Errorf("expected no tool calls in output, got %d", len(msg.ChatAssistantMessage.ToolCalls)) + } + + // Finish reason must be "stop", not "tool_calls". + if choice.FinishReason == nil { + t.Fatal("expected FinishReason to be set") + } + if *choice.FinishReason != string(schemas.BifrostFinishReasonStop) { + t.Errorf("expected FinishReason=%q, got %q", schemas.BifrostFinishReasonStop, *choice.FinishReason) + } +} + +// TestToBifrostChatResponse_StructuredOutput_MixedWithRealTools verifies that when +// both the SO tool and a real tool call appear in the response, finish_reason remains +// "tool_calls" so the caller knows to handle the real tool. +func TestToBifrostChatResponse_StructuredOutput_MixedWithRealTools(t *testing.T) { + soToolName := "bf_so_my_schema" + soInput, err := json.Marshal(map[string]interface{}{"color": "blue", "animal": "fox"}) + if err != nil { + t.Fatalf("failed to marshal SO input: %v", err) + } + realInput, err := json.Marshal(map[string]interface{}{"location": "NYC"}) + if err != nil { + t.Fatalf("failed to marshal real tool input: %v", err) + } + + response := &AnthropicMessageResponse{ + ID: "msg_so_mixed", + Type: "message", + Role: "assistant", + Model: "claude-opus-4-6", + Content: []AnthropicContentBlock{ + { + Type: AnthropicContentBlockTypeToolUse, + ID: schemas.Ptr("toolu_001"), + Name: schemas.Ptr(soToolName), + Input: json.RawMessage(soInput), + }, + { + Type: AnthropicContentBlockTypeToolUse, + ID: schemas.Ptr("toolu_real_001"), + Name: schemas.Ptr("get_weather"), + Input: json.RawMessage(realInput), + }, + }, + StopReason: AnthropicStopReasonToolUse, + } + + ctx, cancel := schemas.NewBifrostContextWithCancel(context.Background()) + defer cancel() + ctx.SetValue(schemas.BifrostContextKeyStructuredOutputToolName, soToolName) + + result := response.ToBifrostChatResponse(ctx) + if result == nil { + t.Fatal("expected non-nil result") + } + + choice := result.Choices[0] + + // The real tool call must be surfaced. + msg := choice.ChatNonStreamResponseChoice.Message + if msg.ChatAssistantMessage == nil || len(msg.ChatAssistantMessage.ToolCalls) == 0 { + t.Fatal("expected real tool calls to be present") + } + + // Finish reason must remain "tool_calls". + if choice.FinishReason == nil { + t.Fatal("expected FinishReason to be set") + } + if *choice.FinishReason != string(schemas.BifrostFinishReasonToolCalls) { + t.Errorf("expected FinishReason=%q, got %q", schemas.BifrostFinishReasonToolCalls, *choice.FinishReason) + } +} diff --git a/core/providers/anthropic/responses.go b/core/providers/anthropic/responses.go index 1e82290731a..2f68df833db 100644 --- a/core/providers/anthropic/responses.go +++ b/core/providers/anthropic/responses.go @@ -54,6 +54,8 @@ type AnthropicResponsesStreamState struct { HasEmittedMessageDelta bool // Whether we've emitted message_delta (avoids duplicate from response.completed) StructuredOutputToolName string // Name of the structured output tool (if using tool-based SO for Vertex) StructuredOutputIndex *int // Output index of the structured output tool call + UsedStructuredOutputTool bool // True when the SO tool block was actually consumed into text content + SeenRealToolCall bool // True when any non-SO tool_use/server_tool_use/mcp_tool_use content block was started } // anthropicResponsesStreamStatePool provides a pool for Anthropic responses stream state objects. @@ -174,6 +176,8 @@ func acquireAnthropicResponsesStreamState() *AnthropicResponsesStreamState { state.HasEmittedMessageDelta = false state.StructuredOutputToolName = "" state.StructuredOutputIndex = nil + state.UsedStructuredOutputTool = false + state.SeenRealToolCall = false return state } @@ -215,6 +219,8 @@ func (state *AnthropicResponsesStreamState) flush() { state.HasEmittedMessageDelta = false state.StructuredOutputToolName = "" state.StructuredOutputIndex = nil + state.UsedStructuredOutputTool = false + state.SeenRealToolCall = false } // isCompactionItem checks if a ResponsesMessage represents a compaction item @@ -333,6 +339,7 @@ func (chunk *AnthropicStreamEvent) ToBifrostResponsesStream(ctx context.Context, *chunk.ContentBlock.Name == string(AnthropicToolNameComputer) && chunk.ContentBlock.ID != nil { + state.SeenRealToolCall = true // Start accumulating computer tool state.ComputerToolID = chunk.ContentBlock.ID state.ChunkIndex = chunk.Index @@ -362,6 +369,7 @@ func (chunk *AnthropicStreamEvent) ToBifrostResponsesStream(ctx context.Context, *chunk.ContentBlock.Name == string(AnthropicToolNameWebSearch) && chunk.ContentBlock.ID != nil { + state.SeenRealToolCall = true // Start accumulating web search query (reuse shared accumulation fields) state.ChunkIndex = chunk.Index state.AccumulatedJSON = "" @@ -455,6 +463,7 @@ func (chunk *AnthropicStreamEvent) ToBifrostResponsesStream(ctx context.Context, *chunk.ContentBlock.Name == string(AnthropicToolNameWebFetch) && chunk.ContentBlock.ID != nil { + state.SeenRealToolCall = true state.ChunkIndex = chunk.Index state.AccumulatedJSON = "" state.WebFetchToolID = chunk.ContentBlock.ID @@ -620,6 +629,7 @@ func (chunk *AnthropicStreamEvent) ToBifrostResponsesStream(ctx context.Context, return nil, nil, false } + state.SeenRealToolCall = true // Function call starting - emit output_item.added with type "function_call" and status "in_progress" statusInProgress := "in_progress" itemID := "" @@ -662,6 +672,7 @@ func (chunk *AnthropicStreamEvent) ToBifrostResponsesStream(ctx context.Context, Item: item, }}, nil, false case AnthropicContentBlockTypeMCPToolUse: + state.SeenRealToolCall = true // MCP tool call starting - emit output_item.added itemID := "" if chunk.ContentBlock.ID != nil { @@ -1230,6 +1241,7 @@ func (chunk *AnthropicStreamEvent) ToBifrostResponsesStream(ctx context.Context, // Clear the buffer and tracking delete(state.ToolArgumentBuffers, outputIndex) state.StructuredOutputIndex = nil + state.UsedStructuredOutputTool = true return responses, nil, false } @@ -1305,18 +1317,22 @@ func (chunk *AnthropicStreamEvent) ToBifrostResponsesStream(ctx context.Context, case AnthropicStreamEventTypeMessageDelta: if chunk.Delta.StopReason != nil { - state.StopReason = schemas.Ptr(ConvertAnthropicFinishReasonToBifrost(*chunk.Delta.StopReason)) + mapped := ConvertAnthropicFinishReasonToBifrost(*chunk.Delta.StopReason) + if state.UsedStructuredOutputTool && !state.SeenRealToolCall && + mapped == string(schemas.BifrostFinishReasonToolCalls) { + mapped = string(schemas.BifrostFinishReasonStop) + } + state.StopReason = &mapped } // Check if integration type in ctx is anthropic if ctx.Value(schemas.BifrostContextKeyIntegrationType) == "anthropic" { // Convert usage from Anthropic format to Bifrost bifrostUsage := ConvertAnthropicUsageToBifrostUsage(chunk.Usage) - // Convert stop reason if present + // Use the already-remapped stop reason so SO overrides are preserved. var stopReason *string - if chunk.Delta != nil && chunk.Delta.StopReason != nil { - converted := ConvertAnthropicFinishReasonToBifrost(*chunk.Delta.StopReason) - stopReason = &converted + if state.StopReason != nil { + stopReason = state.StopReason } // Create response object with usage and stop reason @@ -2348,10 +2364,14 @@ func ToAnthropicResponsesRequest(ctx *schemas.BifrostContext, bifrostReq *schema anthropicReq.Tools = []AnthropicTool{} } anthropicReq.Tools = append(anthropicReq.Tools, *responseFormatTool) - // Force the model to use this specific tool - anthropicReq.ToolChoice = &AnthropicToolChoice{ - Type: "tool", - Name: responseFormatTool.Name, + thinkingEnabled := bifrostReq.Params.Reasoning != nil && + (bifrostReq.Params.Reasoning.MaxTokens != nil || + (bifrostReq.Params.Reasoning.Effort != nil && *bifrostReq.Params.Reasoning.Effort != "none")) + if !thinkingEnabled { + anthropicReq.ToolChoice = &AnthropicToolChoice{ + Type: "tool", + Name: responseFormatTool.Name, + } } } } @@ -2746,7 +2766,24 @@ func (response *AnthropicMessageResponse) ToBifrostResponsesResponse(ctx *schema bifrostResp.Model = response.Model if response.StopReason != "" { - bifrostResp.StopReason = schemas.Ptr(ConvertAnthropicFinishReasonToBifrost(response.StopReason)) + mapped := ConvertAnthropicFinishReasonToBifrost(response.StopReason) + if mapped == string(schemas.BifrostFinishReasonToolCalls) { + if soToolName, ok := ctx.Value(schemas.BifrostContextKeyStructuredOutputToolName).(string); ok && soToolName != "" { + hasRealToolUse := false + for _, block := range response.Content { + if block.Type == AnthropicContentBlockTypeServerToolUse || + block.Type == AnthropicContentBlockTypeMCPToolUse || + (block.Type == AnthropicContentBlockTypeToolUse && block.Name != nil && *block.Name != soToolName) { + hasRealToolUse = true + break + } + } + if !hasRealToolUse { + mapped = string(schemas.BifrostFinishReasonStop) + } + } + } + bifrostResp.StopReason = &mapped } if response.Usage != nil && response.Usage.ServiceTier != nil { diff --git a/core/providers/bedrock/bedrock.go b/core/providers/bedrock/bedrock.go index 97528e2e90a..bfd4ccde70a 100644 --- a/core/providers/bedrock/bedrock.go +++ b/core/providers/bedrock/bedrock.go @@ -1751,6 +1751,7 @@ func (provider *BedrockProvider) ResponsesStream(ctx *schemas.BifrostContext, po if streamEvent.Start.ToolUse.Name == structuredOutputToolName { // This is the structured output tool - start accumulating, don't forward isAccumulatingStructuredOutput = true + streamState.UsedStructuredOutputTool = true continue } } diff --git a/core/providers/bedrock/bedrock_test.go b/core/providers/bedrock/bedrock_test.go index 861c33626d2..32fc498c383 100644 --- a/core/providers/bedrock/bedrock_test.go +++ b/core/providers/bedrock/bedrock_test.go @@ -4981,3 +4981,190 @@ func TestToBedrockResponsesRequest_NonLlamaConvertResponsesToolChoiceForcesToolC require.NotNil(t, bedrockReq.ToolConfig.ToolChoice.Tool, "expected forced tool_choice for non-Llama models") assert.Equal(t, toolName, bedrockReq.ToolConfig.ToolChoice.Tool.Name) } + +// --------------------------------------------------------------------------- +// Structured output (response_format: json_schema) round-trip tests – Bedrock +// --------------------------------------------------------------------------- + +// TestBedrockToBifrostChatResponse_StructuredOutput_FinishReasonStop verifies that when +// the model returns only the synthetic bf_so_* tool block (no real tool calls), +// finish_reason is mapped to "stop", not "tool_calls". +func TestBedrockToBifrostChatResponse_StructuredOutput_FinishReasonStop(t *testing.T) { + const soToolName = "bf_so_my_schema" + + response := &bedrock.BedrockConverseResponse{ + StopReason: "tool_use", + Output: &bedrock.BedrockConverseOutput{ + Message: &bedrock.BedrockMessage{ + Role: bedrock.BedrockMessageRoleAssistant, + Content: []bedrock.BedrockContentBlock{ + { + ToolUse: &bedrock.BedrockToolUse{ + ToolUseID: "toolu_001", + Name: soToolName, + Input: json.RawMessage(`{"color":"blue","animal":"fox"}`), + }, + }, + }, + }, + }, + Usage: &bedrock.BedrockTokenUsage{InputTokens: 10, OutputTokens: 20, TotalTokens: 30}, + } + + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + ctx.SetValue(schemas.BifrostContextKeyStructuredOutputToolName, soToolName) + + result, err := response.ToBifrostChatResponse(ctx, "claude-opus-4-6") + require.NoError(t, err) + require.NotNil(t, result) + require.Len(t, result.Choices, 1, "expected exactly one choice") + + choice := result.Choices[0] + require.NotNil(t, choice.ChatNonStreamResponseChoice, "expected non-streaming response choice") + + // Content must be the JSON from the SO tool. + msg := choice.ChatNonStreamResponseChoice.Message + assert.NotNil(t, msg.Content.ContentStr, "expected ContentStr to be set from SO tool input") + + // No real tool calls should be surfaced. + if msg.ChatAssistantMessage != nil { + assert.Empty(t, msg.ChatAssistantMessage.ToolCalls, "expected no tool calls in output") + } + + // Finish reason must be "stop", not "tool_calls". + require.NotNil(t, choice.FinishReason) + assert.Equal(t, string(schemas.BifrostFinishReasonStop), *choice.FinishReason, + "expected finish_reason=stop when only SO tool was consumed") +} + +// TestBedrockToBifrostChatResponse_StructuredOutput_MixedWithRealTools verifies that +// when both the SO tool and a real tool call appear in the response, finish_reason +// remains "tool_calls" so the caller knows to handle the real tool. +func TestBedrockToBifrostChatResponse_StructuredOutput_MixedWithRealTools(t *testing.T) { + const soToolName = "bf_so_my_schema" + + response := &bedrock.BedrockConverseResponse{ + StopReason: "tool_use", + Output: &bedrock.BedrockConverseOutput{ + Message: &bedrock.BedrockMessage{ + Role: bedrock.BedrockMessageRoleAssistant, + Content: []bedrock.BedrockContentBlock{ + { + ToolUse: &bedrock.BedrockToolUse{ + ToolUseID: "toolu_001", + Name: soToolName, + Input: json.RawMessage(`{"color":"blue","animal":"fox"}`), + }, + }, + { + ToolUse: &bedrock.BedrockToolUse{ + ToolUseID: "toolu_real_001", + Name: "get_weather", + Input: json.RawMessage(`{"location":"NYC"}`), + }, + }, + }, + }, + }, + Usage: &bedrock.BedrockTokenUsage{InputTokens: 10, OutputTokens: 20, TotalTokens: 30}, + } + + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + ctx.SetValue(schemas.BifrostContextKeyStructuredOutputToolName, soToolName) + + result, err := response.ToBifrostChatResponse(ctx, "claude-opus-4-6") + require.NoError(t, err) + require.NotNil(t, result) + require.Len(t, result.Choices, 1, "expected exactly one choice") + + choice := result.Choices[0] + require.NotNil(t, choice.ChatNonStreamResponseChoice, "expected non-streaming response choice") + + // The real tool call must be surfaced. + msg := choice.ChatNonStreamResponseChoice.Message + require.NotNil(t, msg.ChatAssistantMessage) + assert.NotEmpty(t, msg.ChatAssistantMessage.ToolCalls, "expected real tool calls to be present") + + // Finish reason must remain "tool_calls". + require.NotNil(t, choice.FinishReason) + assert.Equal(t, string(schemas.BifrostFinishReasonToolCalls), *choice.FinishReason, + "expected finish_reason=tool_calls when real tool calls are also present") +} + +// TestBedrockToBifrostResponsesResponse_StructuredOutput_FinishReasonStop verifies that +// ToBifrostResponsesResponse maps stop_reason to "stop" (not "tool_calls") when only the +// synthetic SO tool was consumed. +func TestBedrockToBifrostResponsesResponse_StructuredOutput_FinishReasonStop(t *testing.T) { + const soToolName = "bf_so_user_info" + + response := &bedrock.BedrockConverseResponse{ + StopReason: "tool_use", + Output: &bedrock.BedrockConverseOutput{ + Message: &bedrock.BedrockMessage{ + Role: bedrock.BedrockMessageRoleAssistant, + Content: []bedrock.BedrockContentBlock{ + { + ToolUse: &bedrock.BedrockToolUse{ + ToolUseID: "toolu_001", + Name: soToolName, + Input: json.RawMessage(`{"name":"John Doe","age":28,"city":"Pune"}`), + }, + }, + }, + }, + }, + Usage: &bedrock.BedrockTokenUsage{InputTokens: 10, OutputTokens: 20, TotalTokens: 30}, + } + + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + ctx.SetValue(schemas.BifrostContextKeyStructuredOutputToolName, soToolName) + + result, err := response.ToBifrostResponsesResponse(ctx) + require.NoError(t, err) + require.NotNil(t, result) + require.NotNil(t, result.StopReason) + assert.Equal(t, "stop", *result.StopReason, + "expected stop_reason=stop when only SO tool was consumed") +} + +// TestBedrockToBifrostResponsesResponse_StructuredOutput_MixedWithRealTools verifies that +// stop_reason stays "tool_calls" when both the SO tool and a real tool call are present. +func TestBedrockToBifrostResponsesResponse_StructuredOutput_MixedWithRealTools(t *testing.T) { + const soToolName = "bf_so_user_info" + + response := &bedrock.BedrockConverseResponse{ + StopReason: "tool_use", + Output: &bedrock.BedrockConverseOutput{ + Message: &bedrock.BedrockMessage{ + Role: bedrock.BedrockMessageRoleAssistant, + Content: []bedrock.BedrockContentBlock{ + { + ToolUse: &bedrock.BedrockToolUse{ + ToolUseID: "toolu_001", + Name: soToolName, + Input: json.RawMessage(`{"name":"John Doe","age":28,"city":"Pune"}`), + }, + }, + { + ToolUse: &bedrock.BedrockToolUse{ + ToolUseID: "toolu_real_001", + Name: "get_weather", + Input: json.RawMessage(`{"location":"Pune"}`), + }, + }, + }, + }, + }, + Usage: &bedrock.BedrockTokenUsage{InputTokens: 10, OutputTokens: 20, TotalTokens: 30}, + } + + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + ctx.SetValue(schemas.BifrostContextKeyStructuredOutputToolName, soToolName) + + result, err := response.ToBifrostResponsesResponse(ctx) + require.NoError(t, err) + require.NotNil(t, result) + require.NotNil(t, result.StopReason) + assert.Equal(t, "tool_calls", *result.StopReason, + "expected stop_reason=tool_calls when real tool calls are also present") +} diff --git a/core/providers/bedrock/chat.go b/core/providers/bedrock/chat.go index 70c9bcbf6a8..66437eb6ad9 100644 --- a/core/providers/bedrock/chat.go +++ b/core/providers/bedrock/chat.go @@ -79,6 +79,7 @@ func (response *BedrockConverseResponse) ToBifrostChatResponse(ctx context.Conte var toolCalls []schemas.ChatAssistantMessageToolCall var reasoningDetails []schemas.ChatReasoningDetails var reasoningText string + var usedStructuredOutputTool bool if response.Output.Message != nil { for _, contentBlock := range response.Output.Message.Content { @@ -98,6 +99,7 @@ func (response *BedrockConverseResponse) ToBifrostChatResponse(ctx context.Conte if contentBlock.ToolUse.Input != nil { jsonStr := string(contentBlock.ToolUse.Input) contentStr = &jsonStr + usedStructuredOutputTool = true } continue // Skip adding to toolCalls } @@ -233,7 +235,14 @@ func (response *BedrockConverseResponse) ToBifrostChatResponse(ctx context.Conte ChatAssistantMessage: assistantMessage, }, }, - FinishReason: schemas.Ptr(convertBedrockStopReason(response.StopReason)), + FinishReason: func() *string { + mapped := convertBedrockStopReason(response.StopReason) + if usedStructuredOutputTool && len(toolCalls) == 0 && + mapped == string(schemas.BifrostFinishReasonToolCalls) { + mapped = string(schemas.BifrostFinishReasonStop) + } + return &mapped + }(), }, } var usage *schemas.BifrostLLMUsage diff --git a/core/providers/bedrock/responses.go b/core/providers/bedrock/responses.go index b95c609a5b9..fcb439ced43 100644 --- a/core/providers/bedrock/responses.go +++ b/core/providers/bedrock/responses.go @@ -37,6 +37,7 @@ type BedrockResponsesStreamState struct { CreatedAt int // Timestamp for created_at consistency HasEmittedCreated bool // Whether we've emitted response.created HasEmittedInProgress bool // Whether we've emitted response.in_progress + UsedStructuredOutputTool bool // True when the SO tool block was intercepted and converted to text content } // bedrockResponsesStreamStatePool provides a pool for Bedrock responses stream state objects. @@ -130,6 +131,7 @@ func acquireBedrockResponsesStreamState() *BedrockResponsesStreamState { state.CreatedAt = int(time.Now().Unix()) state.HasEmittedCreated = false state.HasEmittedInProgress = false + state.UsedStructuredOutputTool = false return state } @@ -205,6 +207,7 @@ func (state *BedrockResponsesStreamState) flush() { state.CreatedAt = int(time.Now().Unix()) state.HasEmittedCreated = false state.HasEmittedInProgress = false + state.UsedStructuredOutputTool = false } // ToBifrostResponsesStream converts a Bedrock stream event to a Bifrost Responses Stream response @@ -1402,7 +1405,21 @@ func FinalizeBedrockStream(state *BedrockResponsesStreamState, sequenceNumber in response.Model = *state.Model } if state.StopReason != nil { - response.StopReason = state.StopReason + stopReason := *state.StopReason + // If only the SO tool was consumed (no real tool calls in state), downgrade tool_calls → stop. + if stopReason == string(schemas.BifrostFinishReasonToolCalls) && state.UsedStructuredOutputTool { + hasRealToolCall := false + for _, toolCallID := range state.ToolCallIDs { + if toolCallID != "" { + hasRealToolCall = true + break + } + } + if !hasRealToolCall { + stopReason = string(schemas.BifrostFinishReasonStop) + } + } + response.StopReason = &stopReason } else { // Infer stop reason based on whether tool calls are present hasToolCalls := false @@ -2432,7 +2449,10 @@ func ToBedrockResponsesRequest(ctx *schemas.BifrostContext, bifrostReq *schemas. // support matrix at // https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html // (mirrors the gate applied in convertChatParameters). - if !schemas.IsLlamaModel(bifrostReq.Model) { + thinkingEnabled := bifrostReq.Params.Reasoning != nil && + (bifrostReq.Params.Reasoning.MaxTokens != nil || + (bifrostReq.Params.Reasoning.Effort != nil && *bifrostReq.Params.Reasoning.Effort != "none")) + if !schemas.IsLlamaModel(bifrostReq.Model) && !thinkingEnabled { bedrockReq.ToolConfig.ToolChoice = &BedrockToolChoice{ Tool: &BedrockToolChoiceTool{ Name: responsesStructuredOutputTool.ToolSpec.Name, @@ -2508,6 +2528,28 @@ func (response *BedrockConverseResponse) ToBifrostResponsesResponse(ctx *schemas if response.StopReason != "" { stopReason := convertBedrockStopReason(response.StopReason) + if stopReason == string(schemas.BifrostFinishReasonToolCalls) { + if toolName, hasSO := ctx.Value(schemas.BifrostContextKeyStructuredOutputToolName).(string); hasSO && toolName != "" { + hasRealToolCall := false + for _, msg := range bifrostResp.Output { + if msg.Type == nil { + continue + } + switch *msg.Type { + case schemas.ResponsesMessageTypeFunctionCall, + schemas.ResponsesMessageTypeWebSearchCall, + schemas.ResponsesMessageTypeCodeInterpreterCall: + hasRealToolCall = true + } + if hasRealToolCall { + break + } + } + if !hasRealToolCall { + stopReason = string(schemas.BifrostFinishReasonStop) + } + } + } bifrostResp.StopReason = &stopReason } diff --git a/core/providers/bedrock/utils.go b/core/providers/bedrock/utils.go index 3555d27ad17..d8ed7c9944c 100644 --- a/core/providers/bedrock/utils.go +++ b/core/providers/bedrock/utils.go @@ -354,7 +354,10 @@ func convertChatParameters(ctx *schemas.BifrostContext, bifrostReq *schemas.Bifr // and the langchain-aws ChatBedrockConverse implementation at // https://github.com/langchain-ai/langchain-aws/blob/main/libs/aws/langchain_aws/chat_models/bedrock_converse.py // (supports_tool_choice_values), which ships the same model-family gate. - if !schemas.IsLlamaModel(bifrostReq.Model) { + thinkingEnabled := bifrostReq.Params.Reasoning != nil && + (bifrostReq.Params.Reasoning.MaxTokens != nil || + (bifrostReq.Params.Reasoning.Effort != nil && *bifrostReq.Params.Reasoning.Effort != "none")) + if !schemas.IsLlamaModel(bifrostReq.Model) && !thinkingEnabled { bedrockReq.ToolConfig.ToolChoice = &BedrockToolChoice{ Tool: &BedrockToolChoiceTool{ Name: responseFormatTool.ToolSpec.Name,