Skip to content

[superseded] [fix]: Preserve document blocks in Bedrock tool results #5662

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
michaeldunn9 wants to merge 1 commit into from
Closed

[superseded] [fix]: Preserve document blocks in Bedrock tool results #5662

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions core/changelog.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[fix]: preserve documents in Bedrock tool results [@michaeldunn9](https://github.com/michaeldunn9)
24 changes: 19 additions & 5 deletions core/providers/anthropic/emptytoolresult_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,17 +36,24 @@ func TestConvertToolResultWithEmptyContent(t *testing.T) {
}
}

// TestConvertToolResultWithUnsupportedBlocks verifies tool_result content made
// solely of block types the converter does not map (e.g. document) still
// yields a serializable output.
func TestConvertToolResultWithUnsupportedBlocks(t *testing.T) {
// TestConvertToolResultWithDocumentBlock verifies a document-only tool result
// remains serializable and retains its canonical Bifrost file block.
func TestConvertToolResultWithDocumentBlock(t *testing.T) {
role := schemas.ResponsesInputMessageRoleUser
blocks := []AnthropicContentBlock{
{
Type: AnthropicContentBlockTypeToolResult,
ToolUseID: schemas.Ptr("toolu_doc"),
Content: &AnthropicContent{ContentBlocks: []AnthropicContentBlock{
{Type: AnthropicContentBlockTypeDocument},
{
Type: AnthropicContentBlockTypeDocument,
Title: schemas.Ptr("report.pdf"),
Source: &AnthropicBlockSource{SourceObj: &AnthropicSource{
Type: "base64",
MediaType: schemas.Ptr("application/pdf"),
Data: schemas.Ptr("JVBERi0xLjQ="),
}},
},
}},
},
}
Expand All @@ -55,6 +62,13 @@ func TestConvertToolResultWithUnsupportedBlocks(t *testing.T) {
if len(msgs) != 1 {
t.Fatalf("expected 1 converted message, got %d", len(msgs))
}
output := msgs[0].Output
if output == nil || len(output.ResponsesFunctionToolCallOutputBlocks) != 1 {
t.Fatalf("expected one preserved document block, got %#v", output)
}
if output.ResponsesFunctionToolCallOutputBlocks[0].Type != schemas.ResponsesInputMessageContentBlockTypeFile {
t.Fatalf("expected canonical file block, got %#v", output.ResponsesFunctionToolCallOutputBlocks[0])
}
if _, err := schemas.MarshalSorted(msgs); err != nil {
t.Fatalf("converted messages must marshal, got: %v", err)
}
Expand Down
85 changes: 37 additions & 48 deletions core/providers/anthropic/responses.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5223,30 +5223,7 @@ func convertAnthropicContentBlocksToResponsesMessagesGrouped(contentBlocks []Ant
if block.Content.ContentStr != nil {
bifrostMsg.ResponsesToolMessage.Output.ResponsesToolCallOutputStr = block.Content.ContentStr
} else if block.Content.ContentBlocks != nil {
var toolMsgContentBlocks []schemas.ResponsesMessageContentBlock
for _, contentBlock := range block.Content.ContentBlocks {
switch contentBlock.Type {
case AnthropicContentBlockTypeText:
if contentBlock.Text != nil {
var blockType schemas.ResponsesMessageContentBlockType
if isOutputMessage {
blockType = schemas.ResponsesOutputMessageContentTypeText
} else {
blockType = schemas.ResponsesInputMessageContentBlockTypeText
}
toolMsgContentBlocks = append(toolMsgContentBlocks, schemas.ResponsesMessageContentBlock{
Type: blockType,
Text: contentBlock.Text,
CacheControl: contentBlock.CacheControl,
})
}
case AnthropicContentBlockTypeImage:
if contentBlock.Source != nil && contentBlock.Source.SourceObj != nil {
toolMsgContentBlocks = append(toolMsgContentBlocks, contentBlock.toBifrostResponsesImageBlock())
}
}
}
bifrostMsg.ResponsesToolMessage.Output.ResponsesFunctionToolCallOutputBlocks = toolMsgContentBlocks
bifrostMsg.Output.ResponsesFunctionToolCallOutputBlocks = convertAnthropicToolResultContentBlocks(block.Content.ContentBlocks, isOutputMessage)
}
// Handle is_error from Anthropic
if block.IsError != nil && *block.IsError {
Expand Down Expand Up @@ -5623,30 +5600,7 @@ func convertAnthropicContentBlocksToResponsesMessages(ctx *schemas.BifrostContex
if block.Content.ContentStr != nil {
bifrostMsg.ResponsesToolMessage.Output.ResponsesToolCallOutputStr = block.Content.ContentStr
} else if block.Content.ContentBlocks != nil {
var toolMsgContentBlocks []schemas.ResponsesMessageContentBlock
for _, contentBlock := range block.Content.ContentBlocks {
switch contentBlock.Type {
case AnthropicContentBlockTypeText:
if contentBlock.Text != nil {
var blockType schemas.ResponsesMessageContentBlockType
if isOutputMessage {
blockType = schemas.ResponsesOutputMessageContentTypeText
} else {
blockType = schemas.ResponsesInputMessageContentBlockTypeText
}
toolMsgContentBlocks = append(toolMsgContentBlocks, schemas.ResponsesMessageContentBlock{
Type: blockType,
Text: contentBlock.Text,
CacheControl: contentBlock.CacheControl,
})
}
case AnthropicContentBlockTypeImage:
if contentBlock.Source != nil && contentBlock.Source.SourceObj != nil {
toolMsgContentBlocks = append(toolMsgContentBlocks, contentBlock.toBifrostResponsesImageBlock())
}
}
}
bifrostMsg.ResponsesToolMessage.Output.ResponsesFunctionToolCallOutputBlocks = toolMsgContentBlocks
bifrostMsg.Output.ResponsesFunctionToolCallOutputBlocks = convertAnthropicToolResultContentBlocks(block.Content.ContentBlocks, isOutputMessage)
}
// Handle is_error from Anthropic
if block.IsError != nil && *block.IsError {
Expand Down Expand Up @@ -7859,6 +7813,7 @@ func (block AnthropicContentBlock) toBifrostResponsesDocumentBlock() schemas.Res
if src.MediaType != nil {
mediaType = *src.MediaType
}
resultBlock.FileType = &mediaType
dataURL := *src.Data
if !strings.HasPrefix(dataURL, "data:") {
dataURL = "data:" + mediaType + ";base64," + *src.Data
Expand All @@ -7881,6 +7836,40 @@ func (block AnthropicContentBlock) toBifrostResponsesDocumentBlock() schemas.Res
return resultBlock
}

// convertAnthropicToolResultContentBlocks maps the content types Anthropic permits
// inside tool_result blocks to their canonical Bifrost Responses representations.
// Both grouped and non-grouped request conversion use this boundary so nested
// content support cannot drift between routing modes.
func convertAnthropicToolResultContentBlocks(contentBlocks []AnthropicContentBlock, isOutputMessage bool) []schemas.ResponsesMessageContentBlock {
var toolMessageContentBlocks []schemas.ResponsesMessageContentBlock
for _, contentBlock := range contentBlocks {
switch contentBlock.Type {
case AnthropicContentBlockTypeText:
if contentBlock.Text == nil {
continue
}
blockType := schemas.ResponsesInputMessageContentBlockTypeText
if isOutputMessage {
blockType = schemas.ResponsesOutputMessageContentTypeText
}
toolMessageContentBlocks = append(toolMessageContentBlocks, schemas.ResponsesMessageContentBlock{
Type: blockType,
Text: contentBlock.Text,
CacheControl: contentBlock.CacheControl,
})
case AnthropicContentBlockTypeImage:
if contentBlock.Source != nil && contentBlock.Source.SourceObj != nil {
toolMessageContentBlocks = append(toolMessageContentBlocks, contentBlock.toBifrostResponsesImageBlock())
}
case AnthropicContentBlockTypeDocument:
if contentBlock.Source != nil && contentBlock.Source.SourceObj != nil {
toolMessageContentBlocks = append(toolMessageContentBlocks, contentBlock.toBifrostResponsesDocumentBlock())
}
}
}
return toolMessageContentBlocks
}

// Helper functions for MCP tool/server conversion
// convertAnthropicMCPServerV2ToBifrostTool converts a new-format MCP server to a Bifrost ResponsesTool.
func convertAnthropicMCPServerV2ToBifrostTool(mcpServer *AnthropicMCPServerV2) *schemas.ResponsesTool {
Expand Down
120 changes: 120 additions & 0 deletions core/providers/anthropic/toolresultdocument_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
package anthropic

import (
"testing"

"github.com/maximhq/bifrost/core/schemas"
)

func convertToolResultDocumentBlocksForTest(grouped bool, content []AnthropicContentBlock) []schemas.ResponsesMessage {
role := schemas.ResponsesInputMessageRoleUser
blocks := []AnthropicContentBlock{
{
Type: AnthropicContentBlockTypeToolResult,
ToolUseID: schemas.Ptr("toolu_document"),
Content: &AnthropicContent{ContentBlocks: content},
},
}
if grouped {
return convertAnthropicContentBlocksToResponsesMessagesGrouped(blocks, &role, false)
}
return convertAnthropicContentBlocksToResponsesMessages(nil, blocks, &role, false, "")
}

func requireToolResultDocumentBlocks(t *testing.T, messages []schemas.ResponsesMessage) []schemas.ResponsesMessageContentBlock {
t.Helper()
if len(messages) != 1 {
t.Fatalf("expected 1 converted message, got %d", len(messages))
}
message := messages[0]
if message.Type == nil || *message.Type != schemas.ResponsesMessageTypeFunctionCallOutput {
t.Fatalf("expected function_call_output, got %#v", message.Type)
}
if message.ResponsesToolMessage == nil || message.Output == nil {
t.Fatal("expected a populated tool message output")
}
return message.Output.ResponsesFunctionToolCallOutputBlocks
}

func TestToolResultDocumentURLPreserved(t *testing.T) {
for _, grouped := range []bool{true, false} {
mode := "non-grouped"
if grouped {
mode = "grouped"
}
t.Run(mode, func(t *testing.T) {
content := []AnthropicContentBlock{
{
Type: AnthropicContentBlockTypeText,
Text: schemas.Ptr("Document generated"),
},
{
Type: AnthropicContentBlockTypeDocument,
Title: schemas.Ptr("report.pdf"),
Source: &AnthropicBlockSource{SourceObj: &AnthropicSource{
Type: "url",
URL: schemas.Ptr("https://example.com/report.pdf"),
}},
},
}

blocks := requireToolResultDocumentBlocks(t, convertToolResultDocumentBlocksForTest(grouped, content))
if len(blocks) != 2 {
t.Fatalf("expected text and document blocks, got %d", len(blocks))
}
if blocks[0].Text == nil || *blocks[0].Text != "Document generated" {
t.Fatalf("expected text block first, got %#v", blocks[0])
}
if blocks[1].Type != schemas.ResponsesInputMessageContentBlockTypeFile {
t.Fatalf("expected file block second, got %q", blocks[1].Type)
}
file := blocks[1].ResponsesInputMessageContentBlockFile
if file == nil {
t.Fatal("expected canonical Bifrost file block")
}
if file.FileURL == nil || *file.FileURL != "https://example.com/report.pdf" {
t.Fatalf("expected document URL to be preserved, got %#v", file.FileURL)
}
if file.Filename == nil || *file.Filename != "report.pdf" {
t.Fatalf("expected document title as filename, got %#v", file.Filename)
}
})
}
}

func TestToolResultDocumentBase64Preserved(t *testing.T) {
for _, grouped := range []bool{true, false} {
mode := "non-grouped"
if grouped {
mode = "grouped"
}
t.Run(mode, func(t *testing.T) {
content := []AnthropicContentBlock{
{
Type: AnthropicContentBlockTypeDocument,
Title: schemas.Ptr("inline.pdf"),
Source: &AnthropicBlockSource{SourceObj: &AnthropicSource{
Type: "base64",
MediaType: schemas.Ptr("application/pdf"),
Data: schemas.Ptr("JVBERi0xLjQ="),
}},
},
}

blocks := requireToolResultDocumentBlocks(t, convertToolResultDocumentBlocksForTest(grouped, content))
if len(blocks) != 1 {
t.Fatalf("expected 1 document block, got %d", len(blocks))
}
file := blocks[0].ResponsesInputMessageContentBlockFile
if file == nil || file.FileData == nil {
t.Fatal("expected inline file data")
}
if got := *file.FileData; got != "data:application/pdf;base64,JVBERi0xLjQ=" {
t.Fatalf("expected base64 data and media type to be preserved, got %q", got)
}
if file.FileType == nil || *file.FileType != "application/pdf" {
t.Fatalf("expected application/pdf file type, got %#v", file.FileType)
}
})
}
}
85 changes: 15 additions & 70 deletions core/providers/bedrock/responses.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package bedrock
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"fmt"
"strings"
Expand Down Expand Up @@ -3446,6 +3445,16 @@ func ConvertBifrostMessagesToBedrockMessages(ctx context.Context, bifrostMessage
} else {
resultContent = append(resultContent, BedrockContentBlock{Image: imageSource})
}
} else if block.Type == schemas.ResponsesInputMessageContentBlockTypeFile &&
block.ResponsesInputMessageContentBlockFile != nil {
file := block.ResponsesInputMessageContentBlockFile
document, err := materializeBedrockDocument(ctx, file.FileData, file.FileURL, file.Filename, file.FileType)
if err != nil {
return nil, nil, fmt.Errorf("bedrock: converting tool result document: %w", err)
}
if document != nil {
resultContent = append(resultContent, BedrockContentBlock{Document: document})
}
}
}
}
Expand Down Expand Up @@ -4524,76 +4533,12 @@ func convertBifrostResponsesMessageContentBlocksToBedrockContentBlocks(ctx conte
continue
case schemas.ResponsesInputMessageContentBlockTypeFile:
if block.ResponsesInputMessageContentBlockFile != nil {
doc := &BedrockDocumentSource{
Name: "document", // Default
Format: "pdf", // Default
Source: &BedrockDocumentSourceData{},
}

// Set filename (normalized for Bedrock)
if block.ResponsesInputMessageContentBlockFile.Filename != nil {
doc.Name = normalizeBedrockFilename(*block.ResponsesInputMessageContentBlockFile.Filename)
}

// Determine format: text or PDF based on FileType
isTextFile := false
if block.ResponsesInputMessageContentBlockFile.FileType != nil {
fileType := *block.ResponsesInputMessageContentBlockFile.FileType
// Check if it's a text type
if fileType == "text/markdown" || fileType == "md" {
doc.Format = "md"
isTextFile = true
} else if fileType == "text/html" || fileType == "html" {
doc.Format = "html"
isTextFile = true
} else if fileType == "text/csv" || fileType == "csv" {
doc.Format = "csv"
isTextFile = true
} else if strings.HasPrefix(fileType, "text/") || fileType == "txt" {
doc.Format = "txt"
isTextFile = true
} else if strings.Contains(fileType, "pdf") || fileType == "pdf" {
doc.Format = "pdf"
} else if strings.Contains(fileType, "spreadsheetml") || fileType == "xlsx" {
doc.Format = "xlsx"
} else if fileType == "application/vnd.ms-excel" || fileType == "xls" {
doc.Format = "xls"
} else if strings.Contains(fileType, "wordprocessingml") || fileType == "docx" {
doc.Format = "docx"
} else if fileType == "application/msword" || fileType == "doc" {
doc.Format = "doc"
}
}

// Handle file data
if block.ResponsesInputMessageContentBlockFile.FileData != nil {
fileData := *block.ResponsesInputMessageContentBlockFile.FileData

// Check if it's a data URL (e.g., "data:application/pdf;base64,...")
if strings.HasPrefix(fileData, "data:") {
urlInfo := schemas.ExtractURLTypeInfo(fileData)
if urlInfo.DataURLWithoutPrefix != nil {
// PDF or other binary - keep as base64
doc.Source.Bytes = urlInfo.DataURLWithoutPrefix
bedrockBlock.Document = doc
break
}
}

// Not a data URL - use as-is
if isTextFile {
// bytes is necessary for bedrock
// base64 string of the text
doc.Source.Text = &fileData
encoded := base64.StdEncoding.EncodeToString([]byte(fileData))
doc.Source.Bytes = &encoded
} else {
doc.Source.Bytes = &fileData
}

bedrockBlock.Document = doc

file := block.ResponsesInputMessageContentBlockFile
document, err := materializeBedrockDocument(ctx, file.FileData, file.FileURL, file.Filename, file.FileType)
if err != nil {
return nil, fmt.Errorf("failed to convert document in responses content block: %w", err)
}
bedrockBlock.Document = document
}
default:
// Don't add anything for unknown types
Expand Down
Loading