Skip to content

feat: refactor message content structure to support multi-modal content blocks #74

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
akshaydeo merged 1 commit into from
Jun 11, 2025
Merged

feat: refactor message content structure to support multi-modal content blocks #74

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,10 +149,12 @@ For additional configurations in HTTP server setup, please read [this](https://g
Provider: schemas.OpenAI,
Model: "gpt-4o-mini", // make sure you have configured gpt-4o-mini in your account interface
Input: schemas.RequestInput{
ChatCompletionInput: bifrost.Ptr([]schemas.Message{{
Role: schemas.RoleUser,
Content: bifrost.Ptr("What is a LLM gateway?"),
}}),
ChatCompletionInput: bifrost.Ptr([]schemas.BifrostMessage{{
Role: schemas.ModelChatMessageRoleUser,
Content: schemas.MessageContent{
ContentStr: bifrost.Ptr("What is a LLM gateway?"),
},
}}),
Comment thread
Pratham-Mishra04 marked this conversation as resolved.
},
},
)
Expand Down
135 changes: 76 additions & 59 deletions core/providers/anthropic.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,12 @@ type AnthropicError struct {
} `json:"error"` // Error details
}

type AnthropicImageContent struct {
Type ImageContentType `json:"type"`
URL string `json:"url"`
MediaType string `json:"media_type,omitempty"`
}

// AnthropicProvider implements the Provider interface for Anthropic's Claude API.
type AnthropicProvider struct {
logger schemas.Logger // Logger for provider operations
Expand Down Expand Up @@ -255,8 +261,10 @@ func (provider *AnthropicProvider) TextCompletion(ctx context.Context, model, ke
{
Index: 0,
Message: schemas.BifrostMessage{
Role: schemas.ModelChatMessageRoleAssistant,
Content: &response.Completion,
Role: schemas.ModelChatMessageRoleAssistant,
Content: schemas.MessageContent{
ContentStr: &response.Completion,
},
},
},
}
Expand Down Expand Up @@ -318,22 +326,34 @@ func (provider *AnthropicProvider) ChatCompletion(ctx context.Context, model, ke
}

// buildAnthropicImageSourceMap creates the "source" map for an Anthropic image content part.
func buildAnthropicImageSourceMap(imgContent *schemas.ImageContent) map[string]interface{} {
func buildAnthropicImageSourceMap(imgContent *schemas.ImageURLStruct) map[string]interface{} {
if imgContent == nil {
return nil
}

formattedImgContent := *FormatImageContent(imgContent, false)
sanitizedURL, _ := SanitizeImageURL(imgContent.URL)
urlTypeInfo := ExtractURLTypeInfo(sanitizedURL)

formattedImgContent := AnthropicImageContent{
Type: urlTypeInfo.Type,
MediaType: *urlTypeInfo.MediaType,
}

if urlTypeInfo.DataURLWithoutPrefix != nil {
formattedImgContent.URL = *urlTypeInfo.DataURLWithoutPrefix
} else {
formattedImgContent.URL = sanitizedURL
}

sourceMap := map[string]interface{}{
"type": string(formattedImgContent.Type), // "base64" or "url"
}

if formattedImgContent.Type == schemas.ImageContentTypeURL {
if formattedImgContent.Type == ImageContentTypeURL {
sourceMap["url"] = formattedImgContent.URL
} else {
if formattedImgContent.MediaType != nil {
sourceMap["media_type"] = *formattedImgContent.MediaType
if formattedImgContent.MediaType != "" {
sourceMap["media_type"] = formattedImgContent.MediaType
}
sourceMap["data"] = formattedImgContent.URL // URL field contains base64 data string
}
Expand All @@ -345,10 +365,18 @@ func prepareAnthropicChatRequest(messages []schemas.BifrostMessage, params *sche
var systemMessages []BedrockAnthropicSystemMessage
for _, msg := range messages {
if msg.Role == schemas.ModelChatMessageRoleSystem {
if msg.Content != nil {
if msg.Content.ContentStr != nil {
systemMessages = append(systemMessages, BedrockAnthropicSystemMessage{
Text: *msg.Content,
Text: *msg.Content.ContentStr,
})
} else if msg.Content.ContentBlocks != nil {
for _, block := range *msg.Content.ContentBlocks {
if block.Text != nil {
systemMessages = append(systemMessages, BedrockAnthropicSystemMessage{
Text: *block.Text,
})
}
}
}
}
}
Expand All @@ -367,61 +395,49 @@ func prepareAnthropicChatRequest(messages []schemas.BifrostMessage, params *sche

var toolCallResultContent []map[string]interface{}

if msg.Content != nil {
if msg.Content.ContentStr != nil {
toolCallResultContent = append(toolCallResultContent, map[string]interface{}{
"type": "text",
"text": *msg.Content,
"text": *msg.Content.ContentStr,
})
}

if (msg.UserMessage != nil && msg.UserMessage.ImageContent != nil) || (msg.ToolMessage != nil && msg.ToolMessage.ImageContent != nil) {
var messageImageContent schemas.ImageContent
if msg.UserMessage != nil && msg.UserMessage.ImageContent != nil {
// Create a copy to avoid modifying the original
messageImageContent = *msg.UserMessage.ImageContent
} else if msg.ToolMessage != nil && msg.ToolMessage.ImageContent != nil {
// Create a copy to avoid modifying the original
messageImageContent = *msg.ToolMessage.ImageContent
}

imageSource := buildAnthropicImageSourceMap(&messageImageContent)
if imageSource != nil {
toolCallResultContent = append(toolCallResultContent, map[string]interface{}{
"type": "image",
"source": imageSource,
})
} else if msg.Content.ContentBlocks != nil {
for _, block := range *msg.Content.ContentBlocks {
if block.Text != nil {
toolCallResultContent = append(toolCallResultContent, map[string]interface{}{
"type": "text",
"text": *block.Text,
})
}
}
}

toolCallResult["content"] = toolCallResultContent

content = append(content, toolCallResult)
} else {
if (msg.UserMessage != nil && msg.UserMessage.ImageContent != nil) || (msg.ToolMessage != nil && msg.ToolMessage.ImageContent != nil) {
var messageImageContent schemas.ImageContent
if msg.UserMessage != nil && msg.UserMessage.ImageContent != nil {
// Create a copy to avoid modifying the original
messageImageContent = *msg.UserMessage.ImageContent
} else if msg.ToolMessage != nil && msg.ToolMessage.ImageContent != nil {
// Create a copy to avoid modifying the original
messageImageContent = *msg.ToolMessage.ImageContent
}

imageSource := buildAnthropicImageSourceMap(&messageImageContent)
if imageSource != nil {
content = append(content, map[string]interface{}{
"type": "image",
"source": imageSource,
})
}
}

// Add text content if present
if msg.Content != nil && *msg.Content != "" {
if msg.Content.ContentStr != nil && *msg.Content.ContentStr != "" {
content = append(content, map[string]interface{}{
"type": "text",
"text": *msg.Content,
"text": *msg.Content.ContentStr,
})
} else if msg.Content.ContentBlocks != nil {
for _, block := range *msg.Content.ContentBlocks {
if block.Text != nil && *block.Text != "" {
content = append(content, map[string]interface{}{
"type": "text",
"text": *block.Text,
})
}
if block.ImageURL != nil {
imageSource := buildAnthropicImageSourceMap(block.ImageURL)
if imageSource != nil {
content = append(content, map[string]interface{}{
"type": "image",
"source": imageSource,
})
}
}
}
}

// Add thinking content if present in AssistantMessage
Expand Down Expand Up @@ -577,20 +593,20 @@ func prepareAnthropicChatRequest(messages []schemas.BifrostMessage, params *sche

func parseAnthropicResponse(response *AnthropicChatResponse, bifrostResponse *schemas.BifrostResponse) (*schemas.BifrostResponse, *schemas.BifrostError) {
// Collect all content and tool calls into a single message
var content strings.Builder
var toolCalls []schemas.ToolCall
var thinking string

var contentBlocks []schemas.ContentBlock
// Process content and tool calls
for _, c := range response.Content {
switch c.Type {
case "thinking":
thinking = c.Thinking
case "text":
if content.Len() > 0 {
content.WriteString("\n")
}
content.WriteString(c.Text)
contentBlocks = append(contentBlocks, schemas.ContentBlock{
Type: "text",
Text: &c.Text,
})
case "tool_use":
function := schemas.FunctionCall{
Name: &c.Name,
Expand All @@ -612,7 +628,6 @@ func parseAnthropicResponse(response *AnthropicChatResponse, bifrostResponse *sc
}

// Create the assistant message
messageContent := content.String()
var assistantMessage *schemas.AssistantMessage

// Create AssistantMessage if we have tool calls or thinking
Expand All @@ -632,8 +647,10 @@ func parseAnthropicResponse(response *AnthropicChatResponse, bifrostResponse *sc
{
Index: 0,
Message: schemas.BifrostMessage{
Role: schemas.ModelChatMessageRoleAssistant,
Content: &messageContent,
Role: schemas.ModelChatMessageRoleAssistant,
Content: schemas.MessageContent{
ContentBlocks: &contentBlocks,
},
AssistantMessage: assistantMessage,
},
FinishReason: &response.StopReason,
Expand Down
6 changes: 4 additions & 2 deletions core/providers/azure.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,8 +269,10 @@ func (provider *AzureProvider) TextCompletion(ctx context.Context, model, key, t
choices = append(choices, schemas.BifrostResponseChoice{
Index: 0,
Message: schemas.BifrostMessage{
Role: schemas.ModelChatMessageRoleAssistant,
Content: &textCopy,
Role: schemas.ModelChatMessageRoleAssistant,
Content: schemas.MessageContent{
ContentStr: &textCopy,
},
},
Comment thread
Pratham-Mishra04 marked this conversation as resolved.
FinishReason: response.Choices[0].FinishReason,
LogProbs: &schemas.LogProbs{
Expand Down
Loading