diff --git a/bifrost.go b/bifrost.go index 4e8e6368cc..36db4579b3 100644 --- a/bifrost.go +++ b/bifrost.go @@ -180,16 +180,16 @@ func (bifrost *Bifrost) processRequests(provider interfaces.Provider, queue chan } if req.Type == TextCompletionRequest { - if req.Input.TextInput == nil { + if req.Input.TextCompletionInput == nil { err = fmt.Errorf("text not provided for text completion request") } else { - result, err = provider.TextCompletion(req.Model, key, *req.Input.TextInput, req.Params) + result, err = provider.TextCompletion(req.Model, key, *req.Input.TextCompletionInput, req.Params) } } else if req.Type == ChatCompletionRequest { - if req.Input.ChatInput == nil { + if req.Input.ChatCompletionInput == nil { err = fmt.Errorf("chats not provided for chat completion request") } else { - result, err = provider.ChatCompletion(req.Model, key, *req.Input.ChatInput, req.Params) + result, err = provider.ChatCompletion(req.Model, key, *req.Input.ChatCompletionInput, req.Params) } } diff --git a/interfaces/bifrost.go b/interfaces/bifrost.go index f1d63395e0..4f4c677715 100644 --- a/interfaces/bifrost.go +++ b/interfaces/bifrost.go @@ -30,8 +30,8 @@ const ( //* Request Structs type RequestInput struct { - TextInput *string - ChatInput *[]Message + TextCompletionInput *string + ChatCompletionInput *[]Message } type BifrostRequest struct { @@ -109,9 +109,10 @@ type Message struct { } type ImageContent struct { - Type string `json:"type"` - URL string `json:"url"` - MediaType string `json:"media_type"` + Type *string `json:"type"` + URL string `json:"url"` + MediaType *string `json:"media_type"` + Detail *string `json:"detail"` } //* Response Structs diff --git a/providers/anthropic.go b/providers/anthropic.go index 5920ade657..272750d985 100644 --- a/providers/anthropic.go +++ b/providers/anthropic.go @@ -200,10 +200,44 @@ func (provider *AnthropicProvider) ChatCompletion(model, key string, messages [] // Format messages for Anthropic API var formattedMessages []map[string]interface{} for _, msg := range messages { - formattedMessages = append(formattedMessages, map[string]interface{}{ - "role": msg.Role, - "content": msg.Content, - }) + if msg.ImageContent != nil { + var content []map[string]interface{} + + imageContent := map[string]interface{}{ + "type": "image", + "source": map[string]interface{}{ + "type": msg.ImageContent.Type, + }, + } + + // Handle different image source types + if *msg.ImageContent.Type == "url" { + imageContent["source"].(map[string]interface{})["url"] = msg.ImageContent.URL + } else { + imageContent["source"].(map[string]interface{})["media_type"] = msg.ImageContent.MediaType + imageContent["source"].(map[string]interface{})["data"] = msg.ImageContent.URL + } + + content = append(content, imageContent) + + // Add text content if present + if msg.Content != nil { + content = append(content, map[string]interface{}{ + "type": "text", + "text": msg.Content, + }) + } + + formattedMessages = append(formattedMessages, map[string]interface{}{ + "role": msg.Role, + "content": content, + }) + } else { + formattedMessages = append(formattedMessages, map[string]interface{}{ + "role": msg.Role, + "content": msg.Content, + }) + } } preparedParams := PrepareParams(params) diff --git a/providers/bedrock.go b/providers/bedrock.go index f49a59f4f7..a245b6e70e 100644 --- a/providers/bedrock.go +++ b/providers/bedrock.go @@ -68,14 +68,17 @@ type BedrockMistralChatMessage struct { } type BedrockAnthropicImageMessage struct { - Type string `json:"type"` + Type string `json:"type"` + Image BedrockAnthropicImage `json:"image"` +} + +type BedrockAnthropicImage struct { + Format string `json:"string"` Source BedrockAnthropicImageSource `json:"source"` } type BedrockAnthropicImageSource struct { - Type string `json:"type"` - MediaType string `json:"media_type"` - Data string `json:"data"` + Bytes string `json:"bytes"` } type BedrockMistralToolCall struct { @@ -245,10 +248,11 @@ func (provider *BedrockProvider) PrepareChatCompletionMessages(messages []interf } else if msg.ImageContent != nil { content = BedrockAnthropicImageMessage{ Type: "image", - Source: BedrockAnthropicImageSource{ - Type: msg.ImageContent.Type, - MediaType: msg.ImageContent.MediaType, - Data: msg.ImageContent.URL, + Image: BedrockAnthropicImage{ + Format: *msg.ImageContent.Type, + Source: BedrockAnthropicImageSource{ + Bytes: msg.ImageContent.URL, + }, }, } } diff --git a/providers/openai.go b/providers/openai.go index 3c12ec768a..949990840c 100644 --- a/providers/openai.go +++ b/providers/openai.go @@ -47,26 +47,49 @@ func (provider *OpenAIProvider) TextCompletion(model, key, text string, params * func (provider *OpenAIProvider) ChatCompletion(model, key string, messages []interfaces.Message, params *interfaces.ModelParameters) (*interfaces.BifrostResponse, error) { // Format messages for OpenAI API - var openAIMessages []map[string]interface{} + var formattedMessages []map[string]interface{} for _, msg := range messages { - var content any - if msg.Content != nil { - content = msg.Content + if msg.ImageContent != nil { + var content []map[string]interface{} + + // Add text content if present + if msg.Content != nil { + content = append(content, map[string]interface{}{ + "type": "text", + "text": msg.Content, + }) + } + + imageContent := map[string]interface{}{ + "type": "image_url", + "image_url": map[string]interface{}{ + "url": msg.ImageContent.URL, + }, + } + + if msg.ImageContent.Detail != nil { + imageContent["image_url"].(map[string]interface{})["detail"] = msg.ImageContent.Detail + } + + content = append(content, imageContent) + + formattedMessages = append(formattedMessages, map[string]interface{}{ + "role": msg.Role, + "content": content, + }) } else { - content = msg.ImageContent + formattedMessages = append(formattedMessages, map[string]interface{}{ + "role": msg.Role, + "content": msg.Content, + }) } - - openAIMessages = append(openAIMessages, map[string]interface{}{ - "role": msg.Role, - "content": content, - }) } preparedParams := PrepareParams(params) requestBody := MergeConfig(map[string]interface{}{ "model": model, - "messages": openAIMessages, + "messages": formattedMessages, }, preparedParams) jsonBody, err := json.Marshal(requestBody) diff --git a/tests/account.go b/tests/account.go index 174fee9b70..a50a6b69dd 100644 --- a/tests/account.go +++ b/tests/account.go @@ -24,7 +24,7 @@ func (baseAccount *BaseAccount) GetKeysForProvider(providerKey interfaces.Suppor return []interfaces.Key{ { Value: os.Getenv("OPEN_AI_API_KEY"), - Models: []string{"gpt-4o-mini"}, + Models: []string{"gpt-4o-mini", "gpt-4-turbo"}, Weight: 1.0, }, }, nil diff --git a/tests/anthropic_test.go b/tests/anthropic_test.go index 0fa2eb793e..9def641eb9 100644 --- a/tests/anthropic_test.go +++ b/tests/anthropic_test.go @@ -8,6 +8,8 @@ import ( "github.com/maximhq/bifrost" "github.com/maximhq/bifrost/interfaces" + + "github.com/maximhq/maxim-go" ) // setupAnthropicRequests sends multiple test requests to Anthropic @@ -27,7 +29,7 @@ func setupAnthropicRequests(bifrost *bifrost.Bifrost) { result, err := bifrost.TextCompletionRequest(interfaces.Anthropic, &interfaces.BifrostRequest{ Model: "claude-2.1", Input: interfaces.RequestInput{ - TextInput: &text, + TextCompletionInput: &text, }, Params: ¶ms, }, ctx) @@ -58,7 +60,7 @@ func setupAnthropicRequests(bifrost *bifrost.Bifrost) { result, err := bifrost.ChatCompletionRequest(interfaces.Anthropic, &interfaces.BifrostRequest{ Model: "claude-3-7-sonnet-20250219", Input: interfaces.RequestInput{ - ChatInput: &messages, + ChatCompletionInput: &messages, }, Params: ¶ms, }, ctx) @@ -71,10 +73,77 @@ func setupAnthropicRequests(bifrost *bifrost.Bifrost) { }(message, delay, i) } + // Image input tests + setupAnthropicImageTests(bifrost, ctx) + // Tool calls test setupAnthropicToolCalls(bifrost, ctx) } +// setupAnthropicImageTests tests Anthropic's image input capabilities +func setupAnthropicImageTests(bifrost *bifrost.Bifrost, ctx context.Context) { + // Test with URL image + urlImageMessages := []interfaces.Message{ + { + Role: interfaces.RoleUser, + Content: maxim.StrPtr("What is Happening in this picture?"), + ImageContent: &interfaces.ImageContent{ + Type: maxim.StrPtr("url"), + URL: "https://upload.wikimedia.org/wikipedia/commons/a/a7/Camponotus_flavomarginatus_ant.jpg", + }, + }, + } + + maxTokens := 4096 + + params := interfaces.ModelParameters{ + MaxTokens: &maxTokens, + } + + go func() { + result, err := bifrost.ChatCompletionRequest(interfaces.Anthropic, &interfaces.BifrostRequest{ + Model: "claude-3-7-sonnet-20250219", + Input: interfaces.RequestInput{ + ChatCompletionInput: &urlImageMessages, + }, + Params: ¶ms, + }, ctx) + if err != nil { + fmt.Printf("Error in Anthropic URL image request: %v\n", err) + } else { + fmt.Printf("🐒 URL Image Result: %s\n", result.Choices[0].Message.Content) + } + }() + + // Test with base64 image + base64ImageMessages := []interfaces.Message{ + { + Role: interfaces.RoleUser, + Content: maxim.StrPtr("What is this image about?"), + ImageContent: &interfaces.ImageContent{ + Type: maxim.StrPtr("base64"), + URL: "/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAAIAAoDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAb/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/8QAFQEBAQAAAAAAAAAAAAAAAAAAAAX/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIRAxEAPwCdABmX/9k=", + MediaType: maxim.StrPtr("image/jpeg"), + }, + }, + } + + go func() { + result, err := bifrost.ChatCompletionRequest(interfaces.Anthropic, &interfaces.BifrostRequest{ + Model: "claude-3-7-sonnet-20250219", + Input: interfaces.RequestInput{ + ChatCompletionInput: &base64ImageMessages, + }, + Params: ¶ms, + }, ctx) + if err != nil { + fmt.Printf("Error in Anthropic base64 image request: %v\n", err) + } else { + fmt.Printf("🐒 Base64 Image Result: %s\n", result.Choices[0].Message.Content) + } + }() +} + // setupAnthropicToolCalls tests Anthropic's function calling capability func setupAnthropicToolCalls(bifrost *bifrost.Bifrost, ctx context.Context) { anthropicMessages := []string{ @@ -121,7 +190,7 @@ func setupAnthropicToolCalls(bifrost *bifrost.Bifrost, ctx context.Context) { result, err := bifrost.ChatCompletionRequest(interfaces.Anthropic, &interfaces.BifrostRequest{ Model: "claude-3-7-sonnet-20250219", Input: interfaces.RequestInput{ - ChatInput: &messages, + ChatCompletionInput: &messages, }, Params: ¶ms, }, ctx) diff --git a/tests/bedrock_test.go b/tests/bedrock_test.go index 9c1791931a..1777d86125 100644 --- a/tests/bedrock_test.go +++ b/tests/bedrock_test.go @@ -8,6 +8,8 @@ import ( "github.com/maximhq/bifrost" "github.com/maximhq/bifrost/interfaces" + + "github.com/maximhq/maxim-go" ) // setupBedrockRequests sends multiple test requests to Bedrock @@ -27,7 +29,7 @@ func setupBedrockRequests(bifrost *bifrost.Bifrost) { result, err := bifrost.TextCompletionRequest(interfaces.Bedrock, &interfaces.BifrostRequest{ Model: "anthropic.claude-v2:1", Input: interfaces.RequestInput{ - TextInput: &text, + TextCompletionInput: &text, }, Params: ¶ms, }, ctx) @@ -58,7 +60,7 @@ func setupBedrockRequests(bifrost *bifrost.Bifrost) { result, err := bifrost.ChatCompletionRequest(interfaces.Bedrock, &interfaces.BifrostRequest{ Model: "anthropic.claude-3-sonnet-20240229-v1:0", Input: interfaces.RequestInput{ - ChatInput: &messages, + ChatCompletionInput: &messages, }, Params: ¶ms, }, ctx) @@ -71,10 +73,50 @@ func setupBedrockRequests(bifrost *bifrost.Bifrost) { }(message, delay, i) } + // Image input tests + setupBedrockImageTests(bifrost, ctx) + // Tool calls test setupBedrockToolCalls(bifrost, ctx) } +// setupAnthropicImageTests tests Bedrock's image input capabilities +func setupBedrockImageTests(bifrost *bifrost.Bifrost, ctx context.Context) { + maxTokens := 4096 + + params := interfaces.ModelParameters{ + MaxTokens: &maxTokens, + } + + // Test with base64 image + base64ImageMessages := []interfaces.Message{ + { + Role: interfaces.RoleUser, + Content: maxim.StrPtr("What is this image about?"), + ImageContent: &interfaces.ImageContent{ + Type: maxim.StrPtr("base64"), + URL: "/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAAIAAoDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAb/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/8QAFQEBAQAAAAAAAAAAAAAAAAAAAAX/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIRAxEAPwCdABmX/9k=", + MediaType: maxim.StrPtr("image/jpeg"), + }, + }, + } + + go func() { + result, err := bifrost.ChatCompletionRequest(interfaces.Bedrock, &interfaces.BifrostRequest{ + Model: "anthropic.claude-3-sonnet-20240229-v1:0", + Input: interfaces.RequestInput{ + ChatCompletionInput: &base64ImageMessages, + }, + Params: ¶ms, + }, ctx) + if err != nil { + fmt.Printf("Error in Bedrock base64 image request: %v\n", err) + } else { + fmt.Printf("🐒 Base64 Image Result: %s\n", result.Choices[0].Message.Content) + } + }() +} + // setupBedrockToolCalls tests Bedrock's function calling capability func setupBedrockToolCalls(bifrost *bifrost.Bifrost, ctx context.Context) { bedrockMessages := []string{ @@ -121,7 +163,7 @@ func setupBedrockToolCalls(bifrost *bifrost.Bifrost, ctx context.Context) { result, err := bifrost.ChatCompletionRequest(interfaces.Bedrock, &interfaces.BifrostRequest{ Model: "anthropic.claude-3-sonnet-20240229-v1:0", Input: interfaces.RequestInput{ - ChatInput: &messages, + ChatCompletionInput: &messages, }, Params: ¶ms, }, ctx) diff --git a/tests/cohere_test.go b/tests/cohere_test.go index 750d59399d..d9f2bcdecc 100644 --- a/tests/cohere_test.go +++ b/tests/cohere_test.go @@ -20,7 +20,7 @@ func setupCohereRequests(bifrost *bifrost.Bifrost) { result, err := bifrost.TextCompletionRequest(interfaces.Cohere, &interfaces.BifrostRequest{ Model: "command-a-03-2025", Input: interfaces.RequestInput{ - TextInput: &text, + TextCompletionInput: &text, }, Params: nil, }, ctx) @@ -51,7 +51,7 @@ func setupCohereRequests(bifrost *bifrost.Bifrost) { result, err := bifrost.ChatCompletionRequest(interfaces.Cohere, &interfaces.BifrostRequest{ Model: "command-a-03-2025", Input: interfaces.RequestInput{ - ChatInput: &messages, + ChatCompletionInput: &messages, }, Params: nil, }, ctx) @@ -110,7 +110,7 @@ func setupCohereToolCalls(bifrost *bifrost.Bifrost, ctx context.Context) { result, err := bifrost.ChatCompletionRequest(interfaces.Cohere, &interfaces.BifrostRequest{ Model: "command-a-03-2025", Input: interfaces.RequestInput{ - ChatInput: &messages, + ChatCompletionInput: &messages, }, Params: ¶ms, }, ctx) diff --git a/tests/openai_test.go b/tests/openai_test.go index dd73c6855e..91127452f4 100644 --- a/tests/openai_test.go +++ b/tests/openai_test.go @@ -8,6 +8,8 @@ import ( "github.com/maximhq/bifrost" "github.com/maximhq/bifrost/interfaces" + + "github.com/maximhq/maxim-go" ) // setupOpenAIRequests sends multiple test requests to OpenAI @@ -20,7 +22,7 @@ func setupOpenAIRequests(bifrost *bifrost.Bifrost) { result, err := bifrost.TextCompletionRequest(interfaces.OpenAI, &interfaces.BifrostRequest{ Model: "gpt-4o-mini", Input: interfaces.RequestInput{ - TextInput: &text, + TextCompletionInput: &text, }, Params: nil, }, ctx) @@ -51,7 +53,7 @@ func setupOpenAIRequests(bifrost *bifrost.Bifrost) { result, err := bifrost.ChatCompletionRequest(interfaces.OpenAI, &interfaces.BifrostRequest{ Model: "gpt-4o-mini", Input: interfaces.RequestInput{ - ChatInput: &messages, + ChatCompletionInput: &messages, }, Params: nil, }, ctx) @@ -63,10 +65,68 @@ func setupOpenAIRequests(bifrost *bifrost.Bifrost) { }(message, delay, i) } + // Image input tests + setupOpenAIImageTests(bifrost, ctx) + // Tool calls test setupOpenAIToolCalls(bifrost, ctx) } +// setupOpenAIImageTests tests OpenAI's image input capabilities +func setupOpenAIImageTests(bifrost *bifrost.Bifrost, ctx context.Context) { + // Test with URL image + urlImageMessages := []interfaces.Message{ + { + Role: interfaces.RoleUser, + Content: maxim.StrPtr("What is Happening in this picture?"), + ImageContent: &interfaces.ImageContent{ + URL: "https://upload.wikimedia.org/wikipedia/commons/a/a7/Camponotus_flavomarginatus_ant.jpg", + }, + }, + } + + go func() { + result, err := bifrost.ChatCompletionRequest(interfaces.OpenAI, &interfaces.BifrostRequest{ + Model: "gpt-4-turbo", + Input: interfaces.RequestInput{ + ChatCompletionInput: &urlImageMessages, + }, + Params: nil, + }, ctx) + if err != nil { + fmt.Printf("Error in OpenAI URL image request: %v\n", err) + } else { + fmt.Printf("🐒 URL Image Result: %s\n", result.Choices[0].Message.Content) + } + }() + + // Test with base64 image + base64ImageMessages := []interfaces.Message{ + { + Role: interfaces.RoleUser, + Content: maxim.StrPtr("What is this image about?"), + ImageContent: &interfaces.ImageContent{ + URL: "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAAIAAoDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAb/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/8QAFQEBAQAAAAAAAAAAAAAAAAAAAAX/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIRAxEAPwCdABmX/9k=", + }, + }, + } + + go func() { + result, err := bifrost.ChatCompletionRequest(interfaces.OpenAI, &interfaces.BifrostRequest{ + Model: "gpt-4-turbo", + Input: interfaces.RequestInput{ + ChatCompletionInput: &base64ImageMessages, + }, + Params: nil, + }, ctx) + if err != nil { + fmt.Printf("Error in OpenAI base64 image request: %v\n", err) + } else { + fmt.Printf("🐒 Base64 Image Result: %s\n", result.Choices[0].Message.Content) + } + }() +} + // setupOpenAIToolCalls tests OpenAI's function calling capability func setupOpenAIToolCalls(bifrost *bifrost.Bifrost, ctx context.Context) { openAIMessages := []string{ @@ -110,7 +170,7 @@ func setupOpenAIToolCalls(bifrost *bifrost.Bifrost, ctx context.Context) { result, err := bifrost.ChatCompletionRequest(interfaces.OpenAI, &interfaces.BifrostRequest{ Model: "gpt-4o-mini", Input: interfaces.RequestInput{ - ChatInput: &messages, + ChatCompletionInput: &messages, }, Params: ¶ms, }, ctx)