diff --git a/service/relayconvert/internal/oai_chat/to_claude_messages_req.go b/service/relayconvert/internal/oai_chat/to_claude_messages_req.go index 346eb906e48a..e73fa7febd9f 100644 --- a/service/relayconvert/internal/oai_chat/to_claude_messages_req.go +++ b/service/relayconvert/internal/oai_chat/to_claude_messages_req.go @@ -27,6 +27,56 @@ type openRouterRequestReasoning struct { Exclude bool `json:"exclude,omitempty"` } +func convertOpenAIToolResultContentToClaude(c *gin.Context, message dto.Message) (any, error) { + contentItems, ok := message.Content.([]any) + if !ok { + return message.Content, nil + } + + convertedContent := make([]any, 0, len(contentItems)) + convertedAnyImage := false + for _, contentItem := range contentItems { + contentMap, ok := contentItem.(map[string]any) + if !ok || contentMap["type"] != dto.ContentTypeImageURL { + convertedContent = append(convertedContent, contentItem) + continue + } + + mediaMessages := (&dto.Message{Content: []any{contentItem}}).ParseContent() + if len(mediaMessages) != 1 { + return nil, fmt.Errorf("invalid image_url tool result content") + } + source := mediaMessages[0].ToFileSource() + if source == nil { + return nil, fmt.Errorf("invalid image_url tool result source") + } + + base64Data, mimeType, err := relaymedia.ResolveBase64Data(c, source, "formatting tool result image for Claude") + if err != nil { + return nil, fmt.Errorf("get tool result file data failed: %s", err.Error()) + } + + contentType := "image" + if strings.HasPrefix(mimeType, "application/pdf") { + contentType = "document" + } + convertedContent = append(convertedContent, dto.ClaudeMediaMessage{ + Type: contentType, + Source: &dto.ClaudeMessageSource{ + Type: "base64", + MediaType: mimeType, + Data: base64Data, + }, + }) + convertedAnyImage = true + } + + if !convertedAnyImage { + return message.Content, nil + } + return convertedContent, nil +} + func OpenAIChatRequestToClaudeMessages(c *gin.Context, textRequest dto.GeneralOpenAIRequest) (*dto.ClaudeRequest, error) { claudeTools := make([]any, 0, len(textRequest.Tools)) @@ -299,6 +349,10 @@ func OpenAIChatRequestToClaudeMessages(c *gin.Context, textRequest dto.GeneralOp Role: message.Role, } if message.Role == "tool" { + toolResultContent, err := convertOpenAIToolResultContentToClaude(c, message) + if err != nil { + return nil, err + } if len(claudeMessages) > 0 && claudeMessages[len(claudeMessages)-1].Role == "user" { lastClaudeMessage := claudeMessages[len(claudeMessages)-1] if content, ok := lastClaudeMessage.Content.(string); ok { @@ -312,7 +366,7 @@ func OpenAIChatRequestToClaudeMessages(c *gin.Context, textRequest dto.GeneralOp lastClaudeMessage.Content = append(lastClaudeMessage.Content.([]dto.ClaudeMediaMessage), dto.ClaudeMediaMessage{ Type: "tool_result", ToolUseId: message.ToolCallId, - Content: message.Content, + Content: toolResultContent, }) claudeMessages[len(claudeMessages)-1] = lastClaudeMessage continue @@ -323,7 +377,7 @@ func OpenAIChatRequestToClaudeMessages(c *gin.Context, textRequest dto.GeneralOp { Type: "tool_result", ToolUseId: message.ToolCallId, - Content: message.Content, + Content: toolResultContent, }, } } else if message.IsStringContent() && message.ToolCalls == nil { diff --git a/service/relayconvert/internal/oai_chat/to_claude_messages_req_test.go b/service/relayconvert/internal/oai_chat/to_claude_messages_req_test.go new file mode 100644 index 000000000000..a814093798b0 --- /dev/null +++ b/service/relayconvert/internal/oai_chat/to_claude_messages_req_test.go @@ -0,0 +1,106 @@ +package oaichat + +import ( + "encoding/json" + "testing" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/dto" + relaymedia "github.com/QuantumNous/new-api/service/relayconvert/internal/media" + "github.com/QuantumNous/new-api/types" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestOpenAIChatRequestToClaudeMessagesConvertsToolResultImageURL(t *testing.T) { + relaymedia.SetMediaResolver(relaymedia.MediaResolver{ + GetBase64Data: func(_ *gin.Context, source types.FileSource, _ ...string) (string, string, error) { + assert.Equal(t, "https://example.com/tool-result.png", source.GetRawData()) + return "resolved-image-data", "image/png", nil + }, + }) + t.Cleanup(func() { + relaymedia.SetMediaResolver(relaymedia.MediaResolver{}) + }) + + request := toolResultConversionRequest([]any{ + map[string]any{"type": "text", "text": "Image result:"}, + map[string]any{ + "type": dto.ContentTypeImageURL, + "image_url": map[string]any{ + "url": "https://example.com/tool-result.png", + }, + }, + }) + + claudeRequest, err := OpenAIChatRequestToClaudeMessages(nil, request) + require.NoError(t, err) + require.Len(t, claudeRequest.Messages, 3) + + toolResultBlocks, ok := claudeRequest.Messages[2].Content.([]dto.ClaudeMediaMessage) + require.True(t, ok) + require.Len(t, toolResultBlocks, 1) + assert.Equal(t, "tool_result", toolResultBlocks[0].Type) + assert.Equal(t, "call_1", toolResultBlocks[0].ToolUseId) + + toolContent, ok := toolResultBlocks[0].Content.([]any) + require.True(t, ok) + require.Len(t, toolContent, 2) + assert.Equal(t, map[string]any{"type": "text", "text": "Image result:"}, toolContent[0]) + + imageBlock, ok := toolContent[1].(dto.ClaudeMediaMessage) + require.True(t, ok) + assert.Equal(t, "image", imageBlock.Type) + require.NotNil(t, imageBlock.Source) + assert.Equal(t, "base64", imageBlock.Source.Type) + assert.Equal(t, "image/png", imageBlock.Source.MediaType) + assert.Equal(t, "resolved-image-data", imageBlock.Source.Data) + + payload, err := common.Marshal(claudeRequest) + require.NoError(t, err) + assert.NotContains(t, string(payload), `"image_url"`) + assert.Contains(t, string(payload), `"type":"image"`) +} + +func TestOpenAIChatRequestToClaudeMessagesPreservesNonImageURLToolContent(t *testing.T) { + original := []any{ + map[string]any{"type": "text", "text": "plain result"}, + map[string]any{ + "type": "image", + "source": map[string]any{ + "type": "base64", + "media_type": "image/png", + "data": "already-converted", + }, + }, + map[string]any{"type": "custom", "value": "preserve me"}, + } + + claudeRequest, err := OpenAIChatRequestToClaudeMessages(nil, toolResultConversionRequest(original)) + require.NoError(t, err) + require.Len(t, claudeRequest.Messages, 3) + + toolResultBlocks, ok := claudeRequest.Messages[2].Content.([]dto.ClaudeMediaMessage) + require.True(t, ok) + require.Len(t, toolResultBlocks, 1) + assert.Equal(t, original, toolResultBlocks[0].Content) +} + +func toolResultConversionRequest(toolContent any) dto.GeneralOpenAIRequest { + return dto.GeneralOpenAIRequest{ + Model: "test-model", + Messages: []dto.Message{ + {Role: "user", Content: "Use read_image."}, + { + Role: "assistant", + ToolCalls: json.RawMessage(`[{"id":"call_1","type":"function","function":{"name":"read_image","arguments":"{}"}}]`), + }, + { + Role: "tool", + ToolCallId: "call_1", + Content: toolContent, + }, + }, + } +}