diff --git a/examples/basic/anthropic-openai-local.yaml b/examples/basic/anthropic-openai-local.yaml new file mode 100644 index 0000000000..ab7ebdf227 --- /dev/null +++ b/examples/basic/anthropic-openai-local.yaml @@ -0,0 +1,52 @@ +# Copyright Envoy AI Gateway Authors +# SPDX-License-Identifier: Apache-2.0 +# The full text of the Apache license is available in the LICENSE file at +# the root of the repo. + +# This example routes /v1/messages (Anthropic Messages API) requests to a +# local vLLM backend. +# The AIServiceBackend schema is set to OpenAI so the gateway passes through +# the Anthropic request format with translation. + +apiVersion: aigateway.envoyproxy.io/v1alpha1 +kind: AIGatewayRoute +metadata: + name: envoy-ai-gateway-basic-anthropic-openai + namespace: default +spec: + parentRefs: + - name: envoy-ai-gateway-basic + kind: Gateway + group: gateway.networking.k8s.io + rules: + - matches: + - headers: + - type: Exact + name: x-ai-eg-model + value: Qwen/Qwen2.5-0.5B-Instruct # Replace with the model name served by your vLLM instance. + backendRefs: + - name: envoy-ai-gateway-basic-anthropic-openai +--- +apiVersion: aigateway.envoyproxy.io/v1alpha1 +kind: AIServiceBackend +metadata: + name: envoy-ai-gateway-basic-anthropic-openai + namespace: default +spec: + schema: + name: OpenAI # vLLM exposes an OpenAI-compatible API, so Anthropic Messages Requests are translated + backendRef: + name: envoy-ai-gateway-basic-openai + kind: Backend + group: gateway.envoyproxy.io +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: Backend +metadata: + name: envoy-ai-gateway-basic-anthropic-openai + namespace: default +spec: + endpoints: + - ip: + address: 0.0.0.0 # Replace with your vLLM service hostname or IP (e.g. localhost's internal IP from kind cluster). + port: 8000 # Replace with the port your vLLM instance listens on (default: 8000). diff --git a/internal/endpointspec/endpointspec.go b/internal/endpointspec/endpointspec.go index 41bb524ac8..46040bcde5 100644 --- a/internal/endpointspec/endpointspec.go +++ b/internal/endpointspec/endpointspec.go @@ -343,8 +343,11 @@ func (MessagesEndpointSpec) GetTranslator(schema filterapi.VersionedAPISchema, m return translator.NewAnthropicToAWSAnthropicTranslator(schema.Version, modelNameOverride), nil case filterapi.APISchemaAnthropic: return translator.NewAnthropicToAnthropicTranslator(schema.Version, modelNameOverride), nil + case filterapi.APISchemaOpenAI: + // The Anthropic prefix can be altered using values.yaml if necessary + return translator.NewAnthropicToChatCompletionOpenAITranslator(schema.Version, modelNameOverride), nil default: - return nil, fmt.Errorf("/v1/messages endpoint only supports backends that return native Anthropic format (Anthropic, GCPAnthropic, AWSAnthropic). Backend %s uses different model format", schema.Name) + return nil, fmt.Errorf("/v1/messages endpoint only supports backends that return native Anthropic format (Anthropic, GCPAnthropic, AWSAnthropic). OpenAI translation is also supported. Backend %s uses different model format", schema.Name) } } diff --git a/internal/endpointspec/endpointspec_test.go b/internal/endpointspec/endpointspec_test.go index b6eff2f729..bb10f78321 100644 --- a/internal/endpointspec/endpointspec_test.go +++ b/internal/endpointspec/endpointspec_test.go @@ -236,13 +236,14 @@ func TestMessagesEndpointSpec_GetTranslator(t *testing.T) { {Name: filterapi.APISchemaGCPAnthropic}, {Name: filterapi.APISchemaAWSAnthropic}, {Name: filterapi.APISchemaAnthropic}, + {Name: filterapi.APISchemaOpenAI}, // This is for OpenAI-schema backends like vLLM that support the /v1/messages endpoint } { translator, err := spec.GetTranslator(schema, "override") require.NoError(t, err) require.NotNil(t, translator) } - _, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, "override") + _, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaCohere}, "override") require.ErrorContains(t, err, "only supports") } diff --git a/internal/translator/anthropic_openai.go b/internal/translator/anthropic_openai.go new file mode 100644 index 0000000000..f98a55a781 --- /dev/null +++ b/internal/translator/anthropic_openai.go @@ -0,0 +1,275 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package translator + +import ( + "cmp" + "fmt" + "io" + "log/slog" + "strconv" + "strings" + + "github.com/tidwall/sjson" + + "github.com/envoyproxy/ai-gateway/internal/apischema/anthropic" + "github.com/envoyproxy/ai-gateway/internal/apischema/openai" + "github.com/envoyproxy/ai-gateway/internal/internalapi" + "github.com/envoyproxy/ai-gateway/internal/json" + "github.com/envoyproxy/ai-gateway/internal/metrics" + "github.com/envoyproxy/ai-gateway/internal/tracing/tracingapi" +) + +// NewAnthropicToChatCompletionOpenAITranslator implements [Factory] for Anthropic to OpenAI ChatCompletion translation. +// This translator converts Anthropic API format to OpenAI ChatCompletion API requests. +func NewAnthropicToChatCompletionOpenAITranslator(version string, modelNameOverride internalapi.ModelNameOverride) AnthropicMessagesTranslator { + // TODO: use "version" in APISchema struct to set the specific prefix if needed like OpenAI does. However, two questions: + // * Is there any "Anthropic compatible" API that uses a different prefix like OpenAI does? + // * Even if there is, we should refactor the APISchema struct to have "prefix" field instead of abusing "version" field. + _ = version + passthroughTranslator := NewAnthropicToAnthropicTranslator(version, modelNameOverride) + return &anthropicToOpenAIV1ChatCompletionTranslator{passthroughTranslator: &passthroughTranslator, modelNameOverride: modelNameOverride} +} + +type anthropicToOpenAIV1ChatCompletionTranslator struct { + passthroughTranslator *AnthropicMessagesTranslator + modelNameOverride internalapi.ModelNameOverride + requestModel internalapi.RequestModel + stream bool + streamState *openAIStreamToAnthropicState + // Redaction configuration for debug logging + debugLogEnabled bool + enableRedaction bool + logger *slog.Logger +} + +// RequestBody implements [AnthropicMessagesTranslator.RequestBody]. +func (a *anthropicToOpenAIV1ChatCompletionTranslator) RequestBody(_ []byte, body *anthropic.MessagesRequest, _ bool) ( + newHeaders []internalapi.Header, newBody []byte, err error, +) { + // Set translator config based on Anthropic message request + a.stream = body.Stream + // Store the request model to use as fallback for response model + a.requestModel = cmp.Or(a.modelNameOverride, body.Model) + + // Convert Anthropic message request body to OpenAI format. + openAIReq := buildOpenAIChatCompletionRequest(body, a.modelNameOverride) + + newBody, err = json.Marshal(openAIReq) + if err != nil { + return nil, nil, fmt.Errorf("failed to marshal OpenAI request: %w", err) + } + + // Add stop sequences via sjson because ChatCompletionNewParamsStopUnion (from the external openai-go SDK) + // requires importing the external package. Using sjson avoids that dependency. + if len(body.StopSequences) > 0 { + newBody, err = sjson.SetBytesOptions(newBody, "stop", body.StopSequences, sjsonOptions) + if err != nil { + return nil, nil, fmt.Errorf("failed to set stop sequences: %w", err) + } + } + + if body.Stream { + a.streamState = &openAIStreamToAnthropicState{ + activeTools: make(map[int64]*streamToolCall), + requestModel: a.requestModel, + } + } + + newHeaders = []internalapi.Header{ + {pathHeaderName, "/v1/chat/completions"}, + {contentLengthHeaderName, strconv.Itoa(len(newBody))}, + } + return +} + +// ResponseHeaders implements [AnthropicMessagesTranslator.ResponseHeaders]. +func (a *anthropicToOpenAIV1ChatCompletionTranslator) ResponseHeaders(_ map[string]string) ( + newHeaders []internalapi.Header, err error, +) { + return nil, nil +} + +// ResponseBody implements [AnthropicMessagesTranslator.ResponseBody]. +func (a *anthropicToOpenAIV1ChatCompletionTranslator) ResponseBody(_ map[string]string, body io.Reader, endOfStream bool, span tracingapi.MessageSpan) ( + newHeaders []internalapi.Header, newBody []byte, tokenUsage metrics.TokenUsage, responseModel string, err error, +) { + if a.stream { + return a.responseBodyStreaming(body, endOfStream) + } + return a.responseBodyNonStreaming(body, span) +} + +// responseBodyNonStreaming converts an OpenAI ChatCompletionResponse to Anthropic MessagesResponse format. +func (a *anthropicToOpenAIV1ChatCompletionTranslator) responseBodyNonStreaming(body io.Reader, span tracingapi.MessageSpan) ( + newHeaders []internalapi.Header, newBody []byte, tokenUsage metrics.TokenUsage, responseModel string, err error, +) { + responseModel = a.requestModel + + openAIResp := &openai.ChatCompletionResponse{} + if err = json.NewDecoder(body).Decode(openAIResp); err != nil { + return nil, nil, tokenUsage, responseModel, fmt.Errorf("failed to unmarshal OpenAI response: %w", err) + } + + responseModel = cmp.Or(openAIResp.Model, a.requestModel) + + tokenUsage = metrics.ExtractTokenUsageFromExplicitCaching( + int64(openAIResp.Usage.PromptTokens), + int64(openAIResp.Usage.CompletionTokens), + nil, + nil, + ) + + anthropicResp := openAIResponseToAnthropic(openAIResp, responseModel) + + // Redact and log response when enabled + if a.debugLogEnabled && a.enableRedaction && a.logger != nil { + redactedResp := a.RedactAnthropicBody(anthropicResp) + if jsonBody, marshalErr := json.Marshal(redactedResp); marshalErr == nil { + a.logger.Debug("response body processing", slog.Any("response", string(jsonBody))) + } + } + + if span != nil { + span.RecordResponse(anthropicResp) + } + + newBody, err = json.Marshal(anthropicResp) + if err != nil { + return nil, nil, tokenUsage, responseModel, fmt.Errorf("failed to marshal Anthropic response: %w", err) + } + newHeaders = []internalapi.Header{{contentLengthHeaderName, strconv.Itoa(len(newBody))}} + return +} + +// responseBodyStreaming handles converting OpenAI SSE chunks to Anthropic SSE events. +func (a *anthropicToOpenAIV1ChatCompletionTranslator) responseBodyStreaming(body io.Reader, endOfStream bool) ( + newHeaders []internalapi.Header, newBody []byte, tokenUsage metrics.TokenUsage, responseModel string, err error, +) { + responseModel = a.requestModel + + if a.streamState == nil { + return nil, nil, tokenUsage, responseModel, fmt.Errorf("stream state not initialized") + } + + // Read body into streamState's buffer + if _, err = a.streamState.buffer.ReadFrom(body); err != nil { + return nil, nil, tokenUsage, responseModel, fmt.Errorf("failed to read stream body: %w", err) + } + + // Initialize out as a non-nil empty slice so that if no Anthropic events are emitted + // (e.g., for finish_reason-only chunks or [DONE]), we still return a non-nil newBody. + // A non-nil empty body tells Envoy to replace the chunk with nothing, suppressing the + // raw upstream bytes instead of passing them through unchanged. + out := make([]byte, 0) + if err = a.streamState.processBuffer(&out, endOfStream); err != nil { + return nil, nil, tokenUsage, responseModel, err + } + + // Update responseModel if updated in streamState or take requested model + responseModel = cmp.Or(a.streamState.model, a.requestModel) + tokenUsage = a.streamState.tokenUsage + + // Always return newBody (even if empty) to suppress the original upstream chunk. + newBody = out + return +} + +// ResponseError implements [AnthropicMessagesTranslator] for Anthropic to OpenAI translation. +func (a *anthropicToOpenAIV1ChatCompletionTranslator) ResponseError(respHeaders map[string]string, r io.Reader) ( + newHeaders []internalapi.Header, + mutatedBody []byte, + err error, +) { + statusCode := respHeaders[statusHeaderName] + var anthropicError anthropic.ErrorResponse + + if strings.Contains(respHeaders[contentTypeHeaderName], jsonContentType) { + // OpenAI backend returned a structured JSON error; translate to Anthropic error format. + var openaiErr openai.Error + if err = json.NewDecoder(r).Decode(&openaiErr); err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal OpenAI error body: %w", err) + } + anthropicError = anthropic.ErrorResponse{ + Type: "error", + Error: anthropic.ErrorResponseMessage{ + Type: openaiErr.Error.Type, + Message: openaiErr.Error.Message, + }, + } + } else { + var buf []byte + buf, err = io.ReadAll(r) + if err != nil { + return nil, nil, fmt.Errorf("failed to read error body: %w", err) + } + var typ string + switch statusCode { + case "400": + typ = "invalid_request_error" + case "401": + typ = "authentication_error" + case "403": + typ = "permission_error" + case "404": + typ = "not_found_error" + case "413": + typ = "request_too_large" + case "429": + typ = "rate_limit_error" + case "500": + typ = "internal_server_error" + case "503": + typ = "service_unavailable_error" + case "529": + typ = "overloaded_error" + default: + typ = "internal_server_error" + } + anthropicError = anthropic.ErrorResponse{ + Type: "error", // Always "error" at the top level. + Error: anthropic.ErrorResponseMessage{Type: typ, Message: string(buf)}, + } + } + + mutatedBody, err = json.Marshal(anthropicError) + if err != nil { + return nil, nil, fmt.Errorf("failed to marshal error body: %w", err) + } + newHeaders = append(newHeaders, + internalapi.Header{contentTypeHeaderName, jsonContentType}, + internalapi.Header{contentLengthHeaderName, strconv.Itoa(len(mutatedBody))}, + ) + return +} + +// SetRedactionConfig implements [AnthropicResponseRedactor.SetRedactionConfig]. +func (a *anthropicToOpenAIV1ChatCompletionTranslator) SetRedactionConfig(debugLogEnabled, enableRedaction bool, logger *slog.Logger) { + a.debugLogEnabled = debugLogEnabled + a.enableRedaction = enableRedaction + a.logger = logger +} + +// RedactAnthropicBody implements [AnthropicResponseRedactor.RedactAnthropicBody]. +// Creates a redacted copy of the Anthropic response for safe logging without modifying the original. +func (a *anthropicToOpenAIV1ChatCompletionTranslator) RedactAnthropicBody(resp *anthropic.MessagesResponse) *anthropic.MessagesResponse { + if resp == nil { + return nil + } + + // Create a shallow copy of the response + redacted := *resp + + // Redact content blocks (contains AI-generated content) + if len(resp.Content) > 0 { + redacted.Content = make([]anthropic.MessagesContentBlock, len(resp.Content)) + for i := range resp.Content { + redacted.Content[i] = redactAnthropicContent(&resp.Content[i]) + } + } + + return &redacted +} diff --git a/internal/translator/anthropic_openai_test.go b/internal/translator/anthropic_openai_test.go new file mode 100644 index 0000000000..e7d135f4ff --- /dev/null +++ b/internal/translator/anthropic_openai_test.go @@ -0,0 +1,717 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package translator + +import ( + "bytes" + "io" + "log/slog" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/envoyproxy/ai-gateway/internal/apischema/anthropic" + "github.com/envoyproxy/ai-gateway/internal/apischema/openai" + "github.com/envoyproxy/ai-gateway/internal/json" +) + +func TestAnthropicToOpenAITranslator_RequestBody(t *testing.T) { + tests := []struct { + name string + body *anthropic.MessagesRequest + modelOverride string + wantModel string + wantStreaming bool + wantStopSeqs []string + }{ + { + name: "basic request sets correct path header", + body: &anthropic.MessagesRequest{ + Model: "claude-3-haiku", + MaxTokens: 100, + Messages: []anthropic.MessageParam{{Role: anthropic.MessageRoleUser, Content: anthropic.MessageContent{Text: "Hello"}}}, + }, + wantModel: "claude-3-haiku", + }, + { + name: "model override replaces model in body", + body: &anthropic.MessagesRequest{ + Model: "claude-3-haiku", + MaxTokens: 100, + Messages: []anthropic.MessageParam{{Role: anthropic.MessageRoleUser, Content: anthropic.MessageContent{Text: "Hi"}}}, + }, + modelOverride: "gpt-4o", + wantModel: "gpt-4o", + }, + { + name: "streaming request sets stream and stream_options", + body: &anthropic.MessagesRequest{ + Model: "claude-3", + MaxTokens: 50, + Stream: true, + Messages: []anthropic.MessageParam{{Role: anthropic.MessageRoleUser, Content: anthropic.MessageContent{Text: "Hi"}}}, + }, + wantModel: "claude-3", + wantStreaming: true, + }, + { + name: "stop sequences added to body", + body: &anthropic.MessagesRequest{ + Model: "claude-3", + MaxTokens: 100, + Messages: []anthropic.MessageParam{{Role: anthropic.MessageRoleUser, Content: anthropic.MessageContent{Text: "Hi"}}}, + StopSequences: []string{"Human:", "AI:"}, + }, + wantModel: "claude-3", + wantStopSeqs: []string{"Human:", "AI:"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + translator := NewAnthropicToChatCompletionOpenAITranslator("v1", tt.modelOverride) + headers, body, err := translator.RequestBody(nil, tt.body, false) + require.NoError(t, err) + require.NotNil(t, headers) + require.NotNil(t, body) + + // Verify the two headers: path and content-length. + require.Len(t, headers, 2) + assert.Equal(t, pathHeaderName, headers[0].Key()) + assert.Equal(t, "/v1/chat/completions", headers[0].Value()) + assert.Equal(t, contentLengthHeaderName, headers[1].Key()) + + // Verify body contains the correct model. + var req map[string]any + require.NoError(t, json.Unmarshal(body, &req)) + assert.Equal(t, tt.wantModel, req["model"]) + + if tt.wantStreaming { + assert.Equal(t, true, req["stream"]) + streamOpts, ok := req["stream_options"].(map[string]any) + require.True(t, ok, "stream_options should be a map") + assert.Equal(t, true, streamOpts["include_usage"]) + } + + if len(tt.wantStopSeqs) > 0 { + stopSeqs, ok := req["stop"].([]any) + require.True(t, ok, "stop should be an array") + require.Len(t, stopSeqs, len(tt.wantStopSeqs)) + for i, s := range tt.wantStopSeqs { + assert.Equal(t, s, stopSeqs[i]) + } + } + }) + } +} + +func TestAnthropicToOpenAITranslator_ResponseHeaders(t *testing.T) { + translator := NewAnthropicToChatCompletionOpenAITranslator("v1", "") + headers, err := translator.ResponseHeaders(map[string]string{ + "content-type": "application/json", + "x-custom": "value", + }) + require.NoError(t, err) + assert.Nil(t, headers, "ResponseHeaders should always return nil for passthrough") +} + +func TestAnthropicToOpenAITranslator_ResponseBody_NonStreaming(t *testing.T) { + t.Run("text content is converted to Anthropic format", func(t *testing.T) { + translator := NewAnthropicToChatCompletionOpenAITranslator("v1", "") + reqBody := &anthropic.MessagesRequest{ + Model: "claude-3-haiku", + MaxTokens: 100, + Messages: []anthropic.MessageParam{{Role: anthropic.MessageRoleUser, Content: anthropic.MessageContent{Text: "Hi"}}}, + } + _, _, err := translator.RequestBody(nil, reqBody, false) + require.NoError(t, err) + + content := "Hello from OpenAI!" + openAIResp := openai.ChatCompletionResponse{ + ID: "chatcmpl-123", + Model: "gpt-4o", + Choices: []openai.ChatCompletionResponseChoice{ + { + FinishReason: openai.ChatCompletionChoicesFinishReasonStop, + Message: openai.ChatCompletionResponseChoiceMessage{ + Content: &content, + Role: "assistant", + }, + }, + }, + Usage: openai.Usage{PromptTokens: 10, CompletionTokens: 20}, + } + respBytes, err := json.Marshal(openAIResp) + require.NoError(t, err) + + headers, body, tokenUsage, responseModel, err := translator.ResponseBody( + map[string]string{"content-type": "application/json"}, + bytes.NewReader(respBytes), + true, + nil, + ) + require.NoError(t, err) + require.NotNil(t, body) + + // Response model should come from the OpenAI response. + assert.Equal(t, "gpt-4o", responseModel) + + // Verify content-length header. + require.Len(t, headers, 1) + assert.Equal(t, contentLengthHeaderName, headers[0].Key()) + + // Verify token usage extracted correctly. + inputTokens, inputSet := tokenUsage.InputTokens() + outputTokens, outputSet := tokenUsage.OutputTokens() + assert.True(t, inputSet) + assert.Equal(t, uint32(10), inputTokens) + assert.True(t, outputSet) + assert.Equal(t, uint32(20), outputTokens) + + // Verify Anthropic response body. + var anthropicResp anthropic.MessagesResponse + require.NoError(t, json.Unmarshal(body, &anthropicResp)) + assert.Equal(t, "chatcmpl-123", anthropicResp.ID) + assert.Equal(t, "gpt-4o", anthropicResp.Model) + require.Len(t, anthropicResp.Content, 1) + require.NotNil(t, anthropicResp.Content[0].Text) + assert.Equal(t, "Hello from OpenAI!", anthropicResp.Content[0].Text.Text) + require.NotNil(t, anthropicResp.StopReason) + assert.Equal(t, anthropic.StopReasonEndTurn, *anthropicResp.StopReason) + }) + + t.Run("model falls back to request model when absent in response", func(t *testing.T) { + translator := NewAnthropicToChatCompletionOpenAITranslator("v1", "") + reqBody := &anthropic.MessagesRequest{ + Model: "claude-3-haiku", + MaxTokens: 100, + Messages: []anthropic.MessageParam{{Role: anthropic.MessageRoleUser, Content: anthropic.MessageContent{Text: "Hi"}}}, + } + _, _, err := translator.RequestBody(nil, reqBody, false) + require.NoError(t, err) + + content := "Hi!" + openAIResp := openai.ChatCompletionResponse{ + // No Model field. + Choices: []openai.ChatCompletionResponseChoice{ + { + FinishReason: openai.ChatCompletionChoicesFinishReasonStop, + Message: openai.ChatCompletionResponseChoiceMessage{Content: &content}, + }, + }, + } + respBytes, err := json.Marshal(openAIResp) + require.NoError(t, err) + + _, _, _, responseModel, err := translator.ResponseBody( + map[string]string{"content-type": "application/json"}, + bytes.NewReader(respBytes), + true, + nil, + ) + require.NoError(t, err) + assert.Equal(t, "claude-3-haiku", responseModel, "should fall back to request model") + }) + + t.Run("model override is used as fallback", func(t *testing.T) { + translator := NewAnthropicToChatCompletionOpenAITranslator("v1", "gpt-4o-override") + reqBody := &anthropic.MessagesRequest{ + Model: "claude-3", + MaxTokens: 100, + Messages: []anthropic.MessageParam{{Role: anthropic.MessageRoleUser, Content: anthropic.MessageContent{Text: "Hi"}}}, + } + _, _, err := translator.RequestBody(nil, reqBody, false) + require.NoError(t, err) + + // OpenAI response with no model field. + content := "Hi!" + openAIResp := openai.ChatCompletionResponse{ + Choices: []openai.ChatCompletionResponseChoice{ + {Message: openai.ChatCompletionResponseChoiceMessage{Content: &content}}, + }, + } + respBytes, err := json.Marshal(openAIResp) + require.NoError(t, err) + + _, _, _, responseModel, err := translator.ResponseBody( + map[string]string{}, + bytes.NewReader(respBytes), + true, + nil, + ) + require.NoError(t, err) + assert.Equal(t, "gpt-4o-override", responseModel) + }) + + t.Run("tool call response converted to tool_use blocks", func(t *testing.T) { + translator := NewAnthropicToChatCompletionOpenAITranslator("v1", "") + reqBody := &anthropic.MessagesRequest{ + Model: "claude-3", + MaxTokens: 100, + Messages: []anthropic.MessageParam{{Role: anthropic.MessageRoleUser, Content: anthropic.MessageContent{Text: "Get weather"}}}, + } + _, _, err := translator.RequestBody(nil, reqBody, false) + require.NoError(t, err) + + toolID := "call-abc" + openAIResp := openai.ChatCompletionResponse{ + ID: "chatcmpl-456", + Model: "gpt-4o", + Choices: []openai.ChatCompletionResponseChoice{ + { + FinishReason: openai.ChatCompletionChoicesFinishReasonToolCalls, + Message: openai.ChatCompletionResponseChoiceMessage{ + ToolCalls: []openai.ChatCompletionMessageToolCallParam{ + { + ID: &toolID, + Function: openai.ChatCompletionMessageToolCallFunctionParam{ + Name: "get_weather", + Arguments: `{"location":"NYC"}`, + }, + }, + }, + }, + }, + }, + Usage: openai.Usage{PromptTokens: 15, CompletionTokens: 8}, + } + respBytes, err := json.Marshal(openAIResp) + require.NoError(t, err) + + _, body, _, _, err := translator.ResponseBody( + map[string]string{"content-type": "application/json"}, + bytes.NewReader(respBytes), + true, + nil, + ) + require.NoError(t, err) + + var anthropicResp anthropic.MessagesResponse + require.NoError(t, json.Unmarshal(body, &anthropicResp)) + require.Len(t, anthropicResp.Content, 1) + require.NotNil(t, anthropicResp.Content[0].Tool) + assert.Equal(t, "call-abc", anthropicResp.Content[0].Tool.ID) + assert.Equal(t, "get_weather", anthropicResp.Content[0].Tool.Name) + assert.Equal(t, map[string]any{"location": "NYC"}, anthropicResp.Content[0].Tool.Input) + require.NotNil(t, anthropicResp.StopReason) + assert.Equal(t, anthropic.StopReasonToolUse, *anthropicResp.StopReason) + }) +} + +func TestAnthropicToOpenAITranslator_ResponseBody_Streaming(t *testing.T) { + translator := NewAnthropicToChatCompletionOpenAITranslator("v1", "claude-3-haiku") + + // Initialize streaming mode via RequestBody. + reqBody := &anthropic.MessagesRequest{ + Model: "claude-3-haiku", + MaxTokens: 100, + Stream: true, + Messages: []anthropic.MessageParam{{Role: anthropic.MessageRoleUser, Content: anthropic.MessageContent{Text: "Hello"}}}, + } + _, _, err := translator.RequestBody(nil, reqBody, false) + require.NoError(t, err) + + // Feed all OpenAI SSE chunks at once. + // Chunk 1: text delta → message_start + content_block_start + content_block_delta + // Chunk 2: finish_reason → stores stop reason + // Chunk 3: usage-only → content_block_stop + message_delta + message_stop + // [DONE]: skipped + input := "data: {\"id\":\"chatcmpl-xyz\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"Hello!\"}}],\"model\":\"gpt-4o\"}\n\n" + + "data: {\"id\":\"chatcmpl-xyz\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n" + + "data: {\"id\":\"chatcmpl-xyz\",\"choices\":[],\"usage\":{\"prompt_tokens\":10,\"completion_tokens\":5}}\n\n" + + "data: [DONE]\n\n" + + _, body, tokenUsage, responseModel, err := translator.ResponseBody( + map[string]string{"content-type": "text/event-stream"}, + strings.NewReader(input), + true, + nil, + ) + require.NoError(t, err) + require.NotEmpty(t, body) + + // Model should come from the OpenAI chunk, not the model override. + assert.Equal(t, "gpt-4o", responseModel) + + // Verify token usage extracted from the usage chunk. + inputTokens, inputSet := tokenUsage.InputTokens() + outputTokens, outputSet := tokenUsage.OutputTokens() + assert.True(t, inputSet) + assert.Equal(t, uint32(10), inputTokens) + assert.True(t, outputSet) + assert.Equal(t, uint32(5), outputTokens) + + // Verify output SSE event sequence. + events := parseSSEEventsFromBytes(body) + require.Len(t, events, 6) + assert.Equal(t, "message_start", events[0].eventType) + assert.Equal(t, "content_block_start", events[1].eventType) + assert.Equal(t, "content_block_delta", events[2].eventType) + assert.Equal(t, "content_block_stop", events[3].eventType) + assert.Equal(t, "message_delta", events[4].eventType) + assert.Equal(t, "message_stop", events[5].eventType) + + // Spot-check specific event data. + require.JSONEq(t, `{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hello!"}}`, events[2].data) + require.JSONEq(t, `{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":5}}`, events[4].data) + require.JSONEq(t, `{"type":"message_stop"}`, events[5].data) +} + +func TestAnthropicToOpenAITranslator_ResponseBody_StreamingRequestModelFallback(t *testing.T) { + // When the OpenAI chunk has no model, responseModel should fall back to requestModel. + translator := NewAnthropicToChatCompletionOpenAITranslator("v1", "my-override") + reqBody := &anthropic.MessagesRequest{ + Model: "claude-3", + MaxTokens: 100, + Stream: true, + Messages: []anthropic.MessageParam{{Role: anthropic.MessageRoleUser, Content: anthropic.MessageContent{Text: "Hi"}}}, + } + _, _, err := translator.RequestBody(nil, reqBody, false) + require.NoError(t, err) + + // Usage-only chunk with no model information at all. + input := "data: {\"id\":\"chatcmpl-noop\",\"choices\":[],\"usage\":{\"prompt_tokens\":5,\"completion_tokens\":2}}\n\n" + + _, _, _, responseModel, err := translator.ResponseBody( + map[string]string{}, + strings.NewReader(input), + true, + nil, + ) + require.NoError(t, err) + // Model from chunks is empty, so falls back to modelNameOverride (which is the requestModel stored). + assert.Equal(t, "my-override", responseModel) +} + +func TestAnthropicToOpenAITranslator_ResponseError(t *testing.T) { + tests := []struct { + name string + headers map[string]string + body string + wantErrType string + wantErrMsg string + }{ + { + name: "JSON error from OpenAI backend", + headers: map[string]string{contentTypeHeaderName: "application/json"}, + body: `{"type":"error","error":{"type":"invalid_request_error","message":"Bad request"}}`, + wantErrType: "invalid_request_error", + wantErrMsg: "Bad request", + }, + { + name: "non-JSON 400 error", + headers: map[string]string{statusHeaderName: "400", contentTypeHeaderName: "text/plain"}, + body: "Bad request body", + wantErrType: "invalid_request_error", + wantErrMsg: "Bad request body", + }, + { + name: "non-JSON 401 error", + headers: map[string]string{statusHeaderName: "401", contentTypeHeaderName: "text/plain"}, + body: "Unauthorized", + wantErrType: "authentication_error", + wantErrMsg: "Unauthorized", + }, + { + name: "non-JSON 403 error", + headers: map[string]string{statusHeaderName: "403", contentTypeHeaderName: "text/plain"}, + body: "Forbidden", + wantErrType: "permission_error", + wantErrMsg: "Forbidden", + }, + { + name: "non-JSON 404 error", + headers: map[string]string{statusHeaderName: "404", contentTypeHeaderName: "text/plain"}, + body: "Not found", + wantErrType: "not_found_error", + wantErrMsg: "Not found", + }, + { + name: "non-JSON 413 error", + headers: map[string]string{statusHeaderName: "413", contentTypeHeaderName: "text/plain"}, + body: "Request too large", + wantErrType: "request_too_large", + wantErrMsg: "Request too large", + }, + { + name: "non-JSON 429 error", + headers: map[string]string{statusHeaderName: "429", contentTypeHeaderName: "text/plain"}, + body: "Rate limited", + wantErrType: "rate_limit_error", + wantErrMsg: "Rate limited", + }, + { + name: "non-JSON 500 error", + headers: map[string]string{statusHeaderName: "500", contentTypeHeaderName: "text/plain"}, + body: "Internal error", + wantErrType: "internal_server_error", + wantErrMsg: "Internal error", + }, + { + name: "non-JSON 503 error", + headers: map[string]string{statusHeaderName: "503", contentTypeHeaderName: "text/plain"}, + body: "Service unavailable", + wantErrType: "service_unavailable_error", + wantErrMsg: "Service unavailable", + }, + { + name: "non-JSON 529 error", + headers: map[string]string{statusHeaderName: "529", contentTypeHeaderName: "text/plain"}, + body: "Overloaded", + wantErrType: "overloaded_error", + wantErrMsg: "Overloaded", + }, + { + name: "non-JSON unknown status defaults to internal_server_error", + headers: map[string]string{statusHeaderName: "599", contentTypeHeaderName: "text/plain"}, + body: "Unknown error", + wantErrType: "internal_server_error", + wantErrMsg: "Unknown error", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + translator := NewAnthropicToChatCompletionOpenAITranslator("v1", "") + headers, mutatedBody, err := translator.ResponseError(tt.headers, strings.NewReader(tt.body)) + require.NoError(t, err) + require.NotNil(t, mutatedBody) + + // Verify content-type and content-length headers are set. + require.Len(t, headers, 2) + assert.Equal(t, contentTypeHeaderName, headers[0].Key()) + assert.Equal(t, jsonContentType, headers[0].Value()) //nolint:testifylint + assert.Equal(t, contentLengthHeaderName, headers[1].Key()) + + // Verify the Anthropic error body. + var errResp anthropic.ErrorResponse + require.NoError(t, json.Unmarshal(mutatedBody, &errResp)) + assert.Equal(t, "error", errResp.Type) + assert.Equal(t, tt.wantErrType, errResp.Error.Type) + assert.Equal(t, tt.wantErrMsg, errResp.Error.Message) + }) + } +} + +func TestAnthropicToOpenAITranslator_RedactAnthropicBody(t *testing.T) { + translator := NewAnthropicToChatCompletionOpenAITranslator("v1", "") + tr := translator.(*anthropicToOpenAIV1ChatCompletionTranslator) + + t.Run("nil response returns nil", func(t *testing.T) { + assert.Nil(t, tr.RedactAnthropicBody(nil)) + }) + + t.Run("non-nil response is shallow-copied with content redacted", func(t *testing.T) { + resp := &anthropic.MessagesResponse{ + ID: "msg-123", + Model: "gpt-4o", + Content: []anthropic.MessagesContentBlock{ + {Text: &anthropic.TextBlock{Type: "text", Text: "some sensitive text"}}, + }, + } + redacted := tr.RedactAnthropicBody(resp) + require.NotNil(t, redacted) + + // The original response must not be modified. + assert.Equal(t, "some sensitive text", resp.Content[0].Text.Text) + + // Top-level non-content fields are preserved. + assert.Equal(t, "msg-123", redacted.ID) + assert.Equal(t, "gpt-4o", redacted.Model) + + // Content blocks are present (redaction creates a new slice). + require.Len(t, redacted.Content, 1) + }) + + t.Run("empty content response is safe", func(t *testing.T) { + resp := &anthropic.MessagesResponse{ + ID: "msg-empty", + Model: "gpt-4o", + Content: nil, + } + redacted := tr.RedactAnthropicBody(resp) + require.NotNil(t, redacted) + assert.Equal(t, "msg-empty", redacted.ID) + assert.Nil(t, redacted.Content) + }) +} + +// mockMessageSpan implements tracingapi.MessageSpan for testing. +type mockMessageSpan struct { + recordedResponse *anthropic.MessagesResponse +} + +func (m *mockMessageSpan) RecordResponse(resp *anthropic.MessagesResponse) { + m.recordedResponse = resp +} +func (m *mockMessageSpan) RecordResponseChunk(_ *anthropic.MessagesStreamChunk) {} +func (m *mockMessageSpan) EndSpanOnError(_ int, _ []byte) {} +func (m *mockMessageSpan) EndSpan() {} + +// buildOpenAITextResponse is a helper that marshals a simple OpenAI ChatCompletionResponse +// containing a single text choice and returns it as a bytes.Reader. +func buildOpenAITextResponse(id, model, text string) *bytes.Reader { + content := text + resp := openai.ChatCompletionResponse{ + ID: id, + Model: model, + Choices: []openai.ChatCompletionResponseChoice{ + { + FinishReason: openai.ChatCompletionChoicesFinishReasonStop, + Message: openai.ChatCompletionResponseChoiceMessage{Content: &content, Role: "assistant"}, + }, + }, + Usage: openai.Usage{PromptTokens: 5, CompletionTokens: 3}, + } + b, _ := json.Marshal(resp) + return bytes.NewReader(b) +} + +// initNonStreamingTranslator initialises the translator for a basic non-streaming request so +// that requestModel and stream fields are correctly populated before calling ResponseBody. +func initNonStreamingTranslator(t *testing.T, modelOverride string) *anthropicToOpenAIV1ChatCompletionTranslator { + t.Helper() + tr := NewAnthropicToChatCompletionOpenAITranslator("v1", modelOverride).(*anthropicToOpenAIV1ChatCompletionTranslator) + req := &anthropic.MessagesRequest{ + Model: "claude-3-haiku", + MaxTokens: 100, + Messages: []anthropic.MessageParam{{Role: anthropic.MessageRoleUser, Content: anthropic.MessageContent{Text: "Hi"}}}, + } + _, _, err := tr.RequestBody(nil, req, false) + require.NoError(t, err) + return tr +} + +// SetRedactionConfig should store all three parameters on the struct. +func TestAnthropicToOpenAITranslator_SetRedactionConfig(t *testing.T) { + tr := NewAnthropicToChatCompletionOpenAITranslator("v1", "").(*anthropicToOpenAIV1ChatCompletionTranslator) + + assert.False(t, tr.debugLogEnabled) + assert.False(t, tr.enableRedaction) + assert.Nil(t, tr.logger) + + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + tr.SetRedactionConfig(true, true, logger) + + assert.True(t, tr.debugLogEnabled) + assert.True(t, tr.enableRedaction) + assert.Equal(t, logger, tr.logger) +} + +// Debug logging should only fire when debugLogEnabled AND enableRedaction AND logger != nil. +func TestAnthropicToOpenAITranslator_ResponseBody_DebugLogging(t *testing.T) { + makeLogger := func(buf *bytes.Buffer) *slog.Logger { + return slog.New(slog.NewTextHandler(buf, &slog.HandlerOptions{Level: slog.LevelDebug})) + } + + t.Run("logs when all conditions are true", func(t *testing.T) { + var buf bytes.Buffer + tr := initNonStreamingTranslator(t, "") + tr.SetRedactionConfig(true, true, makeLogger(&buf)) + + _, _, _, _, err := tr.ResponseBody( + map[string]string{}, + buildOpenAITextResponse("id-1", "gpt-4o", "hello"), + true, + nil, + ) + require.NoError(t, err) + assert.Contains(t, buf.String(), "response body processing") + }) + + t.Run("no log when debugLogEnabled is false", func(t *testing.T) { + var buf bytes.Buffer + tr := initNonStreamingTranslator(t, "") + tr.SetRedactionConfig(false, true, makeLogger(&buf)) + + _, _, _, _, err := tr.ResponseBody( + map[string]string{}, + buildOpenAITextResponse("id-2", "gpt-4o", "hello"), + true, + nil, + ) + require.NoError(t, err) + assert.Empty(t, buf.String()) + }) + + t.Run("no log when enableRedaction is false", func(t *testing.T) { + var buf bytes.Buffer + tr := initNonStreamingTranslator(t, "") + tr.SetRedactionConfig(true, false, makeLogger(&buf)) + + _, _, _, _, err := tr.ResponseBody( + map[string]string{}, + buildOpenAITextResponse("id-3", "gpt-4o", "hello"), + true, + nil, + ) + require.NoError(t, err) + assert.Empty(t, buf.String()) + }) + + t.Run("no log when logger is nil", func(t *testing.T) { + tr := initNonStreamingTranslator(t, "") + tr.SetRedactionConfig(true, true, nil) + + // Should not panic even though logger is nil (guarded by the nil check). + _, _, _, _, err := tr.ResponseBody( + map[string]string{}, + buildOpenAITextResponse("id-4", "gpt-4o", "hello"), + true, + nil, + ) + require.NoError(t, err) + }) +} + +// When a non-nil span is provided, RecordResponse should be called with the converted response. +func TestAnthropicToOpenAITranslator_ResponseBody_SpanRecording(t *testing.T) { + t.Run("span RecordResponse called with converted response", func(t *testing.T) { + tr := initNonStreamingTranslator(t, "") + span := &mockMessageSpan{} + + _, _, _, _, err := tr.ResponseBody( + map[string]string{}, + buildOpenAITextResponse("chatcmpl-span", "gpt-4o", "span content"), + true, + span, + ) + require.NoError(t, err) + require.NotNil(t, span.recordedResponse, "RecordResponse should have been called") + assert.Equal(t, "chatcmpl-span", span.recordedResponse.ID) + }) + + t.Run("nil span does not panic", func(t *testing.T) { + tr := initNonStreamingTranslator(t, "") + + _, _, _, _, err := tr.ResponseBody( + map[string]string{}, + buildOpenAITextResponse("chatcmpl-nospan", "gpt-4o", "no span"), + true, + nil, + ) + require.NoError(t, err) + }) +} + +// responseBodyStreaming should return an error when streamState is nil. +func TestAnthropicToOpenAITranslator_ResponseBody_StreamStateNilGuard(t *testing.T) { + tr := NewAnthropicToChatCompletionOpenAITranslator("v1", "").(*anthropicToOpenAIV1ChatCompletionTranslator) + // Manually enable streaming without initialising streamState to trigger the nil guard. + tr.stream = true + tr.streamState = nil + + _, _, _, _, err := tr.ResponseBody( + map[string]string{}, + strings.NewReader("data: {}\n\n"), + true, + nil, + ) + require.Error(t, err) + assert.Contains(t, err.Error(), "stream state not initialized") +} diff --git a/internal/translator/openai_helper.go b/internal/translator/openai_helper.go new file mode 100644 index 0000000000..37b60a5bbc --- /dev/null +++ b/internal/translator/openai_helper.go @@ -0,0 +1,684 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package translator + +import ( + "bytes" + "cmp" + "fmt" + "strings" + + "k8s.io/utils/ptr" + + "github.com/envoyproxy/ai-gateway/internal/apischema/anthropic" + "github.com/envoyproxy/ai-gateway/internal/apischema/openai" + "github.com/envoyproxy/ai-gateway/internal/internalapi" + "github.com/envoyproxy/ai-gateway/internal/json" + "github.com/envoyproxy/ai-gateway/internal/metrics" +) + +// The following are helper functions for creating an OpenAI ChatCompletionRequest from an Anthropic MessagesRequest + +// buildOpenAIChatCompletionRequest converts an Anthropic MessagesRequest into an OpenAI ChatCompletionRequest. +// It handles model override, system prompts, message conversion, tools, and tool choice. +func buildOpenAIChatCompletionRequest(body *anthropic.MessagesRequest, modelNameOverride internalapi.ModelNameOverride) *openai.ChatCompletionRequest { + model := cmp.Or(modelNameOverride, body.Model) + messages := anthropicMessagesToOpenAI(body) + tools := anthropicToolsToOpenAI(body.Tools) + var toolChoiceVal anthropic.ToolChoice + if body.ToolChoice != nil { + toolChoiceVal = *body.ToolChoice + } + toolChoice := anthropicToolChoiceToOpenAI(toolChoiceVal, len(tools) > 0) + + maxTokens := int64(body.MaxTokens) + req := &openai.ChatCompletionRequest{ + Model: model, + Messages: messages, + MaxCompletionTokens: &maxTokens, + Temperature: body.Temperature, + TopP: body.TopP, + Stream: body.Stream, + } + + if len(tools) > 0 { + req.Tools = tools + req.ToolChoice = toolChoice + } + + if body.Stream { + req.StreamOptions = &openai.StreamOptions{IncludeUsage: true} + } + + return req +} + +// anthropicMessagesToOpenAI converts Anthropic messages (including the system prompt) to OpenAI message format. +func anthropicMessagesToOpenAI(body *anthropic.MessagesRequest) []openai.ChatCompletionMessageParamUnion { + var messages []openai.ChatCompletionMessageParamUnion + + // Prepend the system prompt as an OpenAI system message. + if body.System != nil { + systemText := anthropicSystemPromptToText(body.System) + if systemText != "" { + messages = append(messages, openai.ChatCompletionMessageParamUnion{ + OfSystem: &openai.ChatCompletionSystemMessageParam{ + Content: openai.ContentUnion{Value: systemText}, + Role: openai.ChatMessageRoleSystem, + }, + }) + } + } + + for _, msg := range body.Messages { + switch msg.Role { + case anthropic.MessageRoleUser: + messages = append(messages, openai.ChatCompletionMessageParamUnion{ + OfUser: &openai.ChatCompletionUserMessageParam{ + Content: openai.StringOrUserRoleContentUnion{Value: anthropicContentToText(msg.Content)}, + Role: openai.ChatMessageRoleUser, + }, + }) + case anthropic.MessageRoleAssistant: + messages = append(messages, openai.ChatCompletionMessageParamUnion{ + OfAssistant: &openai.ChatCompletionAssistantMessageParam{ + Content: openai.StringOrAssistantRoleContentUnion{Value: anthropicContentToText(msg.Content)}, + Role: openai.ChatMessageRoleAssistant, + }, + }) + } + } + + return messages +} + +// anthropicSystemPromptToText extracts a plain string from an Anthropic system prompt, +// concatenating text blocks if the prompt is in array form. +func anthropicSystemPromptToText(s *anthropic.SystemPrompt) string { + if s.Text != "" { + return s.Text + } + var sb strings.Builder + for _, t := range s.Texts { + sb.WriteString(t.Text) + } + return sb.String() +} + +// anthropicContentToText extracts a plain text string from Anthropic message content. +// For array content, text blocks are concatenated in order. +func anthropicContentToText(content anthropic.MessageContent) string { + if content.Text != "" { + return content.Text + } + var sb strings.Builder + for _, block := range content.Array { + if block.Text != nil { + sb.WriteString(block.Text.Text) + } + } + return sb.String() +} + +// anthropicToolsToOpenAI converts Anthropic custom tools to OpenAI function tools. +// Only ToolUnion entries with a custom Tool variant are converted; built-in tool types are skipped. +func anthropicToolsToOpenAI(tools []anthropic.ToolUnion) []openai.Tool { + if len(tools) == 0 { + return nil + } + result := make([]openai.Tool, 0, len(tools)) + for _, t := range tools { + if t.Tool == nil { + continue + } + result = append(result, openai.Tool{ + Type: openai.ToolTypeFunction, + Function: &openai.FunctionDefinition{ + Name: t.Tool.Name, + Description: t.Tool.Description, + Parameters: t.Tool.InputSchema, + }, + }) + } + if len(result) == 0 { + return nil + } + return result +} + +// anthropicToolChoiceToOpenAI converts an Anthropic tool_choice value to an OpenAI ChatCompletionToolChoiceUnion. +// Returns nil if no tools are present or the tool choice has no variant set. +func anthropicToolChoiceToOpenAI(tc anthropic.ToolChoice, hasTools bool) *openai.ChatCompletionToolChoiceUnion { + if !hasTools { + return nil + } + switch { + case tc.Auto != nil: + return &openai.ChatCompletionToolChoiceUnion{Value: string(openai.ToolChoiceTypeAuto)} + case tc.None != nil: + return &openai.ChatCompletionToolChoiceUnion{Value: string(openai.ToolChoiceTypeNone)} + case tc.Any != nil: + // Anthropic "any" maps to OpenAI "required" (model must call a tool). + return &openai.ChatCompletionToolChoiceUnion{Value: string(openai.ToolChoiceTypeRequired)} + case tc.Tool != nil: + return &openai.ChatCompletionToolChoiceUnion{ + Value: openai.ChatCompletionNamedToolChoice{ + Type: openai.ToolTypeFunction, + Function: openai.ChatCompletionNamedToolChoiceFunction{Name: tc.Tool.Name}, + }, + } + default: + return nil + } +} + +// The following are helper functions that convert an OpenAI ChatCompletionResponse to an Anthropic MessagesRepsonse + +// openAIResponseToAnthropic converts an OpenAI ChatCompletionResponse to an Anthropic MessagesResponse. +func openAIResponseToAnthropic(resp *openai.ChatCompletionResponse, model string) *anthropic.MessagesResponse { + var content []anthropic.MessagesContentBlock + var stopReason *anthropic.StopReason + + if len(resp.Choices) > 0 { + choice := &resp.Choices[0] + msg := &choice.Message + + // Convert text content. + if msg.Content != nil && *msg.Content != "" { + content = append(content, anthropic.MessagesContentBlock{ + Text: &anthropic.TextBlock{Type: "text", Text: *msg.Content}, + }) + } + + // Convert tool calls to tool_use content blocks. + for _, tc := range msg.ToolCalls { + var input map[string]any + if tc.Function.Arguments != "" { + _ = json.Unmarshal([]byte(tc.Function.Arguments), &input) + } + // If tool call json string is malformed (OpenAI allows this because of it being cut off mid-stream) + // then we set input to an empty map + if input == nil { + input = map[string]any{} + } + id := "" + if tc.ID != nil { + id = *tc.ID + } + content = append(content, anthropic.MessagesContentBlock{ + Tool: &anthropic.ToolUseBlock{ + Type: "tool_use", + ID: id, + Name: tc.Function.Name, + Input: input, + }, + }) + } + + sr := openAIFinishReasonToAnthropic(choice.FinishReason) + stopReason = &sr + } + + usage := &anthropic.Usage{ + InputTokens: float64(resp.Usage.PromptTokens), + OutputTokens: float64(resp.Usage.CompletionTokens), + } + + return &anthropic.MessagesResponse{ + ID: resp.ID, + Type: anthropic.ConstantMessagesResponseTypeMessages("message"), + Role: anthropic.ConstantMessagesResponseRoleAssistant("assistant"), + Content: content, + Model: model, + StopReason: stopReason, + Usage: usage, + } +} + +// openAIFinishReasonToAnthropic maps an OpenAI finish_reason to an Anthropic StopReason. +func openAIFinishReasonToAnthropic(reason openai.ChatCompletionChoicesFinishReason) anthropic.StopReason { + switch reason { + case openai.ChatCompletionChoicesFinishReasonStop: + return anthropic.StopReasonEndTurn + case openai.ChatCompletionChoicesFinishReasonLength: + return anthropic.StopReasonMaxTokens + case openai.ChatCompletionChoicesFinishReasonToolCalls: + return anthropic.StopReasonToolUse + case openai.ChatCompletionChoicesFinishReasonContentFilter: + return anthropic.StopReasonRefusal + default: + return anthropic.StopReasonEndTurn + } +} + +// The following are helpers that convert an OpenAI Stream to an Anthropic Stream (SSE conversion) + +// The following structs are used to produce deterministic JSON for Anthropic SSE events. +// Using typed structs (instead of map[string]any) ensures sonic serializes fields in +// declaration order, making the output stable across runs. + +type sseMessageStart struct { + Type string `json:"type"` + Message sseMessageBody `json:"message"` +} + +type sseMessageBody struct { + ID string `json:"id"` + Type string `json:"type"` + Role string `json:"role"` + Content []any `json:"content"` + Model string `json:"model"` + StopReason any `json:"stop_reason"` + StopSequence any `json:"stop_sequence"` + Usage sseMessageUsage `json:"usage"` +} + +type sseMessageUsage struct { + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` +} + +type sseContentBlockStartText struct { + Type string `json:"type"` + Index int `json:"index"` + ContentBlock sseTextBlock `json:"content_block"` +} + +type sseTextBlock struct { + Type string `json:"type"` + Text string `json:"text"` +} + +type sseContentBlockStartTool struct { + Type string `json:"type"` + Index int `json:"index"` + ContentBlock sseToolBlock `json:"content_block"` +} + +type sseToolBlock struct { + Type string `json:"type"` + ID string `json:"id"` + Name string `json:"name"` + Input map[string]any `json:"input"` +} + +type sseContentBlockDeltaText struct { + Type string `json:"type"` + Index int `json:"index"` + Delta sseTextDelta `json:"delta"` +} + +type sseTextDelta struct { + Type string `json:"type"` + Text string `json:"text"` +} + +type sseContentBlockDeltaTool struct { + Type string `json:"type"` + Index int `json:"index"` + Delta sseInputJSONDelta `json:"delta"` +} + +type sseInputJSONDelta struct { + Type string `json:"type"` + PartialJSON string `json:"partial_json"` +} + +type sseContentBlockStop struct { + Type string `json:"type"` + Index int `json:"index"` +} + +type sseMessageDelta struct { + Type string `json:"type"` + Delta sseMessageDeltaBody `json:"delta"` + Usage sseOutputUsage `json:"usage"` +} + +type sseMessageDeltaBody struct { + StopReason string `json:"stop_reason"` + StopSequence any `json:"stop_sequence"` +} + +type sseOutputUsage struct { + OutputTokens int `json:"output_tokens"` +} + +type sseMessageStop struct { + Type string `json:"type"` +} + +// openAIStreamToAnthropicState tracks the state for converting OpenAI SSE chunks to Anthropic SSE events. +type openAIStreamToAnthropicState struct { + buffer bytes.Buffer + messageStarted bool // flag indicating emitted message_start + hasOpenBlock bool // flag indicating emitted content_block_start but not content_block_stop + closingEmitted bool // flag indicating emitted content_block_stop + message_delta + message_stop + messageID string + model string + stopReason string // Anthropic stop_reason, mapped from OpenAI finish_reason + inputTokens int + outputTokens int + tokenUsage metrics.TokenUsage + blockIndex int // current Anthropic content block index + activeTools map[int64]*streamToolCall // keyed by OpenAI tool_call index + requestModel string +} + +type streamToolCall struct { + blockIdx int + id string + name string +} + +// processBuffer processes the buffered OpenAI SSE data and emits Anthropic SSE events. +func (s *openAIStreamToAnthropicState) processBuffer(out *[]byte, endOfStream bool) error { + // Loop through all event blocks that are separated by a blank line + for { + eventBlock, remaining, found := bytes.Cut(s.buffer.Bytes(), []byte("\n\n")) + if !found { + break + } + if err := s.processEventBlock(eventBlock, out); err != nil { + return err + } + // Clear buffer and add back remaining SSE data + s.buffer.Reset() + s.buffer.Write(remaining) + } + + // Handle any remaining data at end of stream. + if endOfStream { + if s.buffer.Len() > 0 { + remaining := s.buffer.Bytes() + s.buffer.Reset() + if err := s.processEventBlock(remaining, out); err != nil { + return err + } + } + if !s.closingEmitted { + return s.emitClosingEvents(out) + } + } + return nil +} + +// processEventBlock processes a single SSE event block (data between consecutive \n\n separators). +func (s *openAIStreamToAnthropicState) processEventBlock(block []byte, out *[]byte) error { + var eventData []byte + for line := range bytes.SplitSeq(block, []byte("\n")) { + if after, ok := bytes.CutPrefix(line, sseDataPrefix); ok { + data := bytes.TrimSpace(after) + if len(data) > 0 { + eventData = data + } + } + } + + if len(eventData) == 0 { + return nil + } + + // Skip the [DONE] marker; closing events are emitted on the usage chunk or endOfStream. + if bytes.Equal(eventData, sseDoneMessage) { + return nil + } + + var chunk openai.ChatCompletionResponseChunk + if err := json.Unmarshal(eventData, &chunk); err != nil { + // Skip malformed chunks silently. + return nil + } + + return s.handleChunk(&chunk, out) +} + +// handleChunk converts a single OpenAI ChatCompletionResponseChunk to Anthropic SSE events. +func (s *openAIStreamToAnthropicState) handleChunk(chunk *openai.ChatCompletionResponseChunk, out *[]byte) error { + // Update StreamState's message ID and model with chunk ID and model + if chunk.ID != "" && s.messageID == "" { + s.messageID = chunk.ID + } + if chunk.Model != "" && s.model == "" { + s.model = chunk.Model + } + + // Usage-only chunk (emitted when stream_options.include_usage=true) + // One of the two ways to indicate stream end (other is endOfStream) + if len(chunk.Choices) == 0 && chunk.Usage != nil { + s.inputTokens = chunk.Usage.PromptTokens + s.outputTokens = chunk.Usage.CompletionTokens + s.tokenUsage = metrics.ExtractTokenUsageFromExplicitCaching( + int64(s.inputTokens), + int64(s.outputTokens), + ptr.To(int64(0)), + ptr.To(int64(0)), + ) + return s.emitClosingEvents(out) + } + + if len(chunk.Choices) == 0 { + return nil + } + + // Choose first choice in chunk + choice := &chunk.Choices[0] + delta := choice.Delta + + // Emit message_start on the first meaningful delta. + if !s.messageStarted && delta != nil { + if err := s.emitMessageStart(out); err != nil { + return err + } + } + + if delta != nil { + // Handle text content. + if delta.Content != nil && *delta.Content != "" { + // Emit textblockstart if not started + if !s.hasOpenBlock { + if err := s.emitTextBlockStart(out); err != nil { + return err + } + } + if err := s.emitTextDelta(*delta.Content, out); err != nil { + return err + } + } + + // Handle tool call deltas. + for i := range delta.ToolCalls { + if err := s.handleToolCallDelta(&delta.ToolCalls[i], out); err != nil { + return err + } + } + } + + // Store finish_reason for use in the closing events. + if choice.FinishReason != "" { + s.stopReason = string(openAIFinishReasonToAnthropic(choice.FinishReason)) + } + + return nil +} + +// emitMessageStart emits the Anthropic message_start SSE event. +func (s *openAIStreamToAnthropicState) emitMessageStart(out *[]byte) error { + s.messageStarted = true + payload := sseMessageStart{ + Type: "message_start", + Message: sseMessageBody{ + ID: s.messageID, + Type: "message", + Role: "assistant", + Content: []any{}, + Model: cmp.Or(s.model, s.requestModel), + StopReason: nil, + StopSequence: nil, + // Input tokens are not yet known; they will be reported in message_delta.usage. + Usage: sseMessageUsage{InputTokens: 0, OutputTokens: 0}, + }, + } + data, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("failed to marshal message_start: %w", err) + } + appendAnthropicSSEEvent(out, "message_start", data) + return nil +} + +// emitTextBlockStart emits a content_block_start SSE event for a text content block. +func (s *openAIStreamToAnthropicState) emitTextBlockStart(out *[]byte) error { + s.hasOpenBlock = true + payload := sseContentBlockStartText{ + Type: "content_block_start", + Index: s.blockIndex, + ContentBlock: sseTextBlock{Type: "text", Text: ""}, + } + data, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("failed to marshal content_block_start: %w", err) + } + appendAnthropicSSEEvent(out, "content_block_start", data) + return nil +} + +// emitTextDelta emits a content_block_delta SSE event with text content. +func (s *openAIStreamToAnthropicState) emitTextDelta(text string, out *[]byte) error { + payload := sseContentBlockDeltaText{ + Type: "content_block_delta", + Index: s.blockIndex, + Delta: sseTextDelta{Type: "text_delta", Text: text}, + } + data, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("failed to marshal content_block_delta: %w", err) + } + appendAnthropicSSEEvent(out, "content_block_delta", data) + return nil +} + +// handleToolCallDelta handles an OpenAI tool call delta and emits Anthropic tool_use content block events. +func (s *openAIStreamToAnthropicState) handleToolCallDelta(tc *openai.ChatCompletionChunkChoiceDeltaToolCall, out *[]byte) error { + tool, exists := s.activeTools[tc.Index] + if !exists { + // New tool call: close any open block (e.g., text block) and open a new tool_use block. + if s.hasOpenBlock { + if err := s.emitContentBlockStop(out); err != nil { + return err + } + s.blockIndex++ + } + + id := "" + if tc.ID != nil { + id = *tc.ID + } + tool = &streamToolCall{ + blockIdx: s.blockIndex, + id: id, + name: tc.Function.Name, + } + s.activeTools[tc.Index] = tool + s.hasOpenBlock = true + + // Emit content_block_start for the new tool_use block. + payload := sseContentBlockStartTool{ + Type: "content_block_start", + Index: tool.blockIdx, + ContentBlock: sseToolBlock{ + Type: "tool_use", + ID: id, + Name: tc.Function.Name, + Input: map[string]any{}, + }, + } + data, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("failed to marshal tool content_block_start: %w", err) + } + appendAnthropicSSEEvent(out, "content_block_start", data) + } + + // Emit input_json_delta for accumulated tool arguments. + if tc.Function.Arguments != "" { + payload := sseContentBlockDeltaTool{ + Type: "content_block_delta", + Index: tool.blockIdx, + Delta: sseInputJSONDelta{Type: "input_json_delta", PartialJSON: tc.Function.Arguments}, + } + data, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("failed to marshal input_json_delta: %w", err) + } + appendAnthropicSSEEvent(out, "content_block_delta", data) + } + + return nil +} + +// emitContentBlockStop emits a content_block_stop SSE event for the current block. +func (s *openAIStreamToAnthropicState) emitContentBlockStop(out *[]byte) error { + payload := sseContentBlockStop{Type: "content_block_stop", Index: s.blockIndex} + data, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("failed to marshal content_block_stop: %w", err) + } + appendAnthropicSSEEvent(out, "content_block_stop", data) + return nil +} + +// emitClosingEvents emits content_block_stop (if a block is open), message_delta, and message_stop SSE events. +func (s *openAIStreamToAnthropicState) emitClosingEvents(out *[]byte) error { + if s.closingEmitted { + return nil + } + s.closingEmitted = true + + // Close the currently open content block. + if s.hasOpenBlock { + if err := s.emitContentBlockStop(out); err != nil { + return err + } + s.hasOpenBlock = false + } + + stopReason := s.stopReason + if stopReason == "" { + stopReason = string(anthropic.StopReasonEndTurn) + } + + // Emit message_delta with stop_reason and final output token count. + msgDeltaPayload := sseMessageDelta{ + Type: "message_delta", + Delta: sseMessageDeltaBody{StopReason: stopReason, StopSequence: nil}, + Usage: sseOutputUsage{OutputTokens: s.outputTokens}, + } + data, err := json.Marshal(msgDeltaPayload) + if err != nil { + return fmt.Errorf("failed to marshal message_delta: %w", err) + } + appendAnthropicSSEEvent(out, "message_delta", data) + + // Emit message_stop. + data, err = json.Marshal(sseMessageStop{Type: "message_stop"}) + if err != nil { + return fmt.Errorf("failed to marshal message_stop: %w", err) + } + appendAnthropicSSEEvent(out, "message_stop", data) + + return nil +} + +// appendAnthropicSSEEvent appends a formatted Anthropic SSE event to the output buffer. +func appendAnthropicSSEEvent(buf *[]byte, eventType string, data []byte) { + *buf = append(*buf, "event: "...) + *buf = append(*buf, eventType...) + *buf = append(*buf, '\n') + *buf = append(*buf, "data: "...) + *buf = append(*buf, data...) + *buf = append(*buf, '\n', '\n') +} diff --git a/internal/translator/openai_helper_test.go b/internal/translator/openai_helper_test.go new file mode 100644 index 0000000000..291d2bcafb --- /dev/null +++ b/internal/translator/openai_helper_test.go @@ -0,0 +1,775 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package translator + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/utils/ptr" + + "github.com/envoyproxy/ai-gateway/internal/apischema/anthropic" + "github.com/envoyproxy/ai-gateway/internal/apischema/openai" +) + +// sseEvent holds parsed Anthropic SSE event data. +type sseEvent struct { + eventType string + data string +} + +// parseSSEEventsFromBytes parses raw Anthropic SSE output into individual events. +func parseSSEEventsFromBytes(output []byte) []sseEvent { + var events []sseEvent + for block := range bytes.SplitSeq(output, []byte("\n\n")) { + block = bytes.TrimSpace(block) + if len(block) == 0 { + continue + } + var e sseEvent + for line := range bytes.SplitSeq(block, []byte("\n")) { + if after, ok := bytes.CutPrefix(line, []byte("event: ")); ok { + e.eventType = string(after) + } else if after, ok := bytes.CutPrefix(line, []byte("data: ")); ok { + e.data = string(after) + } + } + if e.eventType != "" || e.data != "" { + events = append(events, e) + } + } + return events +} + +func TestBuildOpenAIChatCompletionRequest(t *testing.T) { + t.Run("basic model and message", func(t *testing.T) { + body := &anthropic.MessagesRequest{ + Model: "claude-3-haiku", + MaxTokens: 100, + Messages: []anthropic.MessageParam{ + {Role: anthropic.MessageRoleUser, Content: anthropic.MessageContent{Text: "Hello"}}, + }, + } + req := buildOpenAIChatCompletionRequest(body, "") + assert.Equal(t, "claude-3-haiku", req.Model) + require.NotNil(t, req.MaxCompletionTokens) + assert.Equal(t, int64(100), *req.MaxCompletionTokens) + require.Len(t, req.Messages, 1) + require.NotNil(t, req.Messages[0].OfUser) + assert.Equal(t, openai.ChatMessageRoleUser, req.Messages[0].OfUser.Role) + assert.Equal(t, "Hello", req.Messages[0].OfUser.Content.Value) + }) + + t.Run("model name override", func(t *testing.T) { + body := &anthropic.MessagesRequest{ + Model: "claude-3-haiku", + Messages: []anthropic.MessageParam{{Role: anthropic.MessageRoleUser, Content: anthropic.MessageContent{Text: "Hi"}}}, + } + req := buildOpenAIChatCompletionRequest(body, "gpt-4o") + assert.Equal(t, "gpt-4o", req.Model) + }) + + t.Run("system prompt prepended as first message", func(t *testing.T) { + body := &anthropic.MessagesRequest{ + Model: "claude-3", + System: &anthropic.SystemPrompt{Text: "You are a helpful assistant."}, + Messages: []anthropic.MessageParam{ + {Role: anthropic.MessageRoleUser, Content: anthropic.MessageContent{Text: "Hi"}}, + }, + } + req := buildOpenAIChatCompletionRequest(body, "") + require.Len(t, req.Messages, 2) + require.NotNil(t, req.Messages[0].OfSystem) + assert.Equal(t, openai.ChatMessageRoleSystem, req.Messages[0].OfSystem.Role) + assert.Equal(t, "You are a helpful assistant.", req.Messages[0].OfSystem.Content.Value) + require.NotNil(t, req.Messages[1].OfUser) + }) + + t.Run("empty system prompt not prepended", func(t *testing.T) { + body := &anthropic.MessagesRequest{ + Model: "claude-3", + System: &anthropic.SystemPrompt{}, + Messages: []anthropic.MessageParam{ + {Role: anthropic.MessageRoleUser, Content: anthropic.MessageContent{Text: "Hi"}}, + }, + } + req := buildOpenAIChatCompletionRequest(body, "") + require.Len(t, req.Messages, 1) + assert.Nil(t, req.Messages[0].OfSystem) + }) + + t.Run("multi-turn conversation", func(t *testing.T) { + body := &anthropic.MessagesRequest{ + Model: "claude-3", + Messages: []anthropic.MessageParam{ + {Role: anthropic.MessageRoleUser, Content: anthropic.MessageContent{Text: "Hi"}}, + {Role: anthropic.MessageRoleAssistant, Content: anthropic.MessageContent{Text: "Hello!"}}, + {Role: anthropic.MessageRoleUser, Content: anthropic.MessageContent{Text: "Bye"}}, + }, + } + req := buildOpenAIChatCompletionRequest(body, "") + require.Len(t, req.Messages, 3) + assert.NotNil(t, req.Messages[0].OfUser) + assert.NotNil(t, req.Messages[1].OfAssistant) + assert.Equal(t, openai.ChatMessageRoleAssistant, req.Messages[1].OfAssistant.Role) + assert.Equal(t, "Hello!", req.Messages[1].OfAssistant.Content.Value) + assert.NotNil(t, req.Messages[2].OfUser) + }) + + t.Run("streaming sets stream_options", func(t *testing.T) { + body := &anthropic.MessagesRequest{ + Model: "claude-3", + Stream: true, + Messages: []anthropic.MessageParam{{Role: anthropic.MessageRoleUser, Content: anthropic.MessageContent{Text: "Hi"}}}, + } + req := buildOpenAIChatCompletionRequest(body, "") + assert.True(t, req.Stream) + require.NotNil(t, req.StreamOptions) + assert.True(t, req.StreamOptions.IncludeUsage) + }) + + t.Run("non-streaming has no stream_options", func(t *testing.T) { + body := &anthropic.MessagesRequest{ + Model: "claude-3", + Messages: []anthropic.MessageParam{{Role: anthropic.MessageRoleUser, Content: anthropic.MessageContent{Text: "Hi"}}}, + } + req := buildOpenAIChatCompletionRequest(body, "") + assert.False(t, req.Stream) + assert.Nil(t, req.StreamOptions) + }) + + t.Run("temperature and topP passthrough", func(t *testing.T) { + temp := 0.7 + topP := 0.95 + body := &anthropic.MessagesRequest{ + Model: "claude-3", + Temperature: &temp, + TopP: &topP, + Messages: []anthropic.MessageParam{{Role: anthropic.MessageRoleUser, Content: anthropic.MessageContent{Text: "Hi"}}}, + } + req := buildOpenAIChatCompletionRequest(body, "") + require.NotNil(t, req.Temperature) + assert.Equal(t, &temp, req.Temperature) + require.NotNil(t, req.TopP) + assert.Equal(t, &topP, req.TopP) + }) + + t.Run("tools conversion", func(t *testing.T) { + body := &anthropic.MessagesRequest{ + Model: "claude-3", + Messages: []anthropic.MessageParam{ + {Role: anthropic.MessageRoleUser, Content: anthropic.MessageContent{Text: "Get weather"}}, + }, + Tools: []anthropic.ToolUnion{ + {Tool: &anthropic.Tool{ + Name: "get_weather", + Description: "Retrieve current weather information", + InputSchema: anthropic.ToolInputSchema{ + Type: "object", + Required: []string{"location"}, + }, + }}, + }, + } + req := buildOpenAIChatCompletionRequest(body, "") + require.Len(t, req.Tools, 1) + assert.Equal(t, openai.ToolTypeFunction, req.Tools[0].Type) + require.NotNil(t, req.Tools[0].Function) + assert.Equal(t, "get_weather", req.Tools[0].Function.Name) + assert.Equal(t, "Retrieve current weather information", req.Tools[0].Function.Description) + }) + + t.Run("no tools means tool choice not set even if body has it", func(t *testing.T) { + tc := anthropic.ToolChoice{Auto: &anthropic.ToolChoiceAuto{Type: "auto"}} + body := &anthropic.MessagesRequest{ + Model: "claude-3", + Messages: []anthropic.MessageParam{{Role: anthropic.MessageRoleUser, Content: anthropic.MessageContent{Text: "Hi"}}}, + ToolChoice: &tc, + } + req := buildOpenAIChatCompletionRequest(body, "") + assert.Nil(t, req.Tools) + assert.Nil(t, req.ToolChoice) + }) + + t.Run("tools with auto tool_choice", func(t *testing.T) { + body := &anthropic.MessagesRequest{ + Model: "claude-3", + Messages: []anthropic.MessageParam{{Role: anthropic.MessageRoleUser, Content: anthropic.MessageContent{Text: "Hi"}}}, + Tools: []anthropic.ToolUnion{{Tool: &anthropic.Tool{Name: "search", InputSchema: anthropic.ToolInputSchema{Type: "object"}}}}, + ToolChoice: ptr.To(anthropic.ToolChoice{Auto: &anthropic.ToolChoiceAuto{Type: "auto"}}), + } + req := buildOpenAIChatCompletionRequest(body, "") + require.NotNil(t, req.Tools) + require.NotNil(t, req.ToolChoice) + assert.Equal(t, string(openai.ToolChoiceTypeAuto), req.ToolChoice.Value) + }) +} + +func TestAnthropicSystemPromptToText(t *testing.T) { + tests := []struct { + name string + system *anthropic.SystemPrompt + expected string + }{ + { + name: "plain text", + system: &anthropic.SystemPrompt{Text: "You are helpful."}, + expected: "You are helpful.", + }, + { + name: "array of text blocks concatenated", + system: &anthropic.SystemPrompt{ + Texts: []anthropic.TextBlockParam{ + {Text: "You are "}, + {Text: "very helpful."}, + }, + }, + expected: "You are very helpful.", + }, + { + name: "empty system prompt", + system: &anthropic.SystemPrompt{}, + expected: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, anthropicSystemPromptToText(tt.system)) + }) + } +} + +func TestAnthropicContentToText(t *testing.T) { + tests := []struct { + name string + content anthropic.MessageContent + expected string + }{ + { + name: "plain text field", + content: anthropic.MessageContent{Text: "Hello!"}, + expected: "Hello!", + }, + { + name: "array of text blocks", + content: anthropic.MessageContent{ + Array: []anthropic.ContentBlockParam{ + {Text: &anthropic.TextBlockParam{Text: "Hello "}}, + {Text: &anthropic.TextBlockParam{Text: "world"}}, + }, + }, + expected: "Hello world", + }, + { + name: "array with nil text block is skipped", + content: anthropic.MessageContent{ + Array: []anthropic.ContentBlockParam{ + {Text: nil}, + {Text: &anthropic.TextBlockParam{Text: "world"}}, + }, + }, + expected: "world", + }, + { + name: "empty content", + content: anthropic.MessageContent{}, + expected: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, anthropicContentToText(tt.content)) + }) + } +} + +func TestAnthropicToolsToOpenAI(t *testing.T) { + t.Run("nil tools returns nil", func(t *testing.T) { + assert.Nil(t, anthropicToolsToOpenAI(nil)) + }) + + t.Run("empty tools returns nil", func(t *testing.T) { + assert.Nil(t, anthropicToolsToOpenAI([]anthropic.ToolUnion{})) + }) + + t.Run("single tool converted correctly", func(t *testing.T) { + tools := []anthropic.ToolUnion{ + {Tool: &anthropic.Tool{ + Name: "search", + Description: "Search the web", + InputSchema: anthropic.ToolInputSchema{ + Type: "object", + Properties: map[string]any{"query": map[string]any{"type": "string"}}, + Required: []string{"query"}, + }, + }}, + } + result := anthropicToolsToOpenAI(tools) + require.Len(t, result, 1) + assert.Equal(t, openai.ToolTypeFunction, result[0].Type) + require.NotNil(t, result[0].Function) + assert.Equal(t, "search", result[0].Function.Name) + assert.Equal(t, "Search the web", result[0].Function.Description) + }) + + t.Run("multiple tools all converted", func(t *testing.T) { + tools := []anthropic.ToolUnion{ + {Tool: &anthropic.Tool{Name: "tool1", InputSchema: anthropic.ToolInputSchema{Type: "object"}}}, + {Tool: &anthropic.Tool{Name: "tool2", InputSchema: anthropic.ToolInputSchema{Type: "object"}}}, + } + result := anthropicToolsToOpenAI(tools) + require.Len(t, result, 2) + assert.Equal(t, "tool1", result[0].Function.Name) + assert.Equal(t, "tool2", result[1].Function.Name) + }) +} + +func TestAnthropicToolChoiceToOpenAI(t *testing.T) { + tests := []struct { + name string + tc anthropic.ToolChoice + hasTools bool + expectNil bool + expectVal any + }{ + { + name: "zero value tool choice returns nil", + tc: anthropic.ToolChoice{}, + hasTools: true, + expectNil: true, + }, + { + name: "no tools returns nil even with valid choice", + tc: anthropic.ToolChoice{Auto: &anthropic.ToolChoiceAuto{Type: "auto"}}, + hasTools: false, + expectNil: true, + }, + { + name: "auto maps to auto", + tc: anthropic.ToolChoice{Auto: &anthropic.ToolChoiceAuto{Type: "auto"}}, + hasTools: true, + expectVal: string(openai.ToolChoiceTypeAuto), + }, + { + name: "none maps to none", + tc: anthropic.ToolChoice{None: &anthropic.ToolChoiceNone{Type: "none"}}, + hasTools: true, + expectVal: string(openai.ToolChoiceTypeNone), + }, + { + name: "any maps to required", + tc: anthropic.ToolChoice{Any: &anthropic.ToolChoiceAny{Type: "any"}}, + hasTools: true, + expectVal: string(openai.ToolChoiceTypeRequired), + }, + { + name: "tool type with name maps to named tool choice", + tc: anthropic.ToolChoice{Tool: &anthropic.ToolChoiceTool{Type: "tool", Name: "search"}}, + hasTools: true, + expectVal: openai.ChatCompletionNamedToolChoice{ + Type: openai.ToolTypeFunction, + Function: openai.ChatCompletionNamedToolChoiceFunction{Name: "search"}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := anthropicToolChoiceToOpenAI(tt.tc, tt.hasTools) + if tt.expectNil { + assert.Nil(t, result) + return + } + require.NotNil(t, result) + assert.Equal(t, tt.expectVal, result.Value) + }) + } +} + +func TestOpenAIResponseToAnthropic(t *testing.T) { + t.Run("text content response", func(t *testing.T) { + content := "Hello from the model!" + resp := &openai.ChatCompletionResponse{ + ID: "chatcmpl-123", + Model: "gpt-4o", + Choices: []openai.ChatCompletionResponseChoice{ + { + FinishReason: openai.ChatCompletionChoicesFinishReasonStop, + Message: openai.ChatCompletionResponseChoiceMessage{ + Content: &content, + Role: "assistant", + }, + }, + }, + Usage: openai.Usage{PromptTokens: 10, CompletionTokens: 20}, + } + result := openAIResponseToAnthropic(resp, "gpt-4o") + assert.Equal(t, "chatcmpl-123", result.ID) + assert.Equal(t, "gpt-4o", result.Model) + require.Len(t, result.Content, 1) + require.NotNil(t, result.Content[0].Text) + assert.Equal(t, "Hello from the model!", result.Content[0].Text.Text) + require.NotNil(t, result.StopReason) + assert.Equal(t, anthropic.StopReasonEndTurn, *result.StopReason) + require.NotNil(t, result.Usage) + assert.Equal(t, float64(10), result.Usage.InputTokens) + assert.Equal(t, float64(20), result.Usage.OutputTokens) + }) + + t.Run("empty string content not added to blocks", func(t *testing.T) { + empty := "" + resp := &openai.ChatCompletionResponse{ + Choices: []openai.ChatCompletionResponseChoice{ + { + FinishReason: openai.ChatCompletionChoicesFinishReasonStop, + Message: openai.ChatCompletionResponseChoiceMessage{Content: &empty}, + }, + }, + } + result := openAIResponseToAnthropic(resp, "test-model") + assert.Nil(t, result.Content) + }) + + t.Run("nil content not added to blocks", func(t *testing.T) { + resp := &openai.ChatCompletionResponse{ + Choices: []openai.ChatCompletionResponseChoice{ + { + FinishReason: openai.ChatCompletionChoicesFinishReasonStop, + Message: openai.ChatCompletionResponseChoiceMessage{Content: nil}, + }, + }, + } + result := openAIResponseToAnthropic(resp, "test-model") + assert.Nil(t, result.Content) + }) + + t.Run("tool call response", func(t *testing.T) { + callID := "call-abc" + resp := &openai.ChatCompletionResponse{ + Choices: []openai.ChatCompletionResponseChoice{ + { + FinishReason: openai.ChatCompletionChoicesFinishReasonToolCalls, + Message: openai.ChatCompletionResponseChoiceMessage{ + ToolCalls: []openai.ChatCompletionMessageToolCallParam{ + { + ID: &callID, + Function: openai.ChatCompletionMessageToolCallFunctionParam{ + Name: "get_weather", + Arguments: `{"location":"NYC"}`, + }, + }, + }, + }, + }, + }, + } + result := openAIResponseToAnthropic(resp, "test-model") + require.Len(t, result.Content, 1) + require.NotNil(t, result.Content[0].Tool) + assert.Equal(t, "call-abc", result.Content[0].Tool.ID) + assert.Equal(t, "get_weather", result.Content[0].Tool.Name) + assert.Equal(t, map[string]any{"location": "NYC"}, result.Content[0].Tool.Input) + require.NotNil(t, result.StopReason) + assert.Equal(t, anthropic.StopReasonToolUse, *result.StopReason) + }) + + t.Run("malformed tool call arguments becomes empty map", func(t *testing.T) { + resp := &openai.ChatCompletionResponse{ + Choices: []openai.ChatCompletionResponseChoice{ + { + FinishReason: openai.ChatCompletionChoicesFinishReasonToolCalls, + Message: openai.ChatCompletionResponseChoiceMessage{ + ToolCalls: []openai.ChatCompletionMessageToolCallParam{ + { + Function: openai.ChatCompletionMessageToolCallFunctionParam{ + Name: "broken_tool", + Arguments: `{invalid json`, + }, + }, + }, + }, + }, + }, + } + result := openAIResponseToAnthropic(resp, "test-model") + require.Len(t, result.Content, 1) + require.NotNil(t, result.Content[0].Tool) + assert.Equal(t, map[string]any{}, result.Content[0].Tool.Input) + }) + + t.Run("missing tool call ID becomes empty string", func(t *testing.T) { + resp := &openai.ChatCompletionResponse{ + Choices: []openai.ChatCompletionResponseChoice{ + { + FinishReason: openai.ChatCompletionChoicesFinishReasonToolCalls, + Message: openai.ChatCompletionResponseChoiceMessage{ + ToolCalls: []openai.ChatCompletionMessageToolCallParam{ + { + ID: nil, + Function: openai.ChatCompletionMessageToolCallFunctionParam{ + Name: "my_tool", + Arguments: `{}`, + }, + }, + }, + }, + }, + }, + } + result := openAIResponseToAnthropic(resp, "test-model") + require.Len(t, result.Content, 1) + assert.Empty(t, result.Content[0].Tool.ID) + }) + + t.Run("no choices produces no content and no stop reason", func(t *testing.T) { + resp := &openai.ChatCompletionResponse{ + ID: "chatcmpl-empty", + Model: "gpt-4o", + Usage: openai.Usage{PromptTokens: 5}, + } + result := openAIResponseToAnthropic(resp, "gpt-4o") + assert.Nil(t, result.Content) + assert.Nil(t, result.StopReason) + assert.Equal(t, float64(5), result.Usage.InputTokens) + }) +} + +func TestOpenAIFinishReasonToAnthropic(t *testing.T) { + tests := []struct { + reason openai.ChatCompletionChoicesFinishReason + expected anthropic.StopReason + }{ + {openai.ChatCompletionChoicesFinishReasonStop, anthropic.StopReasonEndTurn}, + {openai.ChatCompletionChoicesFinishReasonLength, anthropic.StopReasonMaxTokens}, + {openai.ChatCompletionChoicesFinishReasonToolCalls, anthropic.StopReasonToolUse}, + {openai.ChatCompletionChoicesFinishReasonContentFilter, anthropic.StopReasonRefusal}, + {"function_call", anthropic.StopReasonEndTurn}, + {"", anthropic.StopReasonEndTurn}, + } + for _, tt := range tests { + t.Run(string(tt.reason), func(t *testing.T) { + assert.Equal(t, tt.expected, openAIFinishReasonToAnthropic(tt.reason)) + }) + } +} + +func TestOpenAIStreamToAnthropicState_ProcessBuffer_TextStreaming(t *testing.T) { + state := &openAIStreamToAnthropicState{ + activeTools: make(map[int64]*streamToolCall), + requestModel: "claude-3", + } + + // OpenAI SSE input: text streaming with usage chunk and [DONE]. + // Chunk 1: first delta with text → emits message_start, content_block_start, content_block_delta + // Chunk 2: finish_reason → stores stop reason + // Chunk 3: usage-only → emits content_block_stop, message_delta, message_stop + // [DONE]: skipped + input := "data: {\"id\":\"chatcmpl-abc\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"Hello\"}}],\"model\":\"gpt-4o\"}\n\n" + + "data: {\"id\":\"chatcmpl-abc\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" world\"},\"finish_reason\":null}]}\n\n" + + "data: {\"id\":\"chatcmpl-abc\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n" + + "data: {\"id\":\"chatcmpl-abc\",\"choices\":[],\"usage\":{\"prompt_tokens\":10,\"completion_tokens\":5}}\n\n" + + "data: [DONE]\n\n" + + state.buffer.WriteString(input) + + var out []byte + err := state.processBuffer(&out, true) + require.NoError(t, err) + + events := parseSSEEventsFromBytes(out) + // 7 events: message_start, content_block_start, content_block_delta x2, content_block_stop, message_delta, message_stop + require.Len(t, events, 7) + + assert.Equal(t, "message_start", events[0].eventType) + require.JSONEq(t, `{ + "type":"message_start", + "message":{ + "id":"chatcmpl-abc", + "type":"message", + "role":"assistant", + "content":[], + "model":"gpt-4o", + "stop_reason":null, + "stop_sequence":null, + "usage":{"input_tokens":0,"output_tokens":0} + } + }`, events[0].data) + + assert.Equal(t, "content_block_start", events[1].eventType) + require.JSONEq(t, `{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}`, events[1].data) + + assert.Equal(t, "content_block_delta", events[2].eventType) + require.JSONEq(t, `{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hello"}}`, events[2].data) + + assert.Equal(t, "content_block_delta", events[3].eventType) + require.JSONEq(t, `{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" world"}}`, events[3].data) + + assert.Equal(t, "content_block_stop", events[4].eventType) + require.JSONEq(t, `{"type":"content_block_stop","index":0}`, events[4].data) + + assert.Equal(t, "message_delta", events[5].eventType) + require.JSONEq(t, `{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":5}}`, events[5].data) + + assert.Equal(t, "message_stop", events[6].eventType) + require.JSONEq(t, `{"type":"message_stop"}`, events[6].data) +} + +func TestOpenAIStreamToAnthropicState_ProcessBuffer_ToolCallStreaming(t *testing.T) { + state := &openAIStreamToAnthropicState{ + activeTools: make(map[int64]*streamToolCall), + requestModel: "claude-3", + } + + // Chunk 1: new tool call start → emits message_start, content_block_start (tool_use) + // Chunk 2: tool argument delta → emits content_block_delta (input_json_delta) + // Chunk 3: finish_reason=tool_calls → stores stop reason + // Chunk 4: usage → emits content_block_stop, message_delta (stop_reason=tool_use), message_stop + input := "data: {\"id\":\"chatcmpl-def\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"id\":\"call-xyz\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"\"},\"type\":\"function\"}]}}],\"model\":\"gpt-4o\"}\n\n" + + "data: {\"id\":\"chatcmpl-def\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":null,\"function\":{\"name\":\"\",\"arguments\":\"{\\\"city\\\":\\\"NYC\\\"}\"}}]}}]}\n\n" + + "data: {\"id\":\"chatcmpl-def\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\n" + + "data: {\"id\":\"chatcmpl-def\",\"choices\":[],\"usage\":{\"prompt_tokens\":15,\"completion_tokens\":10}}\n\n" + + "data: [DONE]\n\n" + + state.buffer.WriteString(input) + + var out []byte + err := state.processBuffer(&out, true) + require.NoError(t, err) + + events := parseSSEEventsFromBytes(out) + // 6 events: message_start, content_block_start, content_block_delta, content_block_stop, message_delta, message_stop + require.Len(t, events, 6) + + assert.Equal(t, "message_start", events[0].eventType) + + assert.Equal(t, "content_block_start", events[1].eventType) + require.JSONEq(t, `{"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"call-xyz","name":"get_weather","input":{}}}`, events[1].data) + + assert.Equal(t, "content_block_delta", events[2].eventType) + require.JSONEq(t, `{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"city\":\"NYC\"}"}}`, events[2].data) + + assert.Equal(t, "content_block_stop", events[3].eventType) + require.JSONEq(t, `{"type":"content_block_stop","index":0}`, events[3].data) + + assert.Equal(t, "message_delta", events[4].eventType) + require.JSONEq(t, `{"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":10}}`, events[4].data) + + assert.Equal(t, "message_stop", events[5].eventType) + require.JSONEq(t, `{"type":"message_stop"}`, events[5].data) +} + +func TestOpenAIStreamToAnthropicState_ProcessBuffer_EndOfStreamClosing(t *testing.T) { + // Verify endOfStream triggers closing events when no usage chunk is present. + state := &openAIStreamToAnthropicState{ + activeTools: make(map[int64]*streamToolCall), + requestModel: "test-model", + } + + input := "data: {\"id\":\"chatcmpl-eos\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"Hi\"},\"finish_reason\":\"stop\"}]}\n\n" + + "data: [DONE]\n\n" + + state.buffer.WriteString(input) + + var out []byte + err := state.processBuffer(&out, true) + require.NoError(t, err) + + events := parseSSEEventsFromBytes(out) + require.GreaterOrEqual(t, len(events), 4) + + // Last event must be message_stop. + assert.Equal(t, "message_stop", events[len(events)-1].eventType) + + // Find message_delta and verify stop_reason defaults to end_turn. + var msgDeltaData string + for _, e := range events { + if e.eventType == "message_delta" { + msgDeltaData = e.data + break + } + } + require.NotEmpty(t, msgDeltaData) + require.JSONEq(t, `{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":0}}`, msgDeltaData) +} + +func TestOpenAIStreamToAnthropicState_ProcessBuffer_EmptyInput(t *testing.T) { + state := &openAIStreamToAnthropicState{ + activeTools: make(map[int64]*streamToolCall), + requestModel: "test-model", + } + + var out []byte + err := state.processBuffer(&out, false) + require.NoError(t, err) + assert.Empty(t, out) +} + +func TestOpenAIStreamToAnthropicState_ProcessBuffer_SkipsDoneMarker(t *testing.T) { + // Ensure [DONE] marker does not cause errors or spurious events. + state := &openAIStreamToAnthropicState{ + activeTools: make(map[int64]*streamToolCall), + requestModel: "test-model", + } + + input := "data: [DONE]\n\n" + state.buffer.WriteString(input) + + var out []byte + err := state.processBuffer(&out, false) + require.NoError(t, err) + // No events should be emitted for just [DONE]. + assert.Empty(t, out) +} + +func TestOpenAIStreamToAnthropicState_ProcessBuffer_MalformedChunkSkipped(t *testing.T) { + state := &openAIStreamToAnthropicState{ + activeTools: make(map[int64]*streamToolCall), + requestModel: "test-model", + } + + // A malformed JSON chunk should be silently skipped, no error. + input := "data: {not valid json}\n\n" + state.buffer.WriteString(input) + + var out []byte + err := state.processBuffer(&out, false) + require.NoError(t, err) +} + +func TestOpenAIStreamToAnthropicState_handleToolCallDelta_OpenBlock(t *testing.T) { + state := &openAIStreamToAnthropicState{ + activeTools: make(map[int64]*streamToolCall), + hasOpenBlock: true, + blockIndex: 0, + } + toolID := "test_id" + toolCall := &openai.ChatCompletionChunkChoiceDeltaToolCall{ + Index: 5, + ID: &toolID, + Function: openai.ChatCompletionMessageToolCallFunctionParam{ + Name: "test", + Arguments: "test_args", + }, + } + + var out []byte + err := state.handleToolCallDelta(toolCall, &out) + require.NoError(t, err) + require.Equal(t, 1, state.blockIndex) +} + +func TestOpenAIStreamToAnthropicState_handleChunk_ZeroLen(t *testing.T) { + state := &openAIStreamToAnthropicState{} + + chunk := &openai.ChatCompletionResponseChunk{ + Choices: []openai.ChatCompletionResponseChunkChoice{}, + Usage: nil, + } + var out []byte + err := state.handleChunk(chunk, &out) + require.NoError(t, err) +} diff --git a/tests/data-plane/testupstream_test.go b/tests/data-plane/testupstream_test.go index ebc5aadab0..a04219a4e9 100644 --- a/tests/data-plane/testupstream_test.go +++ b/tests/data-plane/testupstream_test.go @@ -1275,6 +1275,128 @@ event: response.completed data: {"type":"response.completed","sequence_number":10,"response":{"id":"resp_67c","object":"response","created_at":1741290958,"status":"completed","error":null,"incomplete_details":null,"instructions":"You are a helpful assistant.","max_output_tokens":null,"model":"gpt-4.1-2025-04-14","output":[{"id":"msg_67c","type":"message","status":"completed","role":"assistant","content":[{"type":"output_text","text":"This is a test.","annotations":[]}]}],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"summary":null},"store":true,"temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[],"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":37,"output_tokens":11,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":48},"user":null,"metadata":{}}} `, }, + { + name: "anthropic-openai - /anthropic/v1/messages - OpenAI Backend with Anthropic messages endpoint", + backend: "openai", + path: "/anthropic/v1/messages", + method: http.MethodPost, + requestBody: `{"model": "foo","max_tokens": 1000, "messages": [{"role": "user", "content": "say hi"}]}`, + expRequestBody: `{"messages":[{"content":"say hi","role":"user"}],"model":"foo","max_completion_tokens":1000}`, + expPath: "/v1/chat/completions", + responseBody: `{"choices":[{"message":{"content":"hi, this is a test."}}]}`, + expStatus: http.StatusOK, + expResponseBody: `{"id":"","type":"message","role":"assistant","content":[{"type":"text","text":"hi, this is a test."}],"model":"foo","stop_reason":"end_turn","usage":{"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"input_tokens":0,"output_tokens":0}}`, + }, + { + name: "anthropic-openai - /anthropic/v1/messages - non-streaming with system prompt", + backend: "openai", + path: "/anthropic/v1/messages", + method: http.MethodPost, + requestBody: `{"model":"claude-test","max_tokens":100,"system":"You are a helpful assistant.","messages":[{"role":"user","content":"Hello"}]}`, + expRequestBody: `{"messages":[{"content":"You are a helpful assistant.","role":"system"},{"content":"Hello","role":"user"}],"model":"claude-test","max_completion_tokens":100}`, + expPath: "/v1/chat/completions", + responseBody: `{"id":"chatcmpl-sys","model":"gpt-4o","choices":[{"message":{"content":"Hello! How can I help you today?","role":"assistant"},"finish_reason":"stop"}],"usage":{"prompt_tokens":25,"completion_tokens":10}}`, + expStatus: http.StatusOK, + expResponseBody: `{"id":"chatcmpl-sys","type":"message","role":"assistant","content":[{"type":"text","text":"Hello! How can I help you today?"}],"model":"gpt-4o","stop_reason":"end_turn","usage":{"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"input_tokens":25,"output_tokens":10}}`, + }, + { + name: "anthropic-openai - /anthropic/v1/messages - non-streaming tool call", + backend: "openai", + path: "/anthropic/v1/messages", + method: http.MethodPost, + requestBody: `{"model":"claude-test","max_tokens":200,"messages":[{"role":"user","content":"What's the weather in Paris?"}],"tools":[{"type":"custom","name":"get_weather","description":"Get weather info","input_schema":{"type":"object","properties":{"location":{"type":"string"}},"required":["location"]}}]}`, + expRequestBody: `{"messages":[{"content":"What's the weather in Paris?","role":"user"}],"model":"claude-test","max_completion_tokens":200,"tools":[{"type":"function","function":{"name":"get_weather","description":"Get weather info","parameters":{"type":"object","properties":{"location":{"type":"string"}},"required":["location"]}}}]}`, + expPath: "/v1/chat/completions", + responseBody: `{"id":"chatcmpl-tool","model":"gpt-4o","choices":[{"message":{"role":"assistant","tool_calls":[{"id":"call_abc","type":"function","function":{"name":"get_weather","arguments":"{\"location\":\"Paris\"}"}}]},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":50,"completion_tokens":15}}`, + expStatus: http.StatusOK, + expResponseBody: `{"id":"chatcmpl-tool","type":"message","role":"assistant","content":[{"type":"tool_use","id":"call_abc","name":"get_weather","input":{"location":"Paris"}}],"model":"gpt-4o","stop_reason":"tool_use","usage":{"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"input_tokens":50,"output_tokens":15}}`, + }, + { + name: "anthropic-openai - /anthropic/v1/messages - streaming text response", + backend: "openai", + path: "/anthropic/v1/messages", + method: http.MethodPost, + responseType: "sse", + requestBody: `{"model":"claude-test","max_tokens":100,"messages":[{"role":"user","content":"Say hi"}],"stream":true}`, + expRequestBody: `{"messages":[{"content":"Say hi","role":"user"}],"model":"claude-test","max_completion_tokens":100,"stream":true,"stream_options":{"include_usage":true}}`, + expPath: "/v1/chat/completions", + responseBody: `{"id":"chatcmpl-stream","model":"gpt-4o","choices":[{"index":0,"delta":{"role":"assistant","content":"Hi"},"finish_reason":null}],"usage":null} +{"id":"chatcmpl-stream","model":"gpt-4o","choices":[{"index":0,"delta":{"content":" there!"},"finish_reason":null}],"usage":null} +{"id":"chatcmpl-stream","model":"gpt-4o","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":null} +{"id":"chatcmpl-stream","model":"gpt-4o","choices":[],"usage":{"prompt_tokens":10,"completion_tokens":3}} +[DONE]`, + expStatus: http.StatusOK, + expResponseBody: `event: message_start +data: {"type":"message_start","message":{"id":"chatcmpl-stream","type":"message","role":"assistant","content":[],"model":"gpt-4o","stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":0,"output_tokens":0}}} + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hi"}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" there!"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":0} + +event: message_delta +data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":3}} + +event: message_stop +data: {"type":"message_stop"}`, + }, + { + name: "anthropic-openai - /anthropic/v1/messages - streaming tool call", + backend: "openai", + path: "/anthropic/v1/messages", + method: http.MethodPost, + responseType: "sse", + requestBody: `{"model":"claude-test","max_tokens":100,"messages":[{"role":"user","content":"Get the weather in Paris"}],"stream":true,"tools":[{"type":"custom","name":"get_weather","description":"Get weather info","input_schema":{"type":"object","properties":{"location":{"type":"string"}},"required":["location"]}}]}`, + expRequestBody: `{"messages":[{"content":"Get the weather in Paris","role":"user"}],"model":"claude-test","max_completion_tokens":100,"stream":true,"stream_options":{"include_usage":true},"tools":[{"type":"function","function":{"name":"get_weather","description":"Get weather info","parameters":{"type":"object","properties":{"location":{"type":"string"}},"required":["location"]}}}]}`, + expPath: "/v1/chat/completions", + responseBody: `{"id":"chatcmpl-tool","model":"gpt-4o","choices":[{"index":0,"delta":{"role":"assistant","tool_calls":[{"index":0,"id":"call_abc","type":"function","function":{"name":"get_weather","arguments":""}}]},"finish_reason":null}],"usage":null} +{"id":"chatcmpl-tool","model":"gpt-4o","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"location\":"}}]},"finish_reason":null}],"usage":null} +{"id":"chatcmpl-tool","model":"gpt-4o","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"Paris\"}"}}]},"finish_reason":null}],"usage":null} +{"id":"chatcmpl-tool","model":"gpt-4o","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}],"usage":null} +{"id":"chatcmpl-tool","model":"gpt-4o","choices":[],"usage":{"prompt_tokens":50,"completion_tokens":15}} +[DONE]`, + expStatus: http.StatusOK, + expResponseBody: `event: message_start +data: {"type":"message_start","message":{"id":"chatcmpl-tool","type":"message","role":"assistant","content":[],"model":"gpt-4o","stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":0,"output_tokens":0}}} + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"call_abc","name":"get_weather","input":{}}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"location\":"}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"\"Paris\"}"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":0} + +event: message_delta +data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":15}} + +event: message_stop +data: {"type":"message_stop"}`, + }, + { + name: "anthropic-openai - /anthropic/v1/messages - OpenAI JSON error translated to Anthropic error", + backend: "openai", + path: "/anthropic/v1/messages", + method: http.MethodPost, + requestBody: `{"model":"claude-test","max_tokens":100,"messages":[{"role":"user","content":"Hello"}]}`, + expPath: "/v1/chat/completions", + expRequestBody: `{"messages":[{"content":"Hello","role":"user"}],"model":"claude-test","max_completion_tokens":100}`, + responseBody: `{"error":{"type":"invalid_request_error","message":"Model not found","code":"model_not_found"}}`, + responseStatus: "400", + expStatus: http.StatusBadRequest, + expResponseBody: `{"error":{"message":"Model not found","type":"invalid_request_error"},"request_id":"","type":"error"}`, + }, } { t.Run(tc.name, func(t *testing.T) { listenerAddress := fmt.Sprintf("http://localhost:%d", listenerPort)