diff --git a/core/changelog.md b/core/changelog.md index c632c334df..4e629bdacc 100644 --- a/core/changelog.md +++ b/core/changelog.md @@ -7,3 +7,4 @@ - fix: capture responses streaming API errors - fix: delete fallbacks from outgoing Anthropic requests - feat: claude-opus-4-7 compatibility +- fix: token usage for vllm \ No newline at end of file diff --git a/core/providers/openai/openai.go b/core/providers/openai/openai.go index 6309445c0f..41eee7b9a9 100644 --- a/core/providers/openai/openai.go +++ b/core/providers/openai/openai.go @@ -1319,7 +1319,7 @@ func HandleOpenAIChatCompletionStreaming( // Handle regular content chunks, including reasoning if choice.ChatStreamResponseChoice != nil && choice.ChatStreamResponseChoice.Delta != nil && - (choice.ChatStreamResponseChoice.Delta.Content != nil || + ((choice.ChatStreamResponseChoice.Delta.Content != nil && *choice.ChatStreamResponseChoice.Delta.Content != "") || choice.ChatStreamResponseChoice.Delta.Reasoning != nil || len(choice.ChatStreamResponseChoice.Delta.ReasoningDetails) > 0 || choice.ChatStreamResponseChoice.Delta.Audio != nil || @@ -1866,7 +1866,6 @@ func HandleOpenAIResponsesStreaming( providerUtils.ProcessAndSendResponse(ctx, postHookRunner, providerUtils.GetBifrostResponseForStreamResponse(nil, nil, &response, nil, nil, nil), responseChan) } } - }() return responseChan, nil @@ -2433,7 +2432,6 @@ func HandleOpenAISpeechStreamRequest( providerUtils.ProcessAndSendResponse(ctx, postHookRunner, providerUtils.GetBifrostResponseForStreamResponse(nil, nil, nil, &response, nil, nil), responseChan) } - }() return responseChan, nil @@ -2591,7 +2589,7 @@ func HandleOpenAITranscriptionRequest( return nil, providerUtils.NewBifrostOperationError(schemas.ErrProviderResponseUnmarshal, err, providerName) } - //TODO: add HandleProviderResponse here + // TODO: add HandleProviderResponse here // Parse raw response for RawResponse field if sendBackRawResponse { @@ -2895,7 +2893,6 @@ func HandleOpenAITranscriptionStreamRequest( providerUtils.ProcessAndSendResponse(ctx, postHookRunner, providerUtils.GetBifrostResponseForStreamResponse(nil, nil, nil, nil, response, nil), responseChan) } - }() return responseChan, nil @@ -2905,8 +2902,8 @@ func HandleOpenAITranscriptionStreamRequest( // It formats the request, sends it to OpenAI, and processes the response. // Returns a BifrostResponse containing the bifrost response or an error if the request fails. func (provider *OpenAIProvider) ImageGeneration(ctx *schemas.BifrostContext, key schemas.Key, - req *schemas.BifrostImageGenerationRequest) (*schemas.BifrostImageGenerationResponse, *schemas.BifrostError) { - + req *schemas.BifrostImageGenerationRequest, +) (*schemas.BifrostImageGenerationResponse, *schemas.BifrostError) { if err := providerUtils.CheckOperationAllowed(schemas.OpenAI, provider.customProviderConfig, schemas.ImageGenerationRequest); err != nil { return nil, err } @@ -2939,7 +2936,6 @@ func HandleOpenAIImageGenerationRequest( sendBackRawResponse bool, logger schemas.Logger, ) (*schemas.BifrostImageGenerationResponse, *schemas.BifrostError) { - // Create request req := fasthttp.AcquireRequest() resp := fasthttp.AcquireResponse() @@ -3060,7 +3056,6 @@ func (provider *OpenAIProvider) ImageGenerationStream( key schemas.Key, request *schemas.BifrostImageGenerationRequest, ) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { - if request == nil { return nil, providerUtils.NewBifrostOperationError("invalid request: nil", nil, provider.GetProviderKey()) } @@ -3109,7 +3104,6 @@ func HandleOpenAIImageGenerationStreaming( postResponseConverter func(*schemas.BifrostImageGenerationStreamResponse) *schemas.BifrostImageGenerationStreamResponse, logger schemas.Logger, ) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { - // Set headers headers := map[string]string{ "Content-Type": "application/json", @@ -3471,7 +3465,6 @@ func HandleOpenAIImageGenerationStreaming( return } } - }() return responseChan, nil @@ -4385,7 +4378,6 @@ func HandleOpenAIImageEditStreamRequest( postResponseConverter func(*schemas.BifrostImageGenerationStreamResponse) *schemas.BifrostImageGenerationStreamResponse, logger schemas.Logger, ) (chan *schemas.BifrostStreamChunk, *schemas.BifrostError) { - reqBody := ToOpenAIImageEditRequest(request) if reqBody == nil { return nil, providerUtils.NewBifrostOperationError("image edit input is not provided", nil, providerName) @@ -4731,7 +4723,6 @@ func HandleOpenAIImageEditStreamRequest( return } } - }() return responseChan, nil