Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 112 additions & 2 deletions core/providers/openai/openai.go
Original file line number Diff line number Diff line change
Expand Up @@ -3541,9 +3541,119 @@ func HandleOpenAIImageGenerationStreaming(
return responseChan, nil
}

// Rerank is not supported by the OpenAI provider.
// Rerank performs a rerank request for custom OpenAI-compatible providers.
func (provider *OpenAIProvider) Rerank(ctx *schemas.BifrostContext, key schemas.Key, request *schemas.BifrostRerankRequest) (*schemas.BifrostRerankResponse, *schemas.BifrostError) {
return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, provider.GetProviderKey())
if provider.customProviderConfig == nil {
return nil, providerUtils.NewUnsupportedOperationError(schemas.RerankRequest, schemas.OpenAI)
}
if err := providerUtils.CheckOperationAllowed(schemas.OpenAI, provider.customProviderConfig, schemas.RerankRequest); err != nil {
return nil, err
}

return HandleOpenAIRerankRequest(
ctx,
provider.client,
provider.buildRequestURL(ctx, "/v1/rerank", schemas.RerankRequest),
request,
key,
provider.networkConfig.ExtraHeaders,
provider.GetProviderKey(),
providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest),
providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse),
provider.logger,
)
}

// HandleOpenAIRerankRequest handles rerank requests for custom OpenAI-compatible APIs.
func HandleOpenAIRerankRequest(
ctx *schemas.BifrostContext,
client *fasthttp.Client,
url string,
request *schemas.BifrostRerankRequest,
key schemas.Key,
extraHeaders map[string]string,
providerName schemas.ModelProvider,
sendBackRawRequest bool,
sendBackRawResponse bool,
logger schemas.Logger,
) (*schemas.BifrostRerankResponse, *schemas.BifrostError) {
req := fasthttp.AcquireRequest()
resp := fasthttp.AcquireResponse()
defer fasthttp.ReleaseRequest(req)
respOwned := true
defer func() {
if respOwned {
fasthttp.ReleaseResponse(resp)
}
}()
// Rerank JSON is always parsed in-process (no transport streaming). Skip
// PrepareResponseStreaming so large-response threshold mode never leaves the body
// on a stream-only path that finalizeOpenAIResponse would treat as unsupported here.
activeClient := client

providerUtils.SetExtraHeaders(ctx, req, extraHeaders, nil)
req.SetRequestURI(url)
req.Header.SetMethod(http.MethodPost)
req.Header.SetContentType("application/json")
for k, v := range BearerAuthHeader(key) {
req.Header.Set(k, v)
}

jsonData, bifrostErr := providerUtils.CheckContextAndGetRequestBody(
ctx,
request,
func() (providerUtils.RequestBodyWithExtraParams, error) {
return ToOpenAIRerankRequest(request), nil
},
)
if bifrostErr != nil {
return nil, bifrostErr
}
// CheckContextAndGetRequestBody returns nil jsonData when large-payload passthrough
// staged the request body as a stream. Mirror the other OpenAI-compatible handlers and
// apply that stream instead of sending an empty body; the normal JSON path is unchanged.
if !providerUtils.ApplyLargePayloadRequestBodyWithModelNormalization(ctx, req, providerName) {
req.SetBody(jsonData)
}

latency, bifrostErr, wait := providerUtils.MakeRequestWithContext(ctx, activeClient, req, resp)
defer wait()
if bifrostErr != nil {
return nil, providerUtils.EnrichError(ctx, bifrostErr, jsonData, nil, sendBackRawRequest, sendBackRawResponse, latency)
}
providerResponseHeaders := providerUtils.ExtractProviderResponseHeaders(resp)
ctx.SetValue(schemas.BifrostContextKeyProviderResponseHeaders, providerResponseHeaders)

if resp.StatusCode() != fasthttp.StatusOK {
providerUtils.MaterializeStreamErrorBody(ctx, resp)
logger.Debug(fmt.Sprintf("error from %s provider: %s", providerName, string(resp.Body())))
return nil, providerUtils.EnrichError(ctx, ParseOpenAIError(resp), jsonData, nil, sendBackRawRequest, sendBackRawResponse, latency)
}

body, _, finalErr := finalizeOpenAIResponse(ctx, resp, latency, providerName, logger)
respOwned = false
if finalErr != nil {
return nil, providerUtils.EnrichError(ctx, finalErr, jsonData, nil, sendBackRawRequest, sendBackRawResponse, latency)
}

response := &OpenAIRerankResponse{}
rawRequest, rawResponse, bifrostErr := providerUtils.HandleProviderResponse(body, response, jsonData, sendBackRawRequest, sendBackRawResponse)
if bifrostErr != nil {
return nil, providerUtils.EnrichError(ctx, bifrostErr, jsonData, body, sendBackRawRequest, sendBackRawResponse, latency)
}

returnDocuments := request.Params != nil && request.Params.ReturnDocuments != nil && *request.Params.ReturnDocuments
bifrostResponse := response.ToBifrostRerankResponse(request.Documents, returnDocuments)
bifrostResponse.Model = request.Model
bifrostResponse.ExtraFields.Latency = latency.Milliseconds()
bifrostResponse.ExtraFields.ProviderResponseHeaders = providerResponseHeaders
if sendBackRawRequest {
bifrostResponse.ExtraFields.RawRequest = rawRequest
}
if sendBackRawResponse {
bifrostResponse.ExtraFields.RawResponse = rawResponse
}
return bifrostResponse, nil
}

// OCR is not supported by the Openai provider.
Expand Down
161 changes: 161 additions & 0 deletions core/providers/openai/rerank.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
package openai

import (
"encoding/json"
"sort"

"github.com/bytedance/sonic"
"github.com/maximhq/bifrost/core/schemas"
)

// ToOpenAIRerankRequest converts a Bifrost rerank request to OpenAI-compatible format
func ToOpenAIRerankRequest(request *schemas.BifrostRerankRequest) *OpenAIRerankRequest {
if request == nil {
return nil
}

converted := &OpenAIRerankRequest{
Model: request.Model,
Query: request.Query,
Documents: request.Documents,
}
if request.Params != nil {
converted.TopN = request.Params.TopN
converted.MaxTokensPerDoc = request.Params.MaxTokensPerDoc
converted.Priority = request.Params.Priority
converted.ExtraParams = request.Params.ExtraParams
}
return converted
}

// ToBifrostRerankResponse converts an OpenAI-compatible rerank response to Bifrost format
func (response *OpenAIRerankResponse) ToBifrostRerankResponse(documents []schemas.RerankDocument, returnDocuments bool) *schemas.BifrostRerankResponse {
if response == nil {
return nil
}

bifrostResponse := &schemas.BifrostRerankResponse{
ID: response.ID,
}
for _, result := range response.Results {
rerankResult := schemas.RerankResult{
Index: result.Index,
RelevanceScore: result.RelevanceScore,
}
if doc := parseOpenAIRerankDocument(result.Document); doc != nil {
rerankResult.Document = doc
}
bifrostResponse.Results = append(bifrostResponse.Results, rerankResult)
}
sort.SliceStable(bifrostResponse.Results, func(i, j int) bool {
if bifrostResponse.Results[i].RelevanceScore == bifrostResponse.Results[j].RelevanceScore {
return bifrostResponse.Results[i].Index < bifrostResponse.Results[j].Index
}
return bifrostResponse.Results[i].RelevanceScore > bifrostResponse.Results[j].RelevanceScore
})
if returnDocuments {
for i := range bifrostResponse.Results {
// Preserve any document the upstream already returned; only backfill from the request otherwise.
if bifrostResponse.Results[i].Document != nil {
continue
}
resultIndex := bifrostResponse.Results[i].Index
if resultIndex >= 0 && resultIndex < len(documents) {
bifrostResponse.Results[i].Document = schemas.Ptr(documents[resultIndex])
}
}
}
if response.Usage != nil {
bifrostResponse.Usage = response.Usage
} else if response.Meta != nil {
bifrostResponse.Usage = openAIRerankUsage(response.Meta.Tokens)
if bifrostResponse.Usage == nil {
bifrostResponse.Usage = openAIRerankUsage(response.Meta.BilledUnits)
}
}
return bifrostResponse
}

// parseOpenAIRerankDocument parses a rerank document returned by the upstream,
// accepting either a bare string or an object with text/id/metadata fields.
func parseOpenAIRerankDocument(raw json.RawMessage) *schemas.RerankDocument {
if len(raw) == 0 {
return nil
}
var text string
if err := sonic.Unmarshal(raw, &text); err == nil {
return &schemas.RerankDocument{Text: text}
}

var docMap map[string]interface{}
if err := sonic.Unmarshal(raw, &docMap); err != nil {
return nil
}
doc := &schemas.RerankDocument{}
populated := false
if text, ok := docMap["text"].(string); ok {
doc.Text = text
populated = true
}
if id, ok := docMap["id"].(string); ok {
doc.ID = &id
populated = true
}
meta := make(map[string]interface{})
if rawMeta, ok := docMap["metadata"].(map[string]interface{}); ok {
for k, v := range rawMeta {
meta[k] = v
}
} else if rawMeta, ok := docMap["meta"].(map[string]interface{}); ok {
for k, v := range rawMeta {
meta[k] = v
}
}
for k, v := range docMap {
if k != "text" && k != "id" && k != "metadata" && k != "meta" {
meta[k] = v
}
}
if len(meta) > 0 {
doc.Meta = meta
populated = true
}
if !populated {
return nil
}
return doc
}

// openAIRerankUsage maps rerank token/billing counts onto Bifrost usage.
func openAIRerankUsage(tokens *OpenAIRerankTokenUsage) *schemas.BifrostLLMUsage {
if tokens == nil {
return nil
}
promptTokens := 0
completionTokens := 0
hasUsage := false
if tokens.InputTokens != nil {
promptTokens = int(*tokens.InputTokens)
hasUsage = true
}
if tokens.OutputTokens != nil {
completionTokens = int(*tokens.OutputTokens)
hasUsage = true
}
totalTokens := promptTokens + completionTokens
if tokens.SearchUnits != nil {
// Cohere-shaped rerank upstreams bill via billed_units.search_units instead of
// token counts (input/output tokens are typically null for rerank). Surface the
// search-unit count in the usage total so search-unit-billed providers aren't dropped.
totalTokens += int(*tokens.SearchUnits)
hasUsage = true
}
if !hasUsage {
return nil
}
return &schemas.BifrostLLMUsage{
PromptTokens: promptTokens,
CompletionTokens: completionTokens,
TotalTokens: totalTokens,
}
}
Loading