diff --git a/README.md b/README.md index 8fc0b70f718..843dec29e38 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ ## The fastest way to build AI applications that never go down -Bifrost is a high-performance AI gateway that connects you to 10+ providers (OpenAI, Anthropic, Bedrock, and more) through a single API. Get automatic failover, load balancing, and zero-downtime deployments in under 30 seconds. +Bifrost is a high-performance AI gateway that connects you to 12+ providers (OpenAI, Anthropic, Bedrock, and more) through a single API. Get automatic failover, load balancing, and zero-downtime deployments in under 30 seconds. ๐Ÿš€ **Just launched:** Native MCP (Model Context Protocol) support for seamless tool integration โšก **Performance:** Adds only 11ยตs latency while handling 5,000+ RPS @@ -75,6 +75,7 @@ Your AI gateway is now running with a beautiful web interface. You can: ## ๐Ÿ“‘ Table of Contents - [Bifrost](#bifrost) + - [The fastest way to build AI applications that never go down](#the-fastest-way-to-build-ai-applications-that-never-go-down) - [โšก Quickstart (30 seconds)](#-quickstart-30-seconds) - [Using Bifrost HTTP Transport](#using-bifrost-http-transport) - [๐Ÿ“‘ Table of Contents](#-table-of-contents) @@ -247,7 +248,7 @@ Choose higher settings (like the t3.xlarge profile above) for raw speed, or lowe
๐ŸŽฏ I want to understand what Bifrost can do -- **[๐Ÿ”— Multi-Provider Support](./docs/usage/providers.md)** - Connect to 10+ AI providers with one API +- **[๐Ÿ”— Multi-Provider Support](./docs/usage/providers.md)** - Connect to 12+ AI providers with one API - **[๐Ÿ›ก๏ธ Fallback & Reliability](./docs/usage/providers.md#fallback-mechanisms)** - Never lose a request with automatic failover - **[๐Ÿ› ๏ธ MCP Tool Integration](./docs/usage/http-transport/configuration/mcp.md)** - Give your AI external capabilities - **[๐Ÿ”Œ Plugin Ecosystem](./docs/usage/http-transport/configuration/plugins.md)** - Extend Bifrost with custom middleware diff --git a/ci/npx/package.json b/ci/npx/package.json index 11feb40d85c..8292620629c 100644 --- a/ci/npx/package.json +++ b/ci/npx/package.json @@ -1,7 +1,7 @@ { "name": "@maximhq/bifrost", "version": "1.0.4", - "description": "High-performance AI gateway CLI - connect to 10+ providers through a single API", + "description": "High-performance AI gateway CLI - connect to 12+ providers through a single API", "keywords": ["ai", "gateway", "openai", "anthropic", "cli", "bifrost"], "homepage": "https://github.com/maximhq/bifrost", "repository": { diff --git a/core/bifrost.go b/core/bifrost.go index 3f0d6cfc27b..c09f6bcfe8b 100644 --- a/core/bifrost.go +++ b/core/bifrost.go @@ -687,6 +687,8 @@ func (bifrost *Bifrost) createProviderFromProviderKey(providerKey schemas.ModelP return providers.NewSGLProvider(config, bifrost.logger) case schemas.Parasail: return providers.NewParasailProvider(config, bifrost.logger) + case schemas.Cerebras: + return providers.NewCerebrasProvider(config, bifrost.logger) default: return nil, fmt.Errorf("unsupported provider: %s", providerKey) } diff --git a/core/providers/azure.go b/core/providers/azure.go index c802eac0299..40d4a1a1574 100644 --- a/core/providers/azure.go +++ b/core/providers/azure.go @@ -50,26 +50,26 @@ var azureTextCompletionResponsePool = sync.Pool{ }, } -// azureChatResponsePool provides a pool for Azure chat response objects. -var azureChatResponsePool = sync.Pool{ - New: func() interface{} { - return &schemas.BifrostResponse{} - }, -} - -// acquireAzureChatResponse gets an Azure chat response from the pool and resets it. -func acquireAzureChatResponse() *schemas.BifrostResponse { - resp := azureChatResponsePool.Get().(*schemas.BifrostResponse) - *resp = schemas.BifrostResponse{} // Reset the struct - return resp -} - -// releaseAzureChatResponse returns an Azure chat response to the pool. -func releaseAzureChatResponse(resp *schemas.BifrostResponse) { - if resp != nil { - azureChatResponsePool.Put(resp) - } -} +// // azureChatResponsePool provides a pool for Azure chat response objects. +// var azureChatResponsePool = sync.Pool{ +// New: func() interface{} { +// return &schemas.BifrostResponse{} +// }, +// } + +// // acquireAzureChatResponse gets an Azure chat response from the pool and resets it. +// func acquireAzureChatResponse() *schemas.BifrostResponse { +// resp := azureChatResponsePool.Get().(*schemas.BifrostResponse) +// *resp = schemas.BifrostResponse{} // Reset the struct +// return resp +// } + +// // releaseAzureChatResponse returns an Azure chat response to the pool. +// func releaseAzureChatResponse(resp *schemas.BifrostResponse) { +// if resp != nil { +// azureChatResponsePool.Put(resp) +// } +// } // acquireAzureTextResponse gets an Azure text completion response from the pool and resets it. func acquireAzureTextResponse() *AzureTextResponse { @@ -113,7 +113,7 @@ func NewAzureProvider(config *schemas.ProviderConfig, logger schemas.Logger) (*A // Pre-warm response pools for range config.ConcurrencyAndBufferSize.Concurrency { - azureChatResponsePool.Put(&schemas.BifrostResponse{}) + // azureChatResponsePool.Put(&schemas.BifrostResponse{}) azureTextCompletionResponsePool.Put(&AzureTextResponse{}) } @@ -308,8 +308,10 @@ func (provider *AzureProvider) ChatCompletion(ctx context.Context, model string, } // Create response object from pool - response := acquireAzureChatResponse() - defer releaseAzureChatResponse(response) + // response := acquireAzureChatResponse() + // defer releaseAzureChatResponse(response) + + response := &schemas.BifrostResponse{} rawResponse, bifrostErr := handleProviderResponse(responseBody, response, provider.sendBackRawResponse) if bifrostErr != nil { @@ -359,8 +361,10 @@ func (provider *AzureProvider) Embedding(ctx context.Context, model string, key } // Pre-allocate response structs from pools - response := acquireAzureChatResponse() - defer releaseAzureChatResponse(response) + // response := acquireAzureChatResponse() + // defer releaseAzureChatResponse(response) + + response := &schemas.BifrostResponse{} // Use enhanced response handler with pre-allocated response rawResponse, bifrostErr := handleProviderResponse(responseBody, response, provider.sendBackRawResponse) diff --git a/core/providers/cerebras.go b/core/providers/cerebras.go new file mode 100644 index 00000000000..95f11491252 --- /dev/null +++ b/core/providers/cerebras.go @@ -0,0 +1,356 @@ +// Package providers implements various LLM providers and their utility functions. +// This file contains the Cerebras provider implementation. +package providers + +import ( + "context" + "fmt" + "net/http" + "strings" + "sync" + "time" + + "github.com/bytedance/sonic" + schemas "github.com/maximhq/bifrost/core/schemas" + "github.com/valyala/fasthttp" +) + +// cerebrasTextResponsePool provides a pool for Cerebras text completion response objects. +var cerebrasTextResponsePool = sync.Pool{ + New: func() interface{} { + return &AzureTextResponse{} + }, +} + +// // cerebrasChatResponsePool provides a pool for Cerebras chat response objects. +// var cerebrasChatResponsePool = sync.Pool{ +// New: func() interface{} { +// return &schemas.BifrostResponse{} +// }, +// } + +// // acquireCerebrasChatResponse gets a Cerebras response from the pool and resets it. +// func acquireCerebrasChatResponse() *schemas.BifrostResponse { +// resp := cerebrasChatResponsePool.Get().(*schemas.BifrostResponse) +// *resp = schemas.BifrostResponse{} // Reset the struct +// return resp +// } + +// // releaseCerebrasChatResponse returns a Cerebras response to the pool. +// func releaseCerebrasChatResponse(resp *schemas.BifrostResponse) { +// if resp != nil { +// cerebrasChatResponsePool.Put(resp) +// } +// } + +// acquireCerebrasTextResponse gets a Cerebras text completion response from the pool and resets it. +func acquireCerebrasTextResponse() *AzureTextResponse { + resp := cerebrasTextResponsePool.Get().(*AzureTextResponse) + *resp = AzureTextResponse{} // Reset the struct + return resp +} + +// releaseCerebrasTextResponse returns a Cerebras text completion response to the pool. +func releaseCerebrasTextResponse(resp *AzureTextResponse) { + if resp != nil { + cerebrasTextResponsePool.Put(resp) + } +} + +// CerebrasProvider implements the Provider interface for Cerebras's API. +type CerebrasProvider struct { + logger schemas.Logger // Logger for provider operations + client *fasthttp.Client // HTTP client for API requests + streamClient *http.Client // HTTP client for streaming requests + networkConfig schemas.NetworkConfig // Network configuration including extra headers + sendBackRawResponse bool // Whether to include raw response in BifrostResponse +} + +// NewCerebrasProvider creates a new Cerebras provider instance. +// It initializes the HTTP client with the provided configuration and sets up response pools. +// The client is configured with timeouts, concurrency limits, and optional proxy settings. +func NewCerebrasProvider(config *schemas.ProviderConfig, logger schemas.Logger) (*CerebrasProvider, error) { + config.CheckAndSetDefaults() + + client := &fasthttp.Client{ + ReadTimeout: time.Second * time.Duration(config.NetworkConfig.DefaultRequestTimeoutInSeconds), + WriteTimeout: time.Second * time.Duration(config.NetworkConfig.DefaultRequestTimeoutInSeconds), + MaxConnsPerHost: config.ConcurrencyAndBufferSize.BufferSize, + } + + // Initialize streaming HTTP client + streamClient := &http.Client{ + Timeout: time.Second * time.Duration(config.NetworkConfig.DefaultRequestTimeoutInSeconds), + } + + // Pre-warm response pools + for range config.ConcurrencyAndBufferSize.Concurrency { + // cerebrasChatResponsePool.Put(&schemas.BifrostResponse{}) + cerebrasTextResponsePool.Put(&AzureTextResponse{}) + } + + // Configure proxy if provided + client = configureProxy(client, config.ProxyConfig, logger) + + // Set default BaseURL if not provided + if config.NetworkConfig.BaseURL == "" { + config.NetworkConfig.BaseURL = "https://api.cerebras.ai" + } + config.NetworkConfig.BaseURL = strings.TrimRight(config.NetworkConfig.BaseURL, "/") + + return &CerebrasProvider{ + logger: logger, + client: client, + streamClient: streamClient, + networkConfig: config.NetworkConfig, + sendBackRawResponse: config.SendBackRawResponse, + }, nil +} + +// GetProviderKey returns the provider identifier for Cerebras. +func (provider *CerebrasProvider) GetProviderKey() schemas.ModelProvider { + return schemas.Cerebras +} + +// TextCompletion performs a text completion request to Cerebras's API. +// It formats the request, sends it to Cerebras, and processes the response. +// Returns a BifrostResponse containing the completion results or an error if the request fails. +func (provider *CerebrasProvider) TextCompletion(ctx context.Context, model string, key schemas.Key, text string, params *schemas.ModelParameters) (*schemas.BifrostResponse, *schemas.BifrostError) { + preparedParams := prepareParams(params) + + // Merge additional parameters + requestBody := mergeConfig(map[string]interface{}{ + "model": model, + "prompt": text, + }, preparedParams) + + // Create request + req := fasthttp.AcquireRequest() + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseRequest(req) + defer fasthttp.ReleaseResponse(resp) + + jsonBody, err := sonic.Marshal(requestBody) + if err != nil { + return nil, newBifrostOperationError(schemas.ErrProviderJSONMarshaling, err, schemas.Cerebras) + } + + // Set any extra headers from network config + setExtraHeaders(req, provider.networkConfig.ExtraHeaders, nil) + + req.SetRequestURI(provider.networkConfig.BaseURL + "/v1/completions") + req.Header.SetMethod("POST") + req.Header.SetContentType("application/json") + req.Header.Set("Authorization", "Bearer "+key.Value) + + req.SetBody(jsonBody) + + // Make request + bifrostErr := makeRequestWithContext(ctx, provider.client, req, resp) + if bifrostErr != nil { + return nil, bifrostErr + } + + // Handle error response + if resp.StatusCode() != fasthttp.StatusOK { + provider.logger.Debug(fmt.Sprintf("error from cerebras provider: %s", string(resp.Body()))) + + var errorResp map[string]interface{} + bifrostErr := handleProviderAPIError(resp, &errorResp) + bifrostErr.Error.Message = fmt.Sprintf("Cerebras error: %v", errorResp) + return nil, bifrostErr + } + + responseBody := resp.Body() + + // Pre-allocate response structs from pools + response := acquireCerebrasTextResponse() + defer releaseCerebrasTextResponse(response) + + rawResponse, bifrostErr := handleProviderResponse(responseBody, response, provider.sendBackRawResponse) + if bifrostErr != nil { + return nil, bifrostErr + } + + choices := []schemas.BifrostResponseChoice{} + + // Create the completion result + if len(response.Choices) > 0 { + // Copy text content to avoid pointer to pooled memory + textCopy := response.Choices[0].Text + + choices = append(choices, schemas.BifrostResponseChoice{ + Index: 0, + BifrostNonStreamResponseChoice: &schemas.BifrostNonStreamResponseChoice{ + Message: schemas.BifrostMessage{ + Role: schemas.ModelChatMessageRoleAssistant, + Content: schemas.MessageContent{ + ContentStr: &textCopy, + }, + }, + LogProbs: &schemas.LogProbs{ + Text: response.Choices[0].LogProbs, + }, + }, + FinishReason: response.Choices[0].FinishReason, + }) + } + + // Copy Usage struct to avoid pointer to pooled memory + usageCopy := response.Usage + + // Create final response + bifrostResponse := &schemas.BifrostResponse{ + ID: response.ID, + Choices: choices, + Model: response.Model, + Created: response.Created, + SystemFingerprint: response.SystemFingerprint, + Usage: &usageCopy, + ExtraFields: schemas.BifrostResponseExtraFields{ + Provider: schemas.Cerebras, + }, + } + + // Set raw response if enabled + if provider.sendBackRawResponse { + bifrostResponse.ExtraFields.RawResponse = rawResponse + } + + if params != nil { + bifrostResponse.ExtraFields.Params = *params + } + + return bifrostResponse, nil +} + +// ChatCompletion performs a chat completion request to the Cerebras API. +func (provider *CerebrasProvider) ChatCompletion(ctx context.Context, model string, key schemas.Key, messages []schemas.BifrostMessage, params *schemas.ModelParameters) (*schemas.BifrostResponse, *schemas.BifrostError) { + formattedMessages, preparedParams := prepareOpenAIChatRequest(messages, params) + + requestBody := mergeConfig(map[string]interface{}{ + "model": model, + "messages": formattedMessages, + }, preparedParams) + + jsonBody, err := sonic.Marshal(requestBody) + if err != nil { + return nil, newBifrostOperationError(schemas.ErrProviderJSONMarshaling, err, schemas.Cerebras) + } + + // Create request + req := fasthttp.AcquireRequest() + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseRequest(req) + defer fasthttp.ReleaseResponse(resp) + + // Set any extra headers from network config + setExtraHeaders(req, provider.networkConfig.ExtraHeaders, nil) + + req.SetRequestURI(provider.networkConfig.BaseURL + "/v1/chat/completions") + req.Header.SetMethod("POST") + req.Header.SetContentType("application/json") + req.Header.Set("Authorization", "Bearer "+key.Value) + + req.SetBody(jsonBody) + + // Make request + bifrostErr := makeRequestWithContext(ctx, provider.client, req, resp) + if bifrostErr != nil { + return nil, bifrostErr + } + + // Handle error response + if resp.StatusCode() != fasthttp.StatusOK { + provider.logger.Debug(fmt.Sprintf("error from cerebras provider: %s", string(resp.Body()))) + + var errorResp map[string]interface{} + bifrostErr := handleProviderAPIError(resp, &errorResp) + bifrostErr.Error.Message = fmt.Sprintf("Cerebras error: %v", errorResp) + return nil, bifrostErr + } + + responseBody := resp.Body() + + // Pre-allocate response structs from pools + // response := acquireCerebrasChatResponse() + // defer releaseCerebrasChatResponse(response) + response := &schemas.BifrostResponse{} + + // Use enhanced response handler with pre-allocated response + rawResponse, bifrostErr := handleProviderResponse(responseBody, response, provider.sendBackRawResponse) + if bifrostErr != nil { + return nil, bifrostErr + } + + // Create final response + response.ExtraFields.Provider = schemas.Cerebras + + if provider.sendBackRawResponse { + response.ExtraFields.RawResponse = rawResponse + } + + if params != nil { + response.ExtraFields.Params = *params + } + + return response, nil +} + +// Embedding is not supported by the Cerebras provider. +func (provider *CerebrasProvider) Embedding(ctx context.Context, model string, key schemas.Key, input *schemas.EmbeddingInput, params *schemas.ModelParameters) (*schemas.BifrostResponse, *schemas.BifrostError) { + return nil, newUnsupportedOperationError("embedding", "cerebras") +} + +// ChatCompletionStream performs a streaming chat completion request to the Cerebras API. +// It supports real-time streaming of responses using Server-Sent Events (SSE). +// Uses Cerebras's OpenAI-compatible streaming format. +// Returns a channel containing BifrostResponse objects representing the stream or an error if the request fails. +func (provider *CerebrasProvider) ChatCompletionStream(ctx context.Context, postHookRunner schemas.PostHookRunner, model string, key schemas.Key, messages []schemas.BifrostMessage, params *schemas.ModelParameters) (chan *schemas.BifrostStream, *schemas.BifrostError) { + formattedMessages, preparedParams := prepareOpenAIChatRequest(messages, params) + + requestBody := mergeConfig(map[string]interface{}{ + "model": model, + "messages": formattedMessages, + "stream": true, + }, preparedParams) + + // Prepare Cerebras headers + headers := map[string]string{ + "Content-Type": "application/json", + "Accept": "text/event-stream", + "Cache-Control": "no-cache", + } + + headers["Authorization"] = "Bearer " + key.Value + + // Use shared OpenAI-compatible streaming logic + return handleOpenAIStreaming( + ctx, + provider.streamClient, + provider.networkConfig.BaseURL+"/v1/chat/completions", + requestBody, + headers, + provider.networkConfig.ExtraHeaders, + schemas.Cerebras, + params, + postHookRunner, + provider.logger, + ) +} + +func (provider *CerebrasProvider) Speech(ctx context.Context, model string, key schemas.Key, input *schemas.SpeechInput, params *schemas.ModelParameters) (*schemas.BifrostResponse, *schemas.BifrostError) { + return nil, newUnsupportedOperationError("speech", "cerebras") +} + +func (provider *CerebrasProvider) SpeechStream(ctx context.Context, postHookRunner schemas.PostHookRunner, model string, key schemas.Key, input *schemas.SpeechInput, params *schemas.ModelParameters) (chan *schemas.BifrostStream, *schemas.BifrostError) { + return nil, newUnsupportedOperationError("speech stream", "cerebras") +} + +func (provider *CerebrasProvider) Transcription(ctx context.Context, model string, key schemas.Key, input *schemas.TranscriptionInput, params *schemas.ModelParameters) (*schemas.BifrostResponse, *schemas.BifrostError) { + return nil, newUnsupportedOperationError("transcription", "cerebras") +} + +func (provider *CerebrasProvider) TranscriptionStream(ctx context.Context, postHookRunner schemas.PostHookRunner, model string, key schemas.Key, input *schemas.TranscriptionInput, params *schemas.ModelParameters) (chan *schemas.BifrostStream, *schemas.BifrostError) { + return nil, newUnsupportedOperationError("transcription stream", "cerebras") +} diff --git a/core/providers/groq.go b/core/providers/groq.go index 1c03fe74629..93d34e11b39 100644 --- a/core/providers/groq.go +++ b/core/providers/groq.go @@ -7,7 +7,6 @@ import ( "fmt" "net/http" "strings" - "sync" "time" "github.com/bytedance/sonic" @@ -15,26 +14,26 @@ import ( "github.com/valyala/fasthttp" ) -// groqResponsePool provides a pool for Groq response objects. -var groqResponsePool = sync.Pool{ - New: func() interface{} { - return &schemas.BifrostResponse{} - }, -} - -// acquireGroqResponse gets a Groq response from the pool and resets it. -func acquireGroqResponse() *schemas.BifrostResponse { - resp := groqResponsePool.Get().(*schemas.BifrostResponse) - *resp = schemas.BifrostResponse{} // Reset the struct - return resp -} - -// releaseGroqResponse returns a Groq response to the pool. -func releaseGroqResponse(resp *schemas.BifrostResponse) { - if resp != nil { - groqResponsePool.Put(resp) - } -} +// // groqResponsePool provides a pool for Groq response objects. +// var groqResponsePool = sync.Pool{ +// New: func() interface{} { +// return &schemas.BifrostResponse{} +// }, +// } + +// // acquireGroqResponse gets a Groq response from the pool and resets it. +// func acquireGroqResponse() *schemas.BifrostResponse { +// resp := groqResponsePool.Get().(*schemas.BifrostResponse) +// *resp = schemas.BifrostResponse{} // Reset the struct +// return resp +// } + +// // releaseGroqResponse returns a Groq response to the pool. +// func releaseGroqResponse(resp *schemas.BifrostResponse) { +// if resp != nil { +// groqResponsePool.Put(resp) +// } +// } // GroqProvider implements the Provider interface for Groq's API. type GroqProvider struct { @@ -62,10 +61,10 @@ func NewGroqProvider(config *schemas.ProviderConfig, logger schemas.Logger) (*Gr Timeout: time.Second * time.Duration(config.NetworkConfig.DefaultRequestTimeoutInSeconds), } - // Pre-warm response pools - for range config.ConcurrencyAndBufferSize.Concurrency { - groqResponsePool.Put(&schemas.BifrostResponse{}) - } + // // Pre-warm response pools + // for range config.ConcurrencyAndBufferSize.Concurrency { + // groqResponsePool.Put(&schemas.BifrostResponse{}) + // } // Configure proxy if provided client = configureProxy(client, config.ProxyConfig, logger) @@ -144,8 +143,9 @@ func (provider *GroqProvider) ChatCompletion(ctx context.Context, model string, responseBody := resp.Body() // Pre-allocate response structs from pools - response := acquireGroqResponse() - defer releaseGroqResponse(response) + // response := acquireGroqResponse() + // defer releaseGroqResponse(response) + response := &schemas.BifrostResponse{} // Use enhanced response handler with pre-allocated response rawResponse, bifrostErr := handleProviderResponse(responseBody, response, provider.sendBackRawResponse) @@ -185,7 +185,7 @@ func (provider *GroqProvider) ChatCompletionStream(ctx context.Context, postHook "stream": true, }, preparedParams) - // Prepare Groq headers (Groq typically doesn't require authorization, but we include it if provided) + // Prepare Groq headers headers := map[string]string{ "Content-Type": "application/json", "Accept": "text/event-stream", diff --git a/core/providers/mistral.go b/core/providers/mistral.go index 9910edb0701..030001ff65a 100644 --- a/core/providers/mistral.go +++ b/core/providers/mistral.go @@ -7,7 +7,6 @@ import ( "fmt" "net/http" "strings" - "sync" "time" "github.com/bytedance/sonic" @@ -15,26 +14,26 @@ import ( "github.com/valyala/fasthttp" ) -// mistralResponsePool provides a pool for Mistral response objects. -var mistralResponsePool = sync.Pool{ - New: func() interface{} { - return &schemas.BifrostResponse{} - }, -} - -// acquireMistralResponse gets a Mistral response from the pool and resets it. -func acquireMistralResponse() *schemas.BifrostResponse { - resp := mistralResponsePool.Get().(*schemas.BifrostResponse) - *resp = schemas.BifrostResponse{} // Reset the struct - return resp -} - -// releaseMistralResponse returns a Mistral response to the pool. -func releaseMistralResponse(resp *schemas.BifrostResponse) { - if resp != nil { - mistralResponsePool.Put(resp) - } -} +// // mistralResponsePool provides a pool for Mistral response objects. +// var mistralResponsePool = sync.Pool{ +// New: func() interface{} { +// return &schemas.BifrostResponse{} +// }, +// } + +// // acquireMistralResponse gets a Mistral response from the pool and resets it. +// func acquireMistralResponse() *schemas.BifrostResponse { +// resp := mistralResponsePool.Get().(*schemas.BifrostResponse) +// *resp = schemas.BifrostResponse{} // Reset the struct +// return resp +// } + +// // releaseMistralResponse returns a Mistral response to the pool. +// func releaseMistralResponse(resp *schemas.BifrostResponse) { +// if resp != nil { +// mistralResponsePool.Put(resp) +// } +// } // MistralProvider implements the Provider interface for Mistral's API. type MistralProvider struct { @@ -63,9 +62,9 @@ func NewMistralProvider(config *schemas.ProviderConfig, logger schemas.Logger) * } // Pre-warm response pools - for range config.ConcurrencyAndBufferSize.Concurrency { - mistralResponsePool.Put(&schemas.BifrostResponse{}) - } + // for range config.ConcurrencyAndBufferSize.Concurrency { + // mistralResponsePool.Put(&schemas.BifrostResponse{}) + // } // Configure proxy if provided client = configureProxy(client, config.ProxyConfig, logger) @@ -144,8 +143,9 @@ func (provider *MistralProvider) ChatCompletion(ctx context.Context, model strin responseBody := resp.Body() // Pre-allocate response structs from pools - response := acquireMistralResponse() - defer releaseMistralResponse(response) + // response := acquireMistralResponse() + // defer releaseMistralResponse(response) + response := &schemas.BifrostResponse{} // Use enhanced response handler with pre-allocated response rawResponse, bifrostErr := handleProviderResponse(responseBody, response, provider.sendBackRawResponse) @@ -239,8 +239,9 @@ func (provider *MistralProvider) Embedding(ctx context.Context, model string, ke responseBody := resp.Body() // Pre-allocate response structs from pools - response := acquireMistralResponse() - defer releaseMistralResponse(response) + // response := acquireMistralResponse() + response := &schemas.BifrostResponse{} + // defer releaseMistralResponse(response) // Use enhanced response handler with pre-allocated response rawResponse, bifrostErr := handleProviderResponse(responseBody, response, provider.sendBackRawResponse) diff --git a/core/providers/ollama.go b/core/providers/ollama.go index fe994cecabf..ff0a5b374ee 100644 --- a/core/providers/ollama.go +++ b/core/providers/ollama.go @@ -7,7 +7,6 @@ import ( "fmt" "net/http" "strings" - "sync" "time" "github.com/bytedance/sonic" @@ -15,26 +14,26 @@ import ( "github.com/valyala/fasthttp" ) -// ollamaResponsePool provides a pool for Ollama response objects. -var ollamaResponsePool = sync.Pool{ - New: func() interface{} { - return &schemas.BifrostResponse{} - }, -} - -// acquireOllamaResponse gets a Ollama response from the pool and resets it. -func acquireOllamaResponse() *schemas.BifrostResponse { - resp := ollamaResponsePool.Get().(*schemas.BifrostResponse) - *resp = schemas.BifrostResponse{} // Reset the struct - return resp -} - -// releaseOllamaResponse returns a Ollama response to the pool. -func releaseOllamaResponse(resp *schemas.BifrostResponse) { - if resp != nil { - ollamaResponsePool.Put(resp) - } -} +// // ollamaResponsePool provides a pool for Ollama response objects. +// var ollamaResponsePool = sync.Pool{ +// New: func() interface{} { +// return &schemas.BifrostResponse{} +// }, +// } + +// // acquireOllamaResponse gets a Ollama response from the pool and resets it. +// func acquireOllamaResponse() *schemas.BifrostResponse { +// resp := ollamaResponsePool.Get().(*schemas.BifrostResponse) +// *resp = schemas.BifrostResponse{} // Reset the struct +// return resp +// } + +// // releaseOllamaResponse returns a Ollama response to the pool. +// func releaseOllamaResponse(resp *schemas.BifrostResponse) { +// if resp != nil { +// ollamaResponsePool.Put(resp) +// } +// } // OllamaProvider implements the Provider interface for Ollama's API. type OllamaProvider struct { @@ -62,10 +61,10 @@ func NewOllamaProvider(config *schemas.ProviderConfig, logger schemas.Logger) (* Timeout: time.Second * time.Duration(config.NetworkConfig.DefaultRequestTimeoutInSeconds), } - // Pre-warm response pools - for range config.ConcurrencyAndBufferSize.Concurrency { - ollamaResponsePool.Put(&schemas.BifrostResponse{}) - } + // // Pre-warm response pools + // for range config.ConcurrencyAndBufferSize.Concurrency { + // ollamaResponsePool.Put(&schemas.BifrostResponse{}) + // } // Configure proxy if provided client = configureProxy(client, config.ProxyConfig, logger) @@ -147,8 +146,9 @@ func (provider *OllamaProvider) ChatCompletion(ctx context.Context, model string responseBody := resp.Body() // Pre-allocate response structs from pools - response := acquireOllamaResponse() - defer releaseOllamaResponse(response) + // response := acquireOllamaResponse() + // defer releaseOllamaResponse(response) + response := &schemas.BifrostResponse{} // Use enhanced response handler with pre-allocated response rawResponse, bifrostErr := handleProviderResponse(responseBody, response, provider.sendBackRawResponse) diff --git a/core/providers/openai.go b/core/providers/openai.go index feab480191e..82e49d89e22 100644 --- a/core/providers/openai.go +++ b/core/providers/openai.go @@ -11,7 +11,6 @@ import ( "mime/multipart" "net/http" "strings" - "sync" "time" "github.com/bytedance/sonic" @@ -19,26 +18,26 @@ import ( "github.com/valyala/fasthttp" ) -// openAIResponsePool provides a pool for OpenAI response objects. -var openAIResponsePool = sync.Pool{ - New: func() interface{} { - return &schemas.BifrostResponse{} - }, -} - -// acquireOpenAIResponse gets an OpenAI response from the pool and resets it. -func acquireOpenAIResponse() *schemas.BifrostResponse { - resp := openAIResponsePool.Get().(*schemas.BifrostResponse) - *resp = schemas.BifrostResponse{} // Reset the struct - return resp -} - -// releaseOpenAIResponse returns an OpenAI response to the pool. -func releaseOpenAIResponse(resp *schemas.BifrostResponse) { - if resp != nil { - openAIResponsePool.Put(resp) - } -} +// // openAIResponsePool provides a pool for OpenAI response objects. +// var openAIResponsePool = sync.Pool{ +// New: func() interface{} { +// return &schemas.BifrostResponse{} +// }, +// } + +// // acquireOpenAIResponse gets an OpenAI response from the pool and resets it. +// func acquireOpenAIResponse() *schemas.BifrostResponse { +// resp := openAIResponsePool.Get().(*schemas.BifrostResponse) +// *resp = schemas.BifrostResponse{} // Reset the struct +// return resp +// } + +// // releaseOpenAIResponse returns an OpenAI response to the pool. +// func releaseOpenAIResponse(resp *schemas.BifrostResponse) { +// if resp != nil { +// openAIResponsePool.Put(resp) +// } +// } // OpenAIProvider implements the Provider interface for OpenAI's GPT API. type OpenAIProvider struct { @@ -66,10 +65,10 @@ func NewOpenAIProvider(config *schemas.ProviderConfig, logger schemas.Logger) *O Timeout: time.Second * time.Duration(config.NetworkConfig.DefaultRequestTimeoutInSeconds), } - // Pre-warm response pools - for range config.ConcurrencyAndBufferSize.Concurrency { - openAIResponsePool.Put(&schemas.BifrostResponse{}) - } + // // Pre-warm response pools + // for range config.ConcurrencyAndBufferSize.Concurrency { + // openAIResponsePool.Put(&schemas.BifrostResponse{}) + // } // Configure proxy if provided client = configureProxy(client, config.ProxyConfig, logger) @@ -147,8 +146,9 @@ func (provider *OpenAIProvider) ChatCompletion(ctx context.Context, model string responseBody := resp.Body() // Pre-allocate response structs from pools - response := acquireOpenAIResponse() - defer releaseOpenAIResponse(response) + // response := acquireOpenAIResponse() + // defer releaseOpenAIResponse(response) + response := &schemas.BifrostResponse{} // Use enhanced response handler with pre-allocated response rawResponse, bifrostErr := handleProviderResponse(responseBody, response, provider.sendBackRawResponse) @@ -165,6 +165,8 @@ func (provider *OpenAIProvider) ChatCompletion(ctx context.Context, model string response.ExtraFields.Params = *params } + response.ExtraFields.Provider = schemas.OpenAI + return response, nil } @@ -279,8 +281,9 @@ func (provider *OpenAIProvider) Embedding(ctx context.Context, model string, key responseBody := resp.Body() // Pre-allocate response structs from pools - response := acquireOpenAIResponse() - defer releaseOpenAIResponse(response) + // response := acquireOpenAIResponse() + // defer releaseOpenAIResponse(response) + response := &schemas.BifrostResponse{} // Use enhanced response handler with pre-allocated response rawResponse, bifrostErr := handleProviderResponse(responseBody, response, provider.sendBackRawResponse) diff --git a/core/providers/parasail.go b/core/providers/parasail.go index ef67721db58..97c70d71469 100644 --- a/core/providers/parasail.go +++ b/core/providers/parasail.go @@ -7,7 +7,6 @@ import ( "fmt" "net/http" "strings" - "sync" "time" "github.com/bytedance/sonic" @@ -15,26 +14,26 @@ import ( "github.com/valyala/fasthttp" ) -// parasailResponsePool provides a pool for Parasail response objects. -var parasailResponsePool = sync.Pool{ - New: func() interface{} { - return &schemas.BifrostResponse{} - }, -} - -// acquireParasailResponse gets a Parasail response from the pool and resets it. -func acquireParasailResponse() *schemas.BifrostResponse { - resp := parasailResponsePool.Get().(*schemas.BifrostResponse) - *resp = schemas.BifrostResponse{} // Reset the struct - return resp -} - -// releaseParasailResponse returns a Parasail response to the pool. -func releaseParasailResponse(resp *schemas.BifrostResponse) { - if resp != nil { - parasailResponsePool.Put(resp) - } -} +// // parasailResponsePool provides a pool for Parasail response objects. +// var parasailResponsePool = sync.Pool{ +// New: func() interface{} { +// return &schemas.BifrostResponse{} +// }, +// } + +// // acquireParasailResponse gets a Parasail response from the pool and resets it. +// func acquireParasailResponse() *schemas.BifrostResponse { +// resp := parasailResponsePool.Get().(*schemas.BifrostResponse) +// *resp = schemas.BifrostResponse{} // Reset the struct +// return resp +// } + +// // releaseParasailResponse returns a Parasail response to the pool. +// func releaseParasailResponse(resp *schemas.BifrostResponse) { +// if resp != nil { +// parasailResponsePool.Put(resp) +// } +// } // ParasailProvider implements the Provider interface for Parasail's API. type ParasailProvider struct { @@ -63,9 +62,9 @@ func NewParasailProvider(config *schemas.ProviderConfig, logger schemas.Logger) } // Pre-warm response pools - for range config.ConcurrencyAndBufferSize.Concurrency { - parasailResponsePool.Put(&schemas.BifrostResponse{}) - } + // for range config.ConcurrencyAndBufferSize.Concurrency { + // parasailResponsePool.Put(&schemas.BifrostResponse{}) + // } // Configure proxy if provided client = configureProxy(client, config.ProxyConfig, logger) @@ -144,7 +143,10 @@ func (provider *ParasailProvider) ChatCompletion(ctx context.Context, model stri responseBody := resp.Body() // Pre-allocate response structs from pools -response := acquireParasailResponse() + // response := acquireParasailResponse() + // defer releaseParasailResponse(response) + response := &schemas.BifrostResponse{} + // Use enhanced response handler with pre-allocated response rawResponse, bifrostErr := handleProviderResponse(responseBody, response, provider.sendBackRawResponse) if bifrostErr != nil { diff --git a/core/providers/sgl.go b/core/providers/sgl.go index ee092d79833..2a5f406ff72 100644 --- a/core/providers/sgl.go +++ b/core/providers/sgl.go @@ -7,7 +7,6 @@ import ( "fmt" "net/http" "strings" - "sync" "time" "github.com/bytedance/sonic" @@ -15,26 +14,26 @@ import ( "github.com/valyala/fasthttp" ) -// sglResponsePool provides a pool for SGL response objects. -var sglResponsePool = sync.Pool{ - New: func() interface{} { - return &schemas.BifrostResponse{} - }, -} - -// acquireSGLResponse gets a SGL response from the pool and resets it. -func acquireSGLResponse() *schemas.BifrostResponse { - resp := sglResponsePool.Get().(*schemas.BifrostResponse) - *resp = schemas.BifrostResponse{} // Reset the struct - return resp -} - -// releaseSGLResponse returns a SGL response to the pool. -func releaseSGLResponse(resp *schemas.BifrostResponse) { - if resp != nil { - sglResponsePool.Put(resp) - } -} +// // sglResponsePool provides a pool for SGL response objects. +// var sglResponsePool = sync.Pool{ +// New: func() interface{} { +// return &schemas.BifrostResponse{} +// }, +// } + +// // acquireSGLResponse gets a SGL response from the pool and resets it. +// func acquireSGLResponse() *schemas.BifrostResponse { +// resp := sglResponsePool.Get().(*schemas.BifrostResponse) +// *resp = schemas.BifrostResponse{} // Reset the struct +// return resp +// } + +// // releaseSGLResponse returns a SGL response to the pool. +// func releaseSGLResponse(resp *schemas.BifrostResponse) { +// if resp != nil { +// sglResponsePool.Put(resp) +// } +// } // SGLProvider implements the Provider interface for SGL's API. type SGLProvider struct { @@ -63,9 +62,9 @@ func NewSGLProvider(config *schemas.ProviderConfig, logger schemas.Logger) (*SGL } // Pre-warm response pools - for range config.ConcurrencyAndBufferSize.Concurrency { - sglResponsePool.Put(&schemas.BifrostResponse{}) - } + // for range config.ConcurrencyAndBufferSize.Concurrency { + // sglResponsePool.Put(&schemas.BifrostResponse{}) + // } // Configure proxy if provided client = configureProxy(client, config.ProxyConfig, logger) @@ -153,8 +152,9 @@ func (provider *SGLProvider) ChatCompletion(ctx context.Context, model string, k responseBody := resp.Body() // Pre-allocate response structs from pools - response := acquireSGLResponse() - defer releaseSGLResponse(response) + // response := acquireSGLResponse() + response := &schemas.BifrostResponse{} + // defer releaseSGLResponse(response) // Use enhanced response handler with pre-allocated response rawResponse, bifrostErr := handleProviderResponse(responseBody, response, provider.sendBackRawResponse) diff --git a/core/providers/vertex.go b/core/providers/vertex.go index fb3d8786fe9..e2f8175f604 100644 --- a/core/providers/vertex.go +++ b/core/providers/vertex.go @@ -66,7 +66,7 @@ func NewVertexProvider(config *schemas.ProviderConfig, logger schemas.Logger) (* // Pre-warm response pools for range config.ConcurrencyAndBufferSize.Concurrency { - openAIResponsePool.Put(&schemas.BifrostResponse{}) + // openAIResponsePool.Put(&schemas.BifrostResponse{}) anthropicChatResponsePool.Put(&AnthropicChatResponse{}) } @@ -276,8 +276,11 @@ func (provider *VertexProvider) ChatCompletion(ctx context.Context, model string } bifrostResponse.ExtraFields = schemas.BifrostResponseExtraFields{ - Provider: schemas.Vertex, - RawResponse: rawResponse, + Provider: schemas.Vertex, + } + + if provider.sendBackRawResponse { + bifrostResponse.ExtraFields.RawResponse = rawResponse } if params != nil { @@ -287,8 +290,9 @@ func (provider *VertexProvider) ChatCompletion(ctx context.Context, model string return bifrostResponse, nil } else { // Pre-allocate response structs from pools - response := acquireOpenAIResponse() - defer releaseOpenAIResponse(response) + // response := acquireOpenAIResponse() + response := &schemas.BifrostResponse{} + // defer releaseOpenAIResponse(response) // Use enhanced response handler with pre-allocated response rawResponse, bifrostErr := handleProviderResponse(body, response, provider.sendBackRawResponse) @@ -296,27 +300,17 @@ func (provider *VertexProvider) ChatCompletion(ctx context.Context, model string return nil, bifrostErr } - // Create final response - bifrostResponse := &schemas.BifrostResponse{ - ID: response.ID, - Object: response.Object, - Choices: response.Choices, - Model: response.Model, - Created: response.Created, - ServiceTier: response.ServiceTier, - SystemFingerprint: response.SystemFingerprint, - Usage: response.Usage, - ExtraFields: schemas.BifrostResponseExtraFields{ - Provider: schemas.Vertex, - RawResponse: rawResponse, - }, + response.ExtraFields.Provider = schemas.Vertex + + if provider.sendBackRawResponse { + response.ExtraFields.RawResponse = rawResponse } if params != nil { - bifrostResponse.ExtraFields.Params = *params + response.ExtraFields.Params = *params } - return bifrostResponse, nil + return response, nil } } diff --git a/core/schemas/bifrost.go b/core/schemas/bifrost.go index c1ede36bff5..21c43781ad4 100644 --- a/core/schemas/bifrost.go +++ b/core/schemas/bifrost.go @@ -49,6 +49,7 @@ const ( Groq ModelProvider = "groq" SGL ModelProvider = "sgl" Parasail ModelProvider = "parasail" + Cerebras ModelProvider = "cerebras" ) //* Request Structs diff --git a/docs/quickstart/go-package.md b/docs/quickstart/go-package.md index 5c7da476a7a..101c94f3a33 100644 --- a/docs/quickstart/go-package.md +++ b/docs/quickstart/go-package.md @@ -220,7 +220,7 @@ response, bifrostErr := client.ChatCompletionRequest(context.Background(), &sche | What You Want | Where to Go | Time | | ---------------------------- | ------------------------------------------------------- | --------- | | **Complete setup guide** | [๐Ÿ“– Go Package Usage](../usage/go-package/) | 10 min | -| **Add all 10+ providers** | [๐Ÿ”— Providers](../providers.md) | 5 min | +| **Add all 12+ providers** | [๐Ÿ”— Providers](../providers.md) | 5 min | | **Production configuration** | [๐Ÿ‘ค Account Management](../usage/go-package/account.md) | 15 min | | **Custom plugins** | [๐Ÿ”Œ Plugins](../usage/go-package/plugins.md) | 20 min | | **MCP integration** | [๐Ÿ› ๏ธ MCP](../usage/go-package/mcp.md) | 15 min | diff --git a/docs/usage/go-package/README.md b/docs/usage/go-package/README.md index a061dc0db4b..75078d5d673 100644 --- a/docs/usage/go-package/README.md +++ b/docs/usage/go-package/README.md @@ -215,7 +215,7 @@ If you need to use Bifrost from non-Go languages (Python, Node.js, etc.) or in m ### **Production Setup** - [Error Handling](../errors.md) - Error types and recovery patterns -- [Provider Configuration](../providers.md) - All 10+ providers setup +- [Provider Configuration](../providers.md) - All 12+ providers setup ### **Development** diff --git a/docs/usage/go-package/account.md b/docs/usage/go-package/account.md index 58070cec5c8..a32b5707aed 100644 --- a/docs/usage/go-package/account.md +++ b/docs/usage/go-package/account.md @@ -579,7 +579,7 @@ func TestAccountWithBifrost(t *testing.T) { ## ๐Ÿ“š Related Documentation - **[๐Ÿค– Bifrost Client](./bifrost-client.md)** - Using your Account with the client -- **[๐Ÿ”— Provider Configuration](../providers.md)** - Settings for all 10+ providers +- **[๐Ÿ”— Provider Configuration](../providers.md)** - Settings for all 12+ providers - **[๐Ÿ”‘ Key Management](../key-management.md)** - Advanced key rotation and distribution - **[๐ŸŒ HTTP Transport](../../quickstart/http-transport.md)** - JSON-based configuration alternative diff --git a/docs/usage/go-package/schemas.md b/docs/usage/go-package/schemas.md index e1eb4efa3e5..d881b47e456 100644 --- a/docs/usage/go-package/schemas.md +++ b/docs/usage/go-package/schemas.md @@ -801,6 +801,9 @@ providers := []schemas.ModelProvider{ schemas.Mistral, // Mistral models schemas.Ollama, // Local Ollama schemas.Groq, // Groq models + schemas.Parasail, // Parasail models + schemas.SGLang, // SGLang models + schemas.Cerebras, // Cerebras models } // Popular model choices diff --git a/docs/usage/providers.md b/docs/usage/providers.md index 1a8243ff3ba..11bbfeeb522 100644 --- a/docs/usage/providers.md +++ b/docs/usage/providers.md @@ -15,8 +15,9 @@ Multi-provider support with unified API across all AI providers. Switch between | **Mistral** | Mistral Large, Medium, Small | European AI, cost-effective | โœ… | | **Ollama** | Llama, Mistral, CodeLlama | Local deployment, privacy | โœ… | | **Groq** | Mixtral, Llama, Gemma | Enterprise AI platform | โœ… | -| **Parasail** | GPT OSS, Llama, Qwen | Enterprise AI platform | โœ… | -| **SGLang** | Qwen | Enterprise AI platform | โœ… | +| **Parasail** | GPT OSS, Llama, Qwen | Enterprise AI platform | โœ… | +| **SGLang** | Qwen | Enterprise AI platform | โœ… | +| **Cerebras** | Llama 3.3 70B | Enterprise AI platform | โœ… | --- @@ -471,17 +472,17 @@ func (a *MyAccount) GetKeysForProvider(ctx *context.Context, provider schemas.Mo ## ๐Ÿ“‹ Provider Features Matrix -| Feature | OpenAI | Anthropic | Azure | Bedrock | Vertex | Cohere | Mistral | Ollama | Groq | Parasail | SGLang | -| -------------------- | ------ | --------- | ----- | ------- | ------ | ------ | ------- | ------ | ------ | -------- | ------ | -| **Chat Completion** | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | -| **Function Calling** | โœ… | โœ… | โœ… | โœ… | โœ… | โŒ | โœ… | โœ… | โœ… | โœ… | โœ… | -| **Streaming** | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | -| **Vision** | โœ… | โœ… | โœ… | โœ… | โœ… | โŒ | โœ… | โœ… | โŒ | โœ… | โœ… | -| **JSON Mode** | โœ… | โœ… | โœ… | โœ… | โœ… | โŒ | โœ… | โœ… | โœ… | โœ… | โœ… | -| **๐Ÿ”Š Audio Speech** | โœ… | โŒ | โŒ | โŒ | โŒ | โŒ | โŒ | โŒ | โŒ | โŒ | โŒ | -| **๐ŸŽค Transcription** | โœ… | โŒ | โŒ | โŒ | โŒ | โŒ | โŒ | โŒ | โŒ | โŒ | โŒ | -| **Custom Base URL** | โœ… | โœ… | โœ… | โŒ | โŒ | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | -| **Proxy Support** | โœ… | โœ… | โœ… | โŒ | โŒ | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | +| Feature | OpenAI | Anthropic | Azure | Bedrock | Vertex | Cohere | Mistral | Ollama | Groq | Parasail | SGLang | Cerebras | +| -------------------- | ------ | --------- | ----- | ------- | ------ | ------ | ------- | ------ | ------ | -------- | ------ | -------- | +| **Text Completion** | โŒ | โœ… | โœ… | โœ… | โŒ | โŒ | โŒ | โŒ | โŒ | โŒ | โŒ | โœ… | +| **Chat Completion** | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | +| **Function Calling** | โœ… | โœ… | โœ… | โœ… | โœ… | โŒ | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | +| **Streaming** | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | +| **Vision** | โœ… | โœ… | โœ… | โœ… | โœ… | โŒ | โœ… | โœ… | โŒ | โœ… | โœ… | โœ… | +| **๐Ÿ”Š Audio Speech** | โœ… | โŒ | โŒ | โŒ | โŒ | โŒ | โŒ | โŒ | โŒ | โŒ | โŒ | โŒ | +| **๐ŸŽค Transcription** | โœ… | โŒ | โŒ | โŒ | โŒ | โŒ | โŒ | โŒ | โŒ | โŒ | โŒ | โŒ | +| **Custom Base URL** | โœ… | โœ… | โœ… | โŒ | โŒ | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | +| **Proxy Support** | โœ… | โœ… | โœ… | โŒ | โŒ | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | โœ… | ### **๐Ÿ”Š Audio Features Details** @@ -498,6 +499,7 @@ func (a *MyAccount) GetKeysForProvider(ctx *context.Context, provider schemas.Mo | **Groq** | โŒ Not Available | โŒ Not Available | N/A | โŒ N/A | | **Parasail** | โŒ Not Available | โŒ Not Available | N/A | โŒ N/A | | **SGLang** | โŒ Not Available | โŒ Not Available | N/A | โŒ N/A | +| **Cerebras** | โŒ Not Available | โŒ Not Available | N/A | โŒ N/A | > **๐Ÿ“ Note:** Audio features are currently supported only through OpenAI. Other providers return `unsupported operation` errors for audio requests. This allows you to use fallback chains where non-audio requests can still use other providers. diff --git a/tests/core-providers/README.md b/tests/core-providers/README.md index 2b5bbcd97c5..e5f23830860 100644 --- a/tests/core-providers/README.md +++ b/tests/core-providers/README.md @@ -12,8 +12,10 @@ This directory contains comprehensive tests for all Bifrost AI providers, ensuri - **Google Vertex AI** - Google Cloud's AI platform - **Mistral** - Mistral AI models with vision capabilities - **Ollama** - Local LLM serving platform -- **Groq** - Groq models -- **SGLang** - SGLang models +- **Groq** - OSS models +- **SGLang** - OSS models +- **Parasail** - OSS models +- **Cerebras** - Llama, Qwen and GPT-OSS models ## ๐Ÿƒโ€โ™‚๏ธ Running Tests diff --git a/tests/core-providers/cerebras_test.go b/tests/core-providers/cerebras_test.go new file mode 100644 index 00000000000..005ba47a80b --- /dev/null +++ b/tests/core-providers/cerebras_test.go @@ -0,0 +1,41 @@ +package tests + +import ( + "testing" + + "github.com/maximhq/bifrost/tests/core-providers/config" + + "github.com/maximhq/bifrost/core/schemas" +) + +func TestCerebras(t *testing.T) { + client, ctx, cancel, err := config.SetupTest() + if err != nil { + t.Fatalf("Error initializing test setup: %v", err) + } + defer cancel() + defer client.Cleanup() + + testConfig := config.ComprehensiveTestConfig{ + Provider: schemas.Cerebras, + ChatModel: "llama-3.3-70b", + TextModel: "llama3.1-8b", + Scenarios: config.TestScenarios{ + TextCompletion: true, + SimpleChat: true, + ChatCompletionStream: true, + MultiTurnConversation: true, + ToolCalls: true, + MultipleToolCalls: true, + End2EndToolCalling: true, + AutomaticFunctionCall: true, + ImageURL: false, + ImageBase64: false, + MultipleImages: false, + CompleteEnd2End: true, + ProviderSpecific: false, + }, + } + + runAllComprehensiveTests(t, client, ctx, testConfig) +} diff --git a/tests/core-providers/config/account.go b/tests/core-providers/config/account.go index 457969d96b7..90468dd2185 100644 --- a/tests/core-providers/config/account.go +++ b/tests/core-providers/config/account.go @@ -70,6 +70,7 @@ func (account *ComprehensiveTestAccount) GetConfiguredProviders() ([]schemas.Mod schemas.Groq, schemas.SGL, schemas.Parasail, + schemas.Cerebras, }, nil } @@ -168,6 +169,14 @@ func (account *ComprehensiveTestAccount) GetKeysForProvider(ctx *context.Context Weight: 1.0, }, }, nil + case schemas.Cerebras: + return []schemas.Key{ + { + Value: os.Getenv("CEREBRAS_API_KEY"), + Models: []string{}, + Weight: 1.0, + }, + }, nil default: return nil, fmt.Errorf("unsupported provider: %s", providerKey) } @@ -275,6 +284,11 @@ func (account *ComprehensiveTestAccount) GetConfigForProvider(providerKey schema NetworkConfig: schemas.DefaultNetworkConfig, ConcurrencyAndBufferSize: schemas.DefaultConcurrencyAndBufferSize, }, nil + case schemas.Cerebras: + return &schemas.ProviderConfig{ + NetworkConfig: schemas.DefaultNetworkConfig, + ConcurrencyAndBufferSize: schemas.DefaultConcurrencyAndBufferSize, + }, nil default: return nil, fmt.Errorf("unsupported provider: %s", providerKey) } diff --git a/tests/core-providers/go.mod b/tests/core-providers/go.mod index 8fd43b89ed4..a8193e1cb55 100644 --- a/tests/core-providers/go.mod +++ b/tests/core-providers/go.mod @@ -7,6 +7,8 @@ require ( github.com/stretchr/testify v1.10.0 ) +replace github.com/maximhq/bifrost/core => ../../core + require ( cloud.google.com/go/compute/metadata v0.3.0 // indirect github.com/andybalholm/brotli v1.1.1 // indirect @@ -31,7 +33,10 @@ require ( github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/mark3labs/mcp-go v0.32.0 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rs/zerolog v1.34.0 // indirect github.com/spf13/cast v1.7.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect @@ -40,6 +45,7 @@ require ( golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect golang.org/x/net v0.39.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect + golang.org/x/sys v0.32.0 // indirect golang.org/x/text v0.24.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/tests/core-providers/go.sum b/tests/core-providers/go.sum index 221db204a5c..407b7957df0 100644 --- a/tests/core-providers/go.sum +++ b/tests/core-providers/go.sum @@ -36,11 +36,13 @@ github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFos github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -56,12 +58,19 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mark3labs/mcp-go v0.32.0 h1:fgwmbfL2gbd67obg57OfV2Dnrhs1HtSdlY/i5fn7MU8= github.com/mark3labs/mcp-go v0.32.0/go.mod h1:rXqOudj/djTORU/ThxYx8fqEVj/5pvTuuebQ2RC7uk4= -github.com/maximhq/bifrost/core v1.1.14 h1:ALPXK3GkhvinXD/NNJ5aZ51F65o0TVMMgwsA23fE8Og= -github.com/maximhq/bifrost/core v1.1.14/go.mod h1:Wa/BtJoHZ0+RXYomGeAL+wyBu6iD1h6vMiUHF5RTlkA= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= +github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -89,6 +98,11 @@ golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= diff --git a/transports/README.md b/transports/README.md index 1c32554d208..bfc3ecd820c 100644 --- a/transports/README.md +++ b/transports/README.md @@ -52,7 +52,7 @@ docker run -p 8080:8080 -v $(pwd)/data:/app/data maximhq/bifrost | Feature | Description | Learn More | | ----------------------------- | ------------------------------------------------------------------- | ---------------------------------------------------------- | | **๐Ÿ–ฅ๏ธ Built-in Web UI** | Visual configuration, live monitoring, request logs, and analytics | Open `http://localhost:8080` after startup | -| **๐Ÿ”„ Multi-Provider Support** | OpenAI, Anthropic, Azure, Bedrock, Vertex, Cohere, Mistral, Ollama, Groq, Parasail, SGLang | [Provider Setup](../docs/usage/providers.md) | +| **๐Ÿ”„ Multi-Provider Support** | OpenAI, Anthropic, Azure, Bedrock, Vertex, Cohere, Mistral, Ollama, Groq, Parasail, SGLang, Cerebras | [Provider Setup](../docs/usage/providers.md) | | **๐Ÿ”Œ Drop-in Compatibility** | Replace OpenAI/Anthropic/GenAI APIs with zero code changes | [Integrations](../docs/usage/http-transport/integrations/) | | **๐Ÿ› ๏ธ MCP Tool Calling** | Enable AI models to use external tools (filesystem, web, databases) | [MCP Guide](../docs/mcp.md) | | **โšก Plugin System** | Add analytics, caching, rate limiting, custom logic | [Plugin System](../docs/plugins.md) | @@ -292,7 +292,7 @@ docker run -p 8080:8080 -e APP_HOST=192.168.1.100 maximhq/bifrost ### ๐Ÿš€ Core Features -- **[๐Ÿ”— Multi-Provider Support](../docs/usage/providers.md)** - 10+ AI providers with fallbacks +- **[๐Ÿ”— Multi-Provider Support](../docs/usage/providers.md)** - 12+ AI providers with fallbacks - **[๐Ÿ› ๏ธ MCP Integration](../docs/mcp.md)** - External tool calling for AI models - **[๐Ÿ”Œ Plugin System](../docs/plugins.md)** - Extensible middleware architecture diff --git a/transports/bifrost-http/integrations/utils.go b/transports/bifrost-http/integrations/utils.go index bcb31441646..d5250e5d77f 100644 --- a/transports/bifrost-http/integrations/utils.go +++ b/transports/bifrost-http/integrations/utils.go @@ -642,6 +642,7 @@ var ValidProviders = map[schemas.ModelProvider]bool{ schemas.Groq: true, schemas.SGL: true, schemas.Parasail: true, + schemas.Cerebras: true, } // ParseModelString extracts provider and model from a model string. diff --git a/transports/go.mod b/transports/go.mod index 4e61df1c5e3..3b63d6d4984 100644 --- a/transports/go.mod +++ b/transports/go.mod @@ -17,6 +17,8 @@ require ( gorm.io/gorm v1.30.0 ) +replace github.com/maximhq/bifrost/core => ../core + require ( cloud.google.com/go v0.121.0 // indirect cloud.google.com/go/auth v0.16.0 // indirect diff --git a/ui/README.md b/ui/README.md index caec46b14d6..cd3b6bbcd7c 100644 --- a/ui/README.md +++ b/ui/README.md @@ -81,7 +81,7 @@ The main dashboard provides comprehensive request monitoring: Manage all your AI providers from a unified interface: -- **Supported Providers**: OpenAI, Azure OpenAI, Anthropic, AWS Bedrock, Cohere, Google Vertex AI, Mistral, Ollama, Groq, Parasail, SGLang +- **Supported Providers**: OpenAI, Azure OpenAI, Anthropic, AWS Bedrock, Cohere, Google Vertex AI, Mistral, Ollama, Groq, Parasail, SGLang, Cerebras - **Key Management**: Multiple API keys with weights and model assignments - **Network Configuration**: Custom base URLs, timeouts, retry policies, proxy settings - **Provider-specific Settings**: Azure deployments, Bedrock regions, Vertex projects diff --git a/ui/app/layout.tsx b/ui/app/layout.tsx index 0e406bf64ca..c1c72157645 100644 --- a/ui/app/layout.tsx +++ b/ui/app/layout.tsx @@ -21,7 +21,7 @@ const geistMono = Geist_Mono({ export const metadata: Metadata = { title: 'Bifrost - The fastest LLM gateway', description: - 'Production-ready fastest LLM gateway that connects to 10+ providers through a single API. Get automatic failover, load balancing, mcp support and zero-downtime deployments.', + 'Production-ready fastest LLM gateway that connects to 12+ providers through a single API. Get automatic failover, load balancing, mcp support and zero-downtime deployments.', } export default function RootLayout({ children }: { children: React.ReactNode }) { diff --git a/ui/lib/constants/icons.tsx b/ui/lib/constants/icons.tsx index 36e903bf455..7f452e6a6fc 100644 --- a/ui/lib/constants/icons.tsx +++ b/ui/lib/constants/icons.tsx @@ -125,6 +125,51 @@ export const ProviderIcons = { ) }, + cerebras: ({ size = 'md', className = '' }: IconProps) => { + const { resolvedTheme } = useTheme() + const resolvedSize = resolveSize(size) + + return resolvedTheme === 'light' ? ( + + Cerebras + + + + ) : ( + + Cerebras + + + + ) + }, + cohere: ({ size = 'md', className = '' }: IconProps) => { const resolvedSize = resolveSize(size) return ( @@ -312,9 +357,14 @@ export const ProviderIcons = { xmlns="http://www.w3.org/2000/svg" className={className} > - - - + + + ) }, diff --git a/ui/lib/constants/logs.ts b/ui/lib/constants/logs.ts index 91bf2b3207e..73bd98f539d 100644 --- a/ui/lib/constants/logs.ts +++ b/ui/lib/constants/logs.ts @@ -1,4 +1,17 @@ -export const PROVIDERS = ['openai', 'anthropic', 'azure', 'bedrock', 'cohere', 'vertex', 'mistral', 'ollama', 'groq', 'parasail', 'sgl'] as const +export const PROVIDERS = [ + 'openai', + 'anthropic', + 'azure', + 'bedrock', + 'cohere', + 'vertex', + 'mistral', + 'ollama', + 'groq', + 'parasail', + 'sgl', + 'cerebras', +] as const export const STATUSES = ['success', 'error', 'processing', 'cancelled'] as const @@ -26,6 +39,7 @@ export const PROVIDER_LABELS = { groq: 'Groq', parasail: 'Parasail', sgl: 'SGLang', + cerebras: 'Cerebras', } as const export const STATUS_COLORS = { diff --git a/ui/lib/types/config.ts b/ui/lib/types/config.ts index 6aea6dc6031..2452aba9a51 100644 --- a/ui/lib/types/config.ts +++ b/ui/lib/types/config.ts @@ -1,7 +1,19 @@ // Configuration types that match the Go backend structures // ModelProvider enum matching Go's schemas.ModelProvider -export type ModelProvider = 'openai' | 'azure' | 'anthropic' | 'bedrock' | 'cohere' | 'vertex' | 'mistral' | 'ollama' | 'groq' | 'parasail' | 'sgl' +export type ModelProvider = + | 'openai' + | 'azure' + | 'anthropic' + | 'bedrock' + | 'cohere' + | 'vertex' + | 'mistral' + | 'ollama' + | 'groq' + | 'parasail' + | 'sgl' + | 'cerebras' // AzureKeyConfig matching Go's schemas.AzureKeyConfig export interface AzureKeyConfig {