diff --git a/README.md b/README.md index 1d4462ecce..aab6d34d77 100644 --- a/README.md +++ b/README.md @@ -115,10 +115,8 @@ For additional configurations in HTTP server setup, please read [this](https://g func (baseAccount *BaseAccount) GetConfigForProvider(providerKey schemas.ModelProvider) (*schemas.ProviderConfig, error) { return &schemas.ProviderConfig{ - ConcurrencyAndBufferSize: schemas.ConcurrencyAndBufferSize{ - Concurrency: 3, - BufferSize: 10, - }, + NetworkConfig: schemas.DefaultNetworkConfig, + ConcurrencyAndBufferSize: schemas.DefaultConcurrencyAndBufferSize, }, nil } ``` @@ -246,7 +244,9 @@ Bifrost is divided into three Go packages: core, plugins, and transports. 2. **plugins**: This package serves as an extension to core. You can download individual packages using `go get github.com/maximhq/bifrost/plugins/{plugin-name}` and pass the plugins while initializing Bifrost. ```golang -plugin, err := plugins.NewMaximLoggerPlugin(os.Getenv("MAXIM_API_KEY"), os.Getenv("MAXIM_LOGGER_ID")) +// go get github.com/maximhq/bifrost/plugins/maxim + +maximPlugin, err := maxim.NewMaximLoggerPlugin(os.Getenv("MAXIM_API_KEY"), os.Getenv("MAXIM_LOGGER_ID")) if err != nil { return nil, err } @@ -254,7 +254,7 @@ if err != nil { // Initialize Bifrost client, err := bifrost.Init(schemas.BifrostConfig{ Account: &account, - Plugins: []schemas.Plugin{plugin}, + Plugins: []schemas.Plugin{maximPlugin}, }) ``` diff --git a/core/schemas/provider.go b/core/schemas/provider.go index 25631887d0..31840a5dcf 100644 --- a/core/schemas/provider.go +++ b/core/schemas/provider.go @@ -30,6 +30,14 @@ type NetworkConfig struct { RetryBackoffMax time.Duration `json:"retry_backoff_max"` // Maximum backoff duration } +// DefaultNetworkConfig is the default network configuration for provider connections. +var DefaultNetworkConfig = NetworkConfig{ + DefaultRequestTimeoutInSeconds: DefaultRequestTimeoutInSeconds, + MaxRetries: DefaultMaxRetries, + RetryBackoffInitial: DefaultRetryBackoffInitial, + RetryBackoffMax: DefaultRetryBackoffMax, +} + // MetaConfig defines the interface for provider-specific configuration. // Check /meta folder for implemented provider-specific meta configurations. type MetaConfig interface { @@ -61,6 +69,12 @@ type ConcurrencyAndBufferSize struct { BufferSize int `json:"buffer_size"` // Size of the buffer } +// DefaultConcurrencyAndBufferSize is the default concurrency and buffer size for provider operations. +var DefaultConcurrencyAndBufferSize = ConcurrencyAndBufferSize{ + Concurrency: DefaultConcurrency, + BufferSize: DefaultBufferSize, +} + // ProxyType defines the type of proxy to use for connections. type ProxyType string diff --git a/core/tests/account.go b/core/tests/account.go index 4a33ea66d7..eb34e27e07 100644 --- a/core/tests/account.go +++ b/core/tests/account.go @@ -136,16 +136,8 @@ func (baseAccount *BaseAccount) GetConfigForProvider(providerKey schemas.ModelPr }, nil case schemas.Anthropic: return &schemas.ProviderConfig{ - NetworkConfig: schemas.NetworkConfig{ - DefaultRequestTimeoutInSeconds: 30, - MaxRetries: 1, - RetryBackoffInitial: 100 * time.Millisecond, - RetryBackoffMax: 2 * time.Second, - }, - ConcurrencyAndBufferSize: schemas.ConcurrencyAndBufferSize{ - Concurrency: 3, - BufferSize: 10, - }, + NetworkConfig: schemas.DefaultNetworkConfig, + ConcurrencyAndBufferSize: schemas.DefaultConcurrencyAndBufferSize, }, nil case schemas.Bedrock: return &schemas.ProviderConfig{ @@ -166,16 +158,8 @@ func (baseAccount *BaseAccount) GetConfigForProvider(providerKey schemas.ModelPr }, nil case schemas.Cohere: return &schemas.ProviderConfig{ - NetworkConfig: schemas.NetworkConfig{ - DefaultRequestTimeoutInSeconds: 30, - MaxRetries: 1, - RetryBackoffInitial: 100 * time.Millisecond, - RetryBackoffMax: 2 * time.Second, - }, - ConcurrencyAndBufferSize: schemas.ConcurrencyAndBufferSize{ - Concurrency: 3, - BufferSize: 10, - }, + NetworkConfig: schemas.DefaultNetworkConfig, + ConcurrencyAndBufferSize: schemas.DefaultConcurrencyAndBufferSize, }, nil case schemas.Azure: return &schemas.ProviderConfig{ diff --git a/plugins/maxim/README.md b/plugins/maxim/README.md new file mode 100644 index 0000000000..a976949cc3 --- /dev/null +++ b/plugins/maxim/README.md @@ -0,0 +1,85 @@ +# Maxim-SDK Plugin for Bifrost + +This plugin integrates the Maxim SDK into Bifrost, enabling seamless observability and evaluation of LLM interactions. It captures and forwards inputs/outputs from Bifrost to the Maxim's observability platform. This facilitates end-to-end tracing, evaluation, and monitoring of your LLM-based application. + +## Usage for Bifrost Go Package + +1. Download the Plugin + +```bash +go get github.com/maximhq/bifrost/plugins/maxim +``` + +2. Initialise the Plugin + +```go + maximPlugin, err := maxim.NewMaximLoggerPlugin("your_maxim_api_key", "your_maxim_log_repo_id") + if err != nil { + return nil, err + } +``` + +3. Pass to plugin to bifrost + +```go + client, err := bifrost.Init(schemas.BifrostConfig{ + Account: &yourAccount, + Plugins: []schemas.Plugin{maximPlugin}, + }) +``` + +## Usage for Bifrost HTTP Transport + +1. Set up the environment variables + +```bash +export MAXIM_API_KEY=your_maxim_api_key +``` + +2. Setup flags to use the plugin + Use include `maxim` in `--plugins` and your maxim log repo id in `--maxim-log-repo-id"` + + eg. `bifrost-http -config config.json -env .env -plugins maxim -maxim-log-repo-id your_maxim_log_repo_id` + + For docker build + + ```bash + docker build \ + --build-arg CONFIG_PATH=./config.example.json \ + --build-arg ENV_PATH=./.env.sample \ + --build-arg PORT=8080 \ + --build-arg POOL_SIZE=300 \ + --build-arg PLUGINS=maxim \ + --build-arg MAXIM_LOG_REPO_ID=your_maxim_log_repo_id \ + -t bifrost-transports . + ``` + +## Viewing Your Traces + +1. Log in to your [Maxim Dashboard](https://getmaxim.ai/dashboard) +2. Navigate to your repository +3. View detailed llm traces, including: + - LLM inputs/outputs + - Tool usage patterns + - Performance metrics + - Cost analytics + +## Additional Features + +The plugin also supports custom `trace-id` and `generation-id` if the uses wish to log the generations to their custom logging implementation. To use it, just pass your trace id to the passed request context with the key `trace-id`, and similarly to `generation-id` for generation id. In these cases no new trace/generation is created and the output is just logged to your provided generation. + +eg. + +```go + ctx = context.WithValue(ctx, "generation-id", "123") + + result, err := bifrostClient.ChatCompletionRequest(schemas.OpenAI, &schemas.BifrostRequest{ + Model: "gpt-4o", + Input: schemas.RequestInput{ + ChatCompletionInput: &messages, + }, + Params: ¶ms, + }, ctx) +``` + +HTTP transport offers out of the box support for this feature(when maxim plugin is used), just pass `x-bf-maxim-trace-id` of `x-bf-maxim-generation-id` header with your request to use this feature. diff --git a/transports/Dockerfile b/transports/Dockerfile index 27f726fcba..d7c2e35660 100644 --- a/transports/Dockerfile +++ b/transports/Dockerfile @@ -38,6 +38,7 @@ ARG POOL_SIZE ARG DROP_EXCESS_REQUESTS ARG PLUGINS ARG MAXIM_LOG_REPO_ID +ARG PROMETHEUS_LABELS # Set default values if args are not provided ENV APP_PORT=${PORT:-8080} @@ -45,6 +46,7 @@ ENV APP_POOL_SIZE=${POOL_SIZE:-300} ENV APP_DROP_EXCESS_REQUESTS=${DROP_EXCESS_REQUESTS:-false} ENV APP_PLUGINS=${PLUGINS:-""} ENV APP_MAXIM_LOG_REPO_ID=${MAXIM_LOG_REPO_ID:-""} +ENV APP_PROMETHEUS_LABELS=${PROMETHEUS_LABELS:-""} # Copy the config and environment files into the image COPY ${CONFIG_PATH} /app/config/config.json @@ -55,7 +57,7 @@ RUN echo '#!/bin/sh' > /app/entrypoint.sh && \ echo 'if [ ! -f /app/config/config.json ]; then echo "Missing config.json"; exit 1; fi' >> /app/entrypoint.sh && \ echo 'if [ ! -f /app/config/.env ]; then echo "Missing .env"; exit 1; fi' >> /app/entrypoint.sh && \ echo 'if [ ! -f /app/main ]; then echo "Missing main binary"; exit 1; fi' >> /app/entrypoint.sh && \ - echo 'exec /app/main -config /app/config/config.json -env /app/config/.env -port "$APP_PORT" -pool-size "$APP_POOL_SIZE" -drop-excess-requests "$APP_DROP_EXCESS_REQUESTS" -plugins "$APP_PLUGINS" -maxim-log-repo-id "$APP_MAXIM_LOG_REPO_ID"' >> /app/entrypoint.sh && \ + echo 'exec /app/main -config /app/config/config.json -env /app/config/.env -port "$APP_PORT" -pool-size "$APP_POOL_SIZE" -drop-excess-requests "$APP_DROP_EXCESS_REQUESTS" -plugins "$APP_PLUGINS" -maxim-log-repo-id "$APP_MAXIM_LOG_REPO_ID" -prometheus-labels "$APP_PROMETHEUS_LABELS"' >> /app/entrypoint.sh && \ chmod +x /app/entrypoint.sh # Expose the port defined by argument diff --git a/transports/README.md b/transports/README.md index 3da747720b..eec2234304 100644 --- a/transports/README.md +++ b/transports/README.md @@ -15,6 +15,8 @@ This package contains clients for various transports that can be used to spin up - [Text Completions](#text-completions) - [Chat Completions](#chat-completions) - [🔧 Advanced Features](#-advanced-features) + - [Prometheus Support](#prometheus-support) + - [Plugin Support](#plugin-support) - [Fallbacks](#fallbacks) --- @@ -165,6 +167,22 @@ curl -X POST http://localhost:8080/v1/chat/completions \ ## 🔧 Advanced Features +### Prometheus Support + +HTTP transport supports Prometheus out of the box. By default all the metrics are available at `/metrics` endpoint. It providers metrics for httpRequestsTotal, httpRequestDuration, httpRequestSizeBytes, httpResponseSizeBytes, bifrostUpstreamRequestsTotal, and bifrostUpstreamLatencySeconds. To add custom labels to these metrics using can pass a flag of `-prometheus-labels` while running the http transport. + +For eg. `-prometheus-labels team-id,task-id,location` + +Values for labels are then picked up from the HTTP request headers with the prefix `x-bf-prom-`. + +### Plugin Support + +You can explore the available plugins [here](https://github.com/maximhq/bifrost/tree/main/plugins). And to attached these plugins to your HTTP transport, just pass the flag `-plugins`. + +For eg. `-plugins maxim` + +Note: Please check plugin specific documentations (github.com/maximhq/bifrost/tree/main/plugins/{plugin_name}) for more nuanced control. + ### Fallbacks Configure fallback options in your requests: