diff --git a/core/bifrost.go b/core/bifrost.go index ba08bd595b..40cc6219b2 100644 --- a/core/bifrost.go +++ b/core/bifrost.go @@ -71,19 +71,19 @@ func NewPluginPipeline(plugins []schemas.Plugin, logger schemas.Logger) *PluginP } } -// RunPreHooks executes PreHooks in order, tracks how many ran, and returns the final request, any short-circuit response, and the count. -func (p *PluginPipeline) RunPreHooks(ctx *context.Context, req *schemas.BifrostRequest) (*schemas.BifrostRequest, *schemas.BifrostResponse, int) { - var resp *schemas.BifrostResponse +// RunPreHooks executes PreHooks in order, tracks how many ran, and returns the final request, any short-circuit decision, and the count. +func (p *PluginPipeline) RunPreHooks(ctx *context.Context, req *schemas.BifrostRequest) (*schemas.BifrostRequest, *schemas.PluginShortCircuit, int) { + var shortCircuit *schemas.PluginShortCircuit var err error for i, plugin := range p.plugins { - req, resp, err = plugin.PreHook(ctx, req) + req, shortCircuit, err = plugin.PreHook(ctx, req) if err != nil { p.preHookErrors = append(p.preHookErrors, err) p.logger.Warn(fmt.Sprintf("Error in PreHook for plugin %s: %v", plugin.GetName(), err)) } p.executedPreHooks = i + 1 - if resp != nil { - return req, resp, p.executedPreHooks // short-circuit: only plugins up to and including i ran + if shortCircuit != nil { + return req, shortCircuit, p.executedPreHooks // short-circuit: only plugins up to and including i ran } } return req, nil, p.executedPreHooks @@ -571,7 +571,14 @@ func (bifrost *Bifrost) TextCompletionRequest(ctx context.Context, req *schemas. return nil, primaryErr } + // Check if this is a short-circuit error that doesn't allow fallbacks + // Note: AllowFallbacks = nil is treated as true (allow fallbacks by default) + if primaryErr.AllowFallbacks != nil && !*primaryErr.AllowFallbacks { + return nil, primaryErr + } + // If primary provider failed and we have fallbacks, try them in order + // This includes both regular provider errors and plugin short-circuit errors with AllowFallbacks=true/nil if len(req.Fallbacks) > 0 { for _, fallback := range req.Fallbacks { // Check if we have config for this fallback provider @@ -618,14 +625,24 @@ func (bifrost *Bifrost) tryTextCompletion(req *schemas.BifrostRequest, ctx conte } pipeline := NewPluginPipeline(bifrost.plugins, bifrost.logger) - preReq, preResp, preCount := pipeline.RunPreHooks(&ctx, req) - if preResp != nil { - resp, bifrostErr := pipeline.RunPostHooks(&ctx, preResp, nil, preCount) - // If PostHooks recovered from error, return resp; if not, return error - if bifrostErr != nil { - return nil, bifrostErr + preReq, shortCircuit, preCount := pipeline.RunPreHooks(&ctx, req) + if shortCircuit != nil { + // Handle short-circuit with response (success case) + if shortCircuit.Response != nil { + resp, bifrostErr := pipeline.RunPostHooks(&ctx, shortCircuit.Response, nil, preCount) + if bifrostErr != nil { + return nil, bifrostErr + } + return resp, nil + } + // Handle short-circuit with error + if shortCircuit.Error != nil { + resp, bifrostErr := pipeline.RunPostHooks(&ctx, nil, shortCircuit.Error, preCount) + if bifrostErr != nil { + return nil, bifrostErr + } + return resp, nil } - return resp, nil } if preReq == nil { return nil, newBifrostErrorFromMsg("bifrost request after plugin hooks cannot be nil") @@ -702,7 +719,14 @@ func (bifrost *Bifrost) ChatCompletionRequest(ctx context.Context, req *schemas. return primaryResult, nil } + // Check if this is a short-circuit error that doesn't allow fallbacks + // Note: AllowFallbacks = nil is treated as true (allow fallbacks by default) + if primaryErr.AllowFallbacks != nil && !*primaryErr.AllowFallbacks { + return nil, primaryErr + } + // If primary provider failed and we have fallbacks, try them in order + // This includes both regular provider errors and plugin short-circuit errors with AllowFallbacks=true/nil if len(req.Fallbacks) > 0 { for _, fallback := range req.Fallbacks { // Check if we have config for this fallback provider @@ -749,13 +773,24 @@ func (bifrost *Bifrost) tryChatCompletion(req *schemas.BifrostRequest, ctx conte } pipeline := NewPluginPipeline(bifrost.plugins, bifrost.logger) - preReq, preResp, preCount := pipeline.RunPreHooks(&ctx, req) - if preResp != nil { - resp, bifrostErr := pipeline.RunPostHooks(&ctx, preResp, nil, preCount) - if bifrostErr != nil { - return nil, bifrostErr + preReq, shortCircuit, preCount := pipeline.RunPreHooks(&ctx, req) + if shortCircuit != nil { + // Handle short-circuit with response (success case) + if shortCircuit.Response != nil { + resp, bifrostErr := pipeline.RunPostHooks(&ctx, shortCircuit.Response, nil, preCount) + if bifrostErr != nil { + return nil, bifrostErr + } + return resp, nil + } + // Handle short-circuit with error + if shortCircuit.Error != nil { + resp, bifrostErr := pipeline.RunPostHooks(&ctx, nil, shortCircuit.Error, preCount) + if bifrostErr != nil { + return nil, bifrostErr + } + return resp, nil } - return resp, nil } if preReq == nil { return nil, newBifrostErrorFromMsg("bifrost request after plugin hooks cannot be nil") diff --git a/core/schemas/bifrost.go b/core/schemas/bifrost.go index f5dace06c7..7fb0320ae3 100644 --- a/core/schemas/bifrost.go +++ b/core/schemas/bifrost.go @@ -415,12 +415,18 @@ const ( ) // BifrostError represents an error from the Bifrost system. +// +// PLUGIN DEVELOPERS: When creating BifrostError in PreHook or PostHook, you can set AllowFallbacks: +// - AllowFallbacks = &true: Bifrost will try fallback providers if available +// - AllowFallbacks = &false: Bifrost will return this error immediately, no fallbacks +// - AllowFallbacks = nil: Treated as true by default (fallbacks allowed for resilience) type BifrostError struct { EventID *string `json:"event_id,omitempty"` Type *string `json:"type,omitempty"` IsBifrostError bool `json:"is_bifrost_error"` StatusCode *int `json:"status_code,omitempty"` Error ErrorField `json:"error"` + AllowFallbacks *bool `json:"allow_fallbacks,omitempty"` // Optional: Controls fallback behavior (nil = true by default) } // ErrorField represents detailed error information. diff --git a/core/schemas/plugin.go b/core/schemas/plugin.go index 207f3d290a..572ecf05eb 100644 --- a/core/schemas/plugin.go +++ b/core/schemas/plugin.go @@ -3,6 +3,13 @@ package schemas import "context" +// PluginShortCircuit represents a plugin's decision to short-circuit the normal flow. +// It can contain either a response (success short-circuit) or an error (error short-circuit). +type PluginShortCircuit struct { + Response *BifrostResponse // If set, short-circuit with this response (skips provider call) + Error *BifrostError // If set, short-circuit with this error (can set AllowFallbacks field) +} + // Plugin defines the interface for Bifrost plugins. // Plugins can intercept and modify requests and responses at different stages // of the processing pipeline. @@ -21,9 +28,15 @@ import "context" // - PreHook and PostHook can both modify the request/response and the error. Plugins can recover from errors (set error to nil and provide a response), or invalidate a response (set response to nil and provide an error). // - PostHook is always called with both the current response and error, and should handle either being nil. // - Only truly empty errors (no message, no error, no status code, no type) are treated as recoveries by the pipeline. -// - If a PreHook returns a response, the provider call is skipped and only the PostHook methods of plugins that had their PreHook executed are called in reverse order. +// - If a PreHook returns a PluginShortCircuit, the provider call may be skipped and only the PostHook methods of plugins that had their PreHook executed are called in reverse order. // - The plugin pipeline ensures symmetry: for every PreHook executed, the corresponding PostHook will be called in reverse order. // +// IMPORTANT: When returning BifrostError from PreHook or PostHook: +// - You can set the AllowFallbacks field to control fallback behavior +// - AllowFallbacks = &true: Allow Bifrost to try fallback providers +// - AllowFallbacks = &false: Do not try fallbacks, return error immediately +// - AllowFallbacks = nil: Treated as true by default (allow fallbacks for resilience) +// // Plugin authors should ensure their hooks are robust to both response and error being nil, and should not assume either is always present. type Plugin interface { @@ -33,9 +46,8 @@ type Plugin interface { // PreHook is called before a request is processed by a provider. // It allows plugins to modify the request before it is sent to the provider. // The context parameter can be used to maintain state across plugin calls. - // Returns the modified request, an optional response (if the plugin wants to short-circuit the provider call), and any error that occurred during processing. - // If a response is returned, the provider call is skipped and only the PostHook methods of plugins that had their PreHook executed are called in reverse order. - PreHook(ctx *context.Context, req *BifrostRequest) (*BifrostRequest, *BifrostResponse, error) + // Returns the modified request, an optional short-circuit decision, and any error that occurred during processing. + PreHook(ctx *context.Context, req *BifrostRequest) (*BifrostRequest, *PluginShortCircuit, error) // PostHook is called after a response is received from a provider or a PreHook short-circuit. // It allows plugins to modify the response and/or error before it is returned to the caller. diff --git a/docs/plugins.md b/docs/plugins.md index 1d7dc29635..a7e796092a 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -1,279 +1,729 @@ # Bifrost Plugin System -Bifrost provides a powerful plugin system that allows you to extend and customize the request/response pipeline. Plugins can be used to implement various functionalities like rate limiting, caching, logging, monitoring, and more. +Bifrost provides a powerful plugin system that allows you to extend and customize the request/response pipeline. Plugins can implement rate limiting, caching, authentication, logging, monitoring, and more. -## 1. How Plugins Work +## Table of Contents -Plugins in Bifrost follow a flexible interface that allows them to intercept and modify requests and responses at different stages of processing: +1. [Plugin Architecture Overview](#1-plugin-architecture-overview) +2. [Plugin Interface](#2-plugin-interface) +3. [Plugin Lifecycle](#3-plugin-lifecycle) +4. [Plugin Execution Flow](#4-plugin-execution-flow) +5. [Short-Circuit Behavior](#5-short-circuit-behavior) +6. [Error Handling & Fallbacks](#6-error-handling--fallbacks) +7. [Building Custom Plugins](#7-building-custom-plugins) +8. [Plugin Examples](#8-plugin-examples) +9. [Best Practices](#9-best-practices) +10. [Plugin Development Guidelines](#10-plugin-development-guidelines) +11. [Troubleshooting Guide](#11-troubleshooting-guide) +12. [Performance Optimization](#12-performance-optimization) -1. **PreHook**: Executed before a request is sent to a provider +## 1. Plugin Architecture Overview - - Can modify the request - - Can add custom headers or parameters - - Can implement rate limiting or validation - - Executed in the order they are registered - - If a PreHook returns a response, the provider call is skipped and only the PostHook methods of plugins that had their PreHook executed are called in reverse order. +Bifrost plugins follow a **PreHook → Provider → PostHook** pattern with support for short-circuiting and fallback control. -2. **PostHook**: Executed after receiving a response from a provider or a PreHook short-circuit - - Can modify the response and/or error - - Can recover from errors (set error to nil and provide a response) - - Can invalidate a response (set response to nil and provide an error) - - Both response and error may be nil; plugins must handle both cases - - Executed in reverse order of PreHooks - - Only truly empty errors (no message, no error, no status code, no type) are treated as recoveries by the pipeline +### Key Concepts -> **Note**: The plugin pipeline ensures symmetry: for every PreHook executed, the corresponding PostHook will be called in reverse order. Plugin authors should ensure their hooks are robust to both response and error being nil, and should not assume either is always present. +- **PreHook**: Executed before provider call - can modify requests or short-circuit +- **PostHook**: Executed after provider response - can modify responses or recover from errors +- **Short-Circuit**: Plugin can skip provider call and return response/error directly +- **Fallback Control**: Plugins can control whether fallback providers should be tried +- **Pipeline Symmetry**: Every PreHook execution gets a corresponding PostHook call ## 2. Plugin Interface ```go -// Plugin interface for Bifrost plugins -// See core/schemas/plugin.go for the authoritative definition +type Plugin interface { + // GetName returns the name of the plugin + GetName() string -GetName() string -PreHook(ctx *context.Context, req *BifrostRequest) (*BifrostRequest, *BifrostResponse, error) -PostHook(ctx *context.Context, result *BifrostResponse, err *BifrostError) (*BifrostResponse, *BifrostError, error) -Cleanup() error + // PreHook is called before a request is processed by a provider + // Can modify request, short-circuit with response, or short-circuit with error + PreHook(ctx *context.Context, req *BifrostRequest) (*BifrostRequest, *PluginShortCircuit, error) + + // PostHook is called after a response or after PreHook short-circuit + // Can modify response/error or recover from errors + PostHook(ctx *context.Context, result *BifrostResponse, err *BifrostError) (*BifrostResponse, *BifrostError, error) + + // Cleanup is called on bifrost shutdown + Cleanup() error +} + +type PluginShortCircuit struct { + Response *BifrostResponse // If set, skip provider and return this response + Error *BifrostError // If set, skip provider and return this error +} ``` -## 3. Building Custom Plugins +## 3. Plugin Lifecycle -### Basic Plugin Structure +```mermaid +stateDiagram-v2 + [*] --> PluginInit: Plugin Creation + PluginInit --> Registered: Add to BifrostConfig + Registered --> PreHookCall: Request Received + + PreHookCall --> ModifyRequest: Normal Flow + PreHookCall --> ShortCircuitResponse: Return Response + PreHookCall --> ShortCircuitError: Return Error + + ModifyRequest --> ProviderCall: Send to Provider + ProviderCall --> PostHookCall: Receive Response + + ShortCircuitResponse --> PostHookCall: Skip Provider + ShortCircuitError --> PostHookCall: Pipeline Symmetry + + PostHookCall --> ModifyResponse: Process Result + PostHookCall --> RecoverError: Error Recovery + PostHookCall --> FallbackCheck: Check AllowFallbacks + PostHookCall --> ResponseReady: Pass Through + + FallbackCheck --> TryFallback: AllowFallbacks=true/nil + FallbackCheck --> ResponseReady: AllowFallbacks=false + TryFallback --> PreHookCall: Next Provider + + ModifyResponse --> ResponseReady: Modified + RecoverError --> ResponseReady: Recovered + ResponseReady --> [*]: Return to Client + + Registered --> CleanupCall: Bifrost Shutdown + CleanupCall --> [*]: Plugin Destroyed +``` + +## 4. Plugin Execution Flow + +### Normal Flow (No Short-Circuit) + +```mermaid +sequenceDiagram + participant Client + participant Bifrost + participant Plugin1 + participant Plugin2 + participant Provider + + Client->>Bifrost: Request + Bifrost->>Plugin1: PreHook(request) + Plugin1-->>Bifrost: modified request + Bifrost->>Plugin2: PreHook(request) + Plugin2-->>Bifrost: modified request + Bifrost->>Provider: API Call + Provider-->>Bifrost: response + Bifrost->>Plugin2: PostHook(response) + Plugin2-->>Bifrost: modified response + Bifrost->>Plugin1: PostHook(response) + Plugin1-->>Bifrost: modified response + Bifrost-->>Client: Final Response +``` + +### With Short-Circuit Response + +```mermaid +sequenceDiagram + participant Client + participant Bifrost + participant Plugin1 + participant Plugin2 + participant Provider + + Client->>Bifrost: Request + Bifrost->>Plugin1: PreHook(request) + Plugin1-->>Bifrost: PluginShortCircuit{Response} + Note over Provider: Provider call skipped + Bifrost->>Plugin1: PostHook(response) + Plugin1-->>Bifrost: modified response + Bifrost-->>Client: Final Response +``` + +### With Short-Circuit Error (Allow Fallbacks) + +```mermaid +sequenceDiagram + participant Client + participant Bifrost + participant Plugin1 + participant Provider1 + participant Provider2 + + Client->>Bifrost: Request (Provider1 + Fallback Provider2) + Bifrost->>Plugin1: PreHook(request) + Plugin1-->>Bifrost: PluginShortCircuit{Error, AllowFallbacks=true} + Note over Provider1: Provider1 call skipped + Bifrost->>Plugin1: PostHook(error) + Plugin1-->>Bifrost: error unchanged + + Note over Bifrost: Try fallback provider + Bifrost->>Plugin1: PreHook(request for Provider2) + Plugin1-->>Bifrost: modified request + Bifrost->>Provider2: API Call + Provider2-->>Bifrost: response + Bifrost->>Plugin1: PostHook(response) + Plugin1-->>Bifrost: modified response + Bifrost-->>Client: Final Response +``` + +### Complex Plugin Decision Flow + +```mermaid +graph TD + A["Client Request"] --> B["Bifrost"] + B --> C["Auth Plugin PreHook"] + C --> D{"Authenticated?"} + D -->|No| E["Return Auth Error
AllowFallbacks=false"] + D -->|Yes| F["RateLimit Plugin PreHook"] + F --> G{"Rate Limited?"} + G -->|Yes| H["Return Rate Error
AllowFallbacks=nil"] + G -->|No| I["Cache Plugin PreHook"] + I --> J{"Cache Hit?"} + J -->|Yes| K["Return Cached Response"] + J -->|No| L["Provider API Call"] + L --> M["Cache Plugin PostHook"] + M --> N["Store in Cache"] + N --> O["RateLimit Plugin PostHook"] + O --> P["Auth Plugin PostHook"] + P --> Q["Final Response"] + + E --> R["Skip Fallbacks"] + H --> S["Try Fallback Provider"] + K --> T["Skip Provider Call"] +``` + +## 5. Short-Circuit Behavior + +Plugins can short-circuit the normal flow in two ways: + +### 1. Short-Circuit with Response (Success) ```go -// Example plugin skeleton +func (p *CachePlugin) PreHook(ctx *context.Context, req *BifrostRequest) (*BifrostRequest, *PluginShortCircuit, error) { + if cachedResponse := p.getFromCache(req); cachedResponse != nil { + // Return cached response, skip provider call + return req, &PluginShortCircuit{ + Response: cachedResponse, + }, nil + } + return req, nil, nil +} +``` + +### 2. Short-Circuit with Error + +```go +func (p *AuthPlugin) PreHook(ctx *context.Context, req *BifrostRequest) (*BifrostRequest, *PluginShortCircuit, error) { + if !p.isAuthenticated(req) { + // Return error, skip provider call + return req, &PluginShortCircuit{ + Error: &BifrostError{ + Error: ErrorField{Message: "authentication failed"}, + AllowFallbacks: &false, // Don't try other providers + }, + }, nil + } + return req, nil, nil +} +``` + +## 6. Error Handling & Fallbacks + +When plugins return errors, they control whether Bifrost should try fallback providers: +### AllowFallbacks Control + +```go +// Allow fallbacks (default behavior) +&BifrostError{ + Error: ErrorField{Message: "rate limit exceeded"}, + AllowFallbacks: nil, // nil = true by default +} + +// Explicitly allow fallbacks +&BifrostError{ + Error: ErrorField{Message: "temporary failure"}, + AllowFallbacks: &true, +} + +// Prevent fallbacks +&BifrostError{ + Error: ErrorField{Message: "authentication failed"}, + AllowFallbacks: &false, +} +``` + +### Fallback Decision Matrix + +| Error Type | AllowFallbacks | Behavior | +| ------------------ | --------------- | ---------------------------------------------------------- | +| Rate Limiting | `nil` or `true` | ✅ Try fallbacks (other providers may not be rate limited) | +| Temporary Failure | `nil` or `true` | ✅ Try fallbacks (may succeed with different provider) | +| Authentication | `false` | ❌ No fallbacks (fundamental failure) | +| Validation Error | `false` | ❌ No fallbacks (request is invalid) | +| Security Violation | `false` | ❌ No fallbacks (security concern) | + +### PostHook Error Recovery + +Plugins can recover from errors in PostHook: + +```go +func (p *RetryPlugin) PostHook(ctx *context.Context, result *BifrostResponse, err *BifrostError) (*BifrostResponse, *BifrostError, error) { + if err != nil && p.shouldRetry(err) { + // Recover by calling provider again + if retryResponse := p.retry(ctx); retryResponse != nil { + return retryResponse, nil, nil // Recovered successfully + } + } + return result, err, nil +} +``` + +## 7. Building Custom Plugins + +### Basic Plugin Structure + +```go type CustomPlugin struct { - // Your plugin fields + config CustomConfig + // Add your fields here +} + +func NewCustomPlugin(config CustomConfig) *CustomPlugin { + return &CustomPlugin{config: config} } func (p *CustomPlugin) GetName() string { return "CustomPlugin" } -func (p *CustomPlugin) PreHook(ctx *context.Context, req *BifrostRequest) (*BifrostRequest, *BifrostResponse, error) { - // Modify request or add custom logic - // Return nil for response to continue with provider call +func (p *CustomPlugin) PreHook(ctx *context.Context, req *BifrostRequest) (*BifrostRequest, *PluginShortCircuit, error) { + // Modify request or short-circuit return req, nil, nil } func (p *CustomPlugin) PostHook(ctx *context.Context, result *BifrostResponse, err *BifrostError) (*BifrostResponse, *BifrostError, error) { - // Modify response or error, or recover from error - // Either result or err may be nil - // To recover from an error, set err to nil and return a valid result - // To invalidate a response, set the result to nil and return a non-nil err. + // Modify response/error or recover from errors return result, err, nil } func (p *CustomPlugin) Cleanup() error { - // Clean up any resources + // Clean up resources return nil } ``` -### Example: Rate Limiting Plugin +### Plugin Development Checklist + +- [ ] Handle nil response and error in PostHook +- [ ] Set appropriate AllowFallbacks for errors +- [ ] Implement proper cleanup in Cleanup() +- [ ] Add configuration validation +- [ ] Write comprehensive tests +- [ ] Document behavior and configuration + +## 8. Plugin Examples + +### Rate Limiting Plugin ```go type RateLimitPlugin struct { - limiter *rate.Limiter + limiters map[ModelProvider]*rate.Limiter + mu sync.RWMutex +} + +func NewRateLimitPlugin(limits map[ModelProvider]float64) *RateLimitPlugin { + limiters := make(map[ModelProvider]*rate.Limiter) + for provider, limit := range limits { + limiters[provider] = rate.NewLimiter(rate.Limit(limit), 1) + } + return &RateLimitPlugin{limiters: limiters} } func (p *RateLimitPlugin) GetName() string { return "RateLimitPlugin" } -func NewRateLimitPlugin(rps float64) *RateLimitPlugin { - return &RateLimitPlugin{ - limiter: rate.NewLimiter(rate.Limit(rps), 1), +func (p *RateLimitPlugin) PreHook(ctx *context.Context, req *BifrostRequest) (*BifrostRequest, *PluginShortCircuit, error) { + p.mu.RLock() + limiter, exists := p.limiters[req.Provider] + p.mu.RUnlock() + + if exists && !limiter.Allow() { + // Rate limited - allow fallbacks to other providers + return req, &PluginShortCircuit{ + Error: &BifrostError{ + Error: ErrorField{ + Message: fmt.Sprintf("rate limit exceeded for %s", req.Provider), + }, + AllowFallbacks: nil, // Allow fallbacks by default + }, + }, nil } -} -func (p *RateLimitPlugin) PreHook(ctx *context.Context, req *BifrostRequest) (*BifrostRequest, *BifrostResponse, error) { - if err := p.limiter.Wait(*ctx); err != nil { - return nil, nil, err - } return req, nil, nil } func (p *RateLimitPlugin) PostHook(ctx *context.Context, result *BifrostResponse, err *BifrostError) (*BifrostResponse, *BifrostError, error) { - // No-op for rate limiting return result, err, nil } func (p *RateLimitPlugin) Cleanup() error { - // Rate limiter doesn't need cleanup return nil } ``` -### Example: Logging Plugin +### Authentication Plugin ```go -type LoggingPlugin struct { - logger schemas.Logger +type AuthPlugin struct { + validator TokenValidator } -func (p *LoggingPlugin) GetName() string { - return "LoggingPlugin" +func NewAuthPlugin(validator TokenValidator) *AuthPlugin { + return &AuthPlugin{validator: validator} } -func NewLoggingPlugin(logger schemas.Logger) *LoggingPlugin { - return &LoggingPlugin{logger: logger} +func (p *AuthPlugin) GetName() string { + return "AuthPlugin" } -func (p *LoggingPlugin) PreHook(ctx *context.Context, req *BifrostRequest) (*BifrostRequest, *BifrostResponse, error) { - p.logger.Info(fmt.Sprintf("Request to %s with model %s", req.Provider, req.Model)) +func (p *AuthPlugin) PreHook(ctx *context.Context, req *BifrostRequest) (*BifrostRequest, *PluginShortCircuit, error) { + if !p.validator.IsValid(*ctx, req) { + // Authentication failed - don't try fallbacks + return req, &PluginShortCircuit{ + Error: &BifrostError{ + Error: ErrorField{ + Message: "authentication failed", + Type: &authErrorType, + }, + AllowFallbacks: &false, // Don't try other providers + }, + }, nil + } + return req, nil, nil } -func (p *LoggingPlugin) PostHook(ctx *context.Context, result *BifrostResponse, err *BifrostError) (*BifrostResponse, *BifrostError, error) { - if result != nil { - p.logger.Info(fmt.Sprintf("Response from %s with %d tokens", result.Model, result.Usage.TotalTokens)) - } - if err != nil { - p.logger.Warn(fmt.Sprintf("Error: %v", err.Error.Message)) - } +func (p *AuthPlugin) PostHook(ctx *context.Context, result *BifrostResponse, err *BifrostError) (*BifrostResponse, *BifrostError, error) { return result, err, nil } -func (p *LoggingPlugin) Cleanup() error { - // Logger doesn't need cleanup - return nil +func (p *AuthPlugin) Cleanup() error { + return p.validator.Cleanup() } ``` -## 4. Using Plugins +### Caching Plugin with Recovery + +```go +type CachePlugin struct { + cache Cache + ttl time.Duration +} + +func (p *CachePlugin) PreHook(ctx *context.Context, req *BifrostRequest) (*BifrostRequest, *PluginShortCircuit, error) { + key := p.generateKey(req) + if cachedResponse := p.cache.Get(key); cachedResponse != nil { + // Return cached response, skip provider + return req, &PluginShortCircuit{ + Response: cachedResponse, + }, nil + } + + return req, nil, nil +} + +func (p *CachePlugin) PostHook(ctx *context.Context, result *BifrostResponse, err *BifrostError) (*BifrostResponse, *BifrostError, error) { + if result != nil { + // Cache successful response + key := p.generateKeyFromResponse(result) + p.cache.Set(key, result, p.ttl) + } + + return result, err, nil +} +``` + +## 9. Best Practices + +### Plugin Design -### Initializing Bifrost with Plugins +1. **Keep plugins focused** - Each plugin should have a single responsibility +2. **Make plugins configurable** - Use configuration structs for flexibility +3. **Handle edge cases** - Always check for nil values and error conditions +4. **Be mindful of performance** - Plugins add latency to every request + +### Error Handling + +1. **Default to allowing fallbacks** - Unless the error is fundamental +2. **Use appropriate error types** - Help categorize different failure modes +3. **Provide clear error messages** - Include context about what failed +4. **Consider error recovery** - PostHooks can recover from certain errors + +### Resource Management + +1. **Implement proper cleanup** - Release resources in Cleanup() +2. **Use context for cancellation** - Respect request timeouts +3. **Avoid memory leaks** - Clean up goroutines and connections +4. **Handle concurrent access** - Use proper synchronization + +### Testing + +1. **Test all code paths** - Including error conditions and edge cases +2. **Test short-circuit behavior** - Verify responses and error handling +3. **Test fallback control** - Ensure AllowFallbacks works correctly +4. **Test plugin interactions** - Verify behavior with multiple plugins + +## 10. Plugin Development Guidelines + +### Plugin Structure Requirements + +Each plugin should be organized as follows: + +``` +plugins/ +└── your-plugin-name/ + ├── main.go # Plugin implementation + ├── plugin_test.go # Comprehensive tests + ├── README.md # Documentation with examples + └── go.mod # Module definition +``` + +### Using Plugins ```go +import ( + "github.com/maximhq/bifrost/core" + "github.com/your-org/your-plugin" +) + client, err := bifrost.Init(schemas.BifrostConfig{ Account: &yourAccount, Plugins: []schemas.Plugin{ - NewRateLimitPlugin(10.0), // 10 requests per second - NewLoggingPlugin(logger), // Custom logging + your_plugin.NewYourPlugin(config), // Add more plugins as needed }, }) ``` -## 5. Plugin Pipeline Symmetry and Order +### Plugin Execution Order + +Plugins execute in the order they are registered: + +```go +Plugins: []schemas.Plugin{ + authPlugin, // PreHook: 1st, PostHook: 3rd + rateLimitPlugin, // PreHook: 2nd, PostHook: 2nd + loggingPlugin, // PreHook: 3rd, PostHook: 1st +} +``` + +**PreHook Order**: Auth → RateLimit → Logging → Provider +**PostHook Order**: Provider → Logging → RateLimit → Auth -- PreHooks are executed in the order they are registered. -- PostHooks are executed in the reverse order of PreHooks. -- If a PreHook returns a response, the provider call is skipped and only the PostHook methods of plugins that had their PreHook executed are called in reverse order. -- The plugin pipeline ensures that for every PreHook executed, the corresponding PostHook will be called. +### Contribution Guidelines -## 6. Best Practices for Plugin Authors +1. **Design Discussion** -- Always check for both response and error being nil in PostHook. -- Do not assume either is always present. -- To recover from an error, set err to nil and return a valid result. Only truly empty errors (no message, no error, no status code, no type) are treated as recoveries by the pipeline. -- To invalidate a response, set the result to nil and return a non-nil err. -- Keep plugin logic lightweight and avoid blocking operations in hooks. -- Use context for cancellation and state passing. -- Write unit tests for your plugins, including error recovery and response invalidation scenarios. + - Open an issue to discuss your plugin idea + - Explain the use case and design approach + - Get feedback before implementation -## 7. Available Plugins +2. **Implementation Standards** -Bifrost provides a **[Plugin Store](https://github.com/maximhq/bifrost/tree/main/plugins)** with one-line integrations. For each plugin, refer to its specific documentation for configuration and usage details: + - Follow Go best practices and conventions + - Include comprehensive error handling + - Ensure thread safety where needed + - Add extensive test coverage (>80%) -`https://github.com/maximhq/bifrost/tree/main/plugins/{plugin_name}` +3. **Testing Requirements** -## 8. Plugin Development Guidelines + - Unit tests for all functionality + - Integration tests with Bifrost + - Test error scenarios and edge cases + - Test short-circuit behavior + - Test fallback control -1. **Documentation** +4. **Documentation Standards** + - Clear, comprehensive README + - Code comments for complex logic + - Usage examples + - Performance characteristics - - Document your plugin's purpose and configuration - - Provide usage examples - - Include performance considerations +### Plugin Testing Best Practices -2. **Configuration** +```go +func TestYourPlugin_PreHook(t *testing.T) { + tests := []struct { + name string + config YourPluginConfig + request *schemas.BifrostRequest + expectShortCircuit bool + expectError bool + expectFallbacks bool + }{ + { + name: "valid request passes through", + config: YourPluginConfig{EnableFeature: true}, + request: &schemas.BifrostRequest{/* valid request */}, + expectShortCircuit: false, + }, + { + name: "invalid request short-circuits with error", + config: YourPluginConfig{EnableFeature: true}, + request: &schemas.BifrostRequest{/* invalid request */}, + expectShortCircuit: true, + expectError: true, + expectFallbacks: false, + }, + // Add more test cases + } - - Make plugins configurable - - Use sensible defaults - - Validate configuration + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + plugin := NewYourPlugin(tt.config) + ctx := context.Background() + + req, shortCircuit, err := plugin.PreHook(&ctx, tt.request) + + // Assertions + if tt.expectError { + assert.NotNil(t, err) + } else { + assert.Nil(t, err) + } + + if tt.expectShortCircuit { + assert.NotNil(t, shortCircuit) + if shortCircuit.Error != nil && shortCircuit.Error.AllowFallbacks != nil { + assert.Equal(t, tt.expectFallbacks, *shortCircuit.Error.AllowFallbacks) + } + } else { + assert.Nil(t, shortCircuit) + } + }) + } +} +``` -3. **Error Handling** +## 11. Troubleshooting Guide - - Use custom error types - - Provide detailed error messages - - Handle edge cases gracefully +### Common Issues -4. **Contribution Process** +#### 1. Plugin Not Being Called - - Open an issue first to discuss the plugin's use case and design - - Create a pull request with a clear explanation of the plugin's purpose - - Follow the plugin structure requirements below +**Symptoms**: Plugin hooks are never executed +**Solutions**: -5. **Plugin Structure** - Each plugin should be organized as follows: +```go +// Ensure plugin is properly registered +client, err := bifrost.Init(schemas.BifrostConfig{ + Account: &account, + Plugins: []schemas.Plugin{ + yourPlugin, // Make sure it's in the list + }, +}) - ``` - plugins/ - └── your-plugin-name/ - ├── main.go # Plugin implementation - ├── plugin_test.go # Plugin tests - ├── README.md # Documentation - └── go.mod # Module definition - ``` +// Check plugin implements interface correctly +var _ schemas.Plugin = (*YourPlugin)(nil) +``` - Example `main.go`: +#### 2. Short-Circuit Not Working - ```go - package your_plugin_name +**Symptoms**: Provider is still called despite returning PluginShortCircuit +**Solutions**: - import ( - "context" - "github.com/maximhq/bifrost/core/schemas" - ) +```go +// Correct: Either Response OR Error, not both +return req, &schemas.PluginShortCircuit{ + Response: cachedResponse, // OR Error, not both +}, nil - type YourPlugin struct { - // Plugin fields - } +// Incorrect: Don't return error with PluginShortCircuit +return req, &schemas.PluginShortCircuit{...}, fmt.Errorf("error") +``` - func (p *YourPlugin) GetName() string { - return "YourPlugin" - } +#### 3. Fallback Behavior Not Working - func (p *YourPlugin) PreHook(ctx *context.Context, req *schemas.BifrostRequest) (*schemas.BifrostRequest, *schemas.BifrostResponse, error) { - // Implementation - return req, nil, nil - } +**Symptoms**: Fallbacks not tried when expected, or tried when they shouldn't be +**Solutions**: - func (p *YourPlugin) PostHook(ctx *context.Context, result *schemas.BifrostResponse, err *schemas.BifrostError) (*schemas.BifrostResponse, *schemas.BifrostError, error) { - // Implementation - return result, err, nil - } +```go +// For PreHook short-circuits, use PluginShortCircuit +return req, &schemas.PluginShortCircuit{ + Error: &schemas.BifrostError{ + Error: schemas.ErrorField{Message: "error"}, + AllowFallbacks: &false, // Explicitly control fallbacks + }, +}, nil +``` - func (p *YourPlugin) Cleanup() error { - // Clean up any resources - return nil - } - ``` +#### 4. Memory Leaks - Example `README.md`: +**Solutions**: - ````markdown - # Your Plugin Name +```go +func (p *YourPlugin) Cleanup() error { + // Close channels + close(p.stopChan) + + // Cancel contexts + p.cancel() + + // Close connections + if p.conn != nil { + p.conn.Close() + } + + // Wait for goroutines + p.wg.Wait() + + return nil +} +``` + +#### 5. Race Conditions + +**Solutions**: + +```go +type ThreadSafePlugin struct { + mu sync.RWMutex + state map[string]interface{} +} + +func (p *ThreadSafePlugin) PreHook(ctx *context.Context, req *schemas.BifrostRequest) (*schemas.BifrostRequest, *schemas.PluginShortCircuit, error) { + p.mu.Lock() + defer p.mu.Unlock() + + // Safe access to shared state + p.state[req.ID] = "processing" + return req, nil, nil +} +``` - Brief description of what the plugin does. +## 12. Performance Optimization - ## Installation +1. **Minimize Hook Latency** - ```bash - go get github.com/maximhq/bifrost/plugins/your-plugin-name - ``` + - Avoid blocking operations in hooks + - Use goroutines for background work + - Cache expensive computations - ## Usage +2. **Efficient Resource Usage** - Explain plugin usage. + - Pool connections and resources + - Use sync.Pool for frequently allocated objects + - Implement proper cleanup - ## Configuration +3. **Monitor Memory Usage** + - Profile your plugin under load + - Watch for memory leaks + - Use appropriate data structures - Describe configuration options. +## Summary - ## Examples +This documentation provides complete coverage for Bifrost plugin development: - Show usage examples. - ```` +- **Architecture & Lifecycle** - Understanding the plugin system and execution flow +- **Interface & Behavior** - Exact method signatures and short-circuit capabilities +- **Error Handling** - Complete control over fallback behavior with AllowFallbacks +- **Practical Examples** - Real-world plugins for rate limiting, auth, and caching +- **Development Guidelines** - Best practices, testing, and contribution standards +- **Troubleshooting** - Solutions for common issues and performance optimization