From 947487a8bd77c126b527ee8fd6754655f8e972f2 Mon Sep 17 00:00:00 2001 From: Pratham Mishra <99235987+Pratham-Mishra04@users.noreply.github.com> Date: Sat, 21 Jun 2025 10:44:19 +0530 Subject: [PATCH] feat: mocker plugin added --- docs/mcp.md | 14 +- docs/plugins.md | 2 +- plugins/mocker/README.md | 1301 +++++++++++++++++++++++ plugins/mocker/benchmark_test.go | 296 ++++++ plugins/mocker/go.mod | 37 + plugins/mocker/go.sum | 76 ++ plugins/mocker/main.go | 1083 +++++++++++++++++++ plugins/mocker/plugin_test.go | 538 ++++++++++ tests/transports-integrations/README.md | 6 +- 9 files changed, 3342 insertions(+), 11 deletions(-) create mode 100644 plugins/mocker/README.md create mode 100644 plugins/mocker/benchmark_test.go create mode 100644 plugins/mocker/go.mod create mode 100644 plugins/mocker/go.sum create mode 100644 plugins/mocker/main.go create mode 100644 plugins/mocker/plugin_test.go diff --git a/docs/mcp.md b/docs/mcp.md index 53c7a0759a..9bb0667d19 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -1341,7 +1341,7 @@ func getContextForUserRole(role string) context.Context { **STDIO Connection Issues:** -``` +```text Error: failed to start command 'npx @modelcontextprotocol/server-filesystem' ``` @@ -1354,7 +1354,7 @@ Error: failed to start command 'npx @modelcontextprotocol/server-filesystem' **HTTP Connection Issues:** -``` +```text Error: failed to initialize external MCP client: connection refused ``` @@ -1367,7 +1367,7 @@ Error: failed to initialize external MCP client: connection refused **SSE Connection Issues:** -``` +```text Error: SSE stream error: context canceled ``` @@ -1383,7 +1383,7 @@ Error: SSE stream error: context canceled **Tool Already Exists:** -``` +```text Error: tool 'echo' already registered ``` @@ -1397,7 +1397,7 @@ Error: tool 'echo' already registered **No Tools Available:** -``` +```text Warning: No MCP tools found in response ``` @@ -1410,7 +1410,7 @@ Warning: No MCP tools found in response **Unexpected Tool Availability:** -``` +```text Warning: Restricted tool 'delete_all_files' is available when it shouldn't be ``` @@ -1424,7 +1424,7 @@ Warning: Restricted tool 'delete_all_files' is available when it shouldn't be **Tool Not Found:** -``` +```text Error: MCP tool 'unknown_tool' not found ``` diff --git a/docs/plugins.md b/docs/plugins.md index a7e796092a..c1154a9682 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -476,7 +476,7 @@ func (p *CachePlugin) PostHook(ctx *context.Context, result *BifrostResponse, er Each plugin should be organized as follows: -``` +```text plugins/ └── your-plugin-name/ ├── main.go # Plugin implementation diff --git a/plugins/mocker/README.md b/plugins/mocker/README.md new file mode 100644 index 0000000000..7b8d0cce22 --- /dev/null +++ b/plugins/mocker/README.md @@ -0,0 +1,1301 @@ +# Bifrost Mocker Plugin + +The Mocker plugin for Bifrost allows you to intercept and mock AI provider responses for testing, development, and simulation purposes. It provides flexible rule-based mocking with support for custom responses, error simulation, latency injection, and comprehensive statistics tracking. + +**⚡ Performance Optimized** - Designed for high-throughput scenarios including benchmarking with minimal overhead. + +## Table of Contents + +1. [Quick Start](#quick-start) +2. [Installation](#installation) +3. [Basic Usage](#basic-usage) +4. [Configuration Reference](#configuration-reference) +5. [Advanced Features](#advanced-features) +6. [Faker Support](#faker-support) +7. [Examples](#examples) +8. [Statistics and Monitoring](#statistics-and-monitoring) +9. [Performance](#performance) +10. [Best Practices](#best-practices) +11. [Troubleshooting](#troubleshooting) + +## Quick Start + +### Minimal Configuration + +The simplest way to use the Mocker plugin is with no configuration - it will create a default catch-all rule: + +```go +package main + +import ( + "context" + bifrost "github.com/maximhq/bifrost/core" + "github.com/maximhq/bifrost/core/schemas" + mocker "github.com/maximhq/bifrost/plugins/mocker" +) + +func main() { + // Create plugin with minimal config + plugin, err := mocker.NewMockerPlugin(mocker.MockerConfig{ + Enabled: true, // That's it! Default rule will be created automatically + }) + if err != nil { + panic(err) + } + + // Initialize Bifrost with the plugin + client, err := bifrost.Init(schemas.BifrostConfig{ + Account: &yourAccount, + Plugins: []schemas.Plugin{plugin}, + }) + if err != nil { + panic(err) + } + defer client.Cleanup() + + // All requests will now return: "This is a mock response from the Mocker plugin" + response, _ := client.ChatCompletionRequest(context.Background(), &schemas.BifrostRequest{ + Provider: schemas.OpenAI, + Model: "gpt-4", + Input: schemas.RequestInput{ + ChatCompletionInput: &[]schemas.BifrostMessage{ + { + Role: schemas.ModelChatMessageRoleUser, + Content: schemas.MessageContent{ + ContentStr: bifrost.Ptr("Hello!"), + }, + }, + }, + }, + }) + + // response.Choices[0].Message.Content.ContentStr == "This is a mock response from the Mocker plugin" +} +``` + +### Quick Custom Response + +```go +plugin, err := mocker.NewMockerPlugin(mocker.MockerConfig{ + Enabled: true, + Rules: []mocker.MockRule{ + { + Name: "openai-mock", + Enabled: true, + Probability: 1.0, // Always trigger + Conditions: mocker.Conditions{ + Providers: []string{"openai"}, + }, + Responses: []mocker.Response{ + { + Type: mocker.ResponseTypeSuccess, + Content: &mocker.SuccessResponse{ + Message: "Hello! This is a custom mock response for OpenAI.", + Usage: &mocker.Usage{ + PromptTokens: 15, + CompletionTokens: 25, + TotalTokens: 40, + }, + }, + }, + }, + }, + }, +}) +``` + +## Installation + +### As a Go Module + +1. Add the plugin to your project: + + ```bash + go get github.com/maximhq/bifrost/plugins/mocker + ``` + +2. Import in your code: + + ```go + import mocker "github.com/maximhq/bifrost/plugins/mocker" + ``` + +### Development Setup + +1. Clone the repository: + + ```bash + git clone https://github.com/maximhq/bifrost.git + cd bifrost/plugins/mocker + ``` + +2. Install dependencies: + + ```bash + go mod tidy + ``` + +3. Run tests: + + ```bash + go test -v + ``` + +## Basic Usage + +### Creating the Plugin + +```go +// Basic configuration +config := mocker.MockerConfig{ + Enabled: true, + DefaultBehavior: mocker.DefaultBehaviorPassthrough, // Optional: "passthrough", "success", "error" + Rules: []mocker.MockRule{ + // Your rules here + }, +} + +plugin, err := mocker.NewMockerPlugin(config) +if err != nil { + // Handle configuration errors + log.Fatal(err) +} +``` + +### Adding to Bifrost + +```go +client, err := bifrost.Init(schemas.BifrostConfig{ + Account: &yourAccount, + Plugins: []schemas.Plugin{ + plugin, // Add your mocker plugin + // Other plugins... + }, + Logger: bifrost.NewDefaultLogger(schemas.LogLevelInfo), +}) +``` + +### Disabling the Plugin + +```go +// Temporarily disable without removing +config := mocker.MockerConfig{ + Enabled: false, // All requests pass through to real providers +} +``` + +## Configuration Reference + +### MockerConfig + +| Field | Type | Required | Default | Description | +| ----------------- | ------------ | -------- | --------------- | ------------------------------------------------------------------- | +| `Enabled` | `bool` | Yes | `false` | Enable/disable the entire plugin | +| `DefaultBehavior` | `string` | No | `"passthrough"` | Action when no rules match: `"passthrough"`, `"success"`, `"error"` | +| `GlobalLatency` | `*Latency` | No | `nil` | Global latency applied to all rules (can be overridden per rule) | +| `Rules` | `[]MockRule` | No | `[]` | List of mock rules evaluated in priority order | + +### MockRule + +| Field | Type | Required | Default | Description | +| ------------- | ------------ | -------- | ------- | -------------------------------------------------- | +| `Name` | `string` | Yes | - | Unique rule name for identification and statistics | +| `Enabled` | `bool` | No | `true` | Enable/disable this specific rule | +| `Priority` | `int` | No | `0` | Higher numbers = higher priority (checked first) | +| `Probability` | `float64` | No | `1.0` | Activation probability (0.0=never, 1.0=always) | +| `Conditions` | `Conditions` | No | `{}` | Matching conditions (empty = match all) | +| `Responses` | `[]Response` | Yes | - | Possible responses (weighted random selection) | +| `Latency` | `*Latency` | No | `nil` | Rule-specific latency override | + +### Conditions + +| Field | Type | Required | Default | Description | +| -------------- | ------------ | -------- | ------- | --------------------------------------------------- | +| `Providers` | `[]string` | No | `[]` | Match specific providers: `["openai", "anthropic"]` | +| `Models` | `[]string` | No | `[]` | Match specific models: `["gpt-4", "claude-3"]` | +| `MessageRegex` | `*string` | No | `nil` | Regex pattern to match message content | +| `RequestSize` | `*SizeRange` | No | `nil` | Request size constraints in bytes | + +### Response + +| Field | Type | Required | Default | Description | +| ---------------- | ------------------ | ----------- | ------- | ------------------------------------------------------ | +| `Type` | `string` | Yes | - | Response type: `"success"` or `"error"` | +| `Weight` | `float64` | No | `1.0` | Weight for random selection (higher = more likely) | +| `Content` | `*SuccessResponse` | Conditional | - | Required if `Type="success"` | +| `Error` | `*ErrorResponse` | Conditional | - | Required if `Type="error"` | +| `AllowFallbacks` | `*bool` | No | `nil` | Control fallback behavior (`nil`=allow, `false`=block) | + +### SuccessResponse + +| Field | Type | Required | Default | Description | +| ----------------- | ------------------------ | ----------- | -------------- | ------------------------------------------------------------------- | +| `Message` | `string` | Conditional | - | Static response message (required if no template) | +| `MessageTemplate` | `*string` | Conditional | - | Template with variables: `{{provider}}`, `{{model}}`, `{{faker.*}}` | +| `Model` | `*string` | No | `nil` | Override model name in response | +| `Usage` | `*Usage` | No | Default values | Token usage information | +| `FinishReason` | `*string` | No | `"stop"` | Completion reason | +| `CustomFields` | `map[string]interface{}` | No | `{}` | Additional metadata fields | + +### ErrorResponse + +| Field | Type | Required | Default | Description | +| ------------ | --------- | -------- | ------- | ------------------------------------------------- | +| `Message` | `string` | Yes | - | Error message to return | +| `Type` | `*string` | No | `nil` | Error type (e.g., `"rate_limit"`, `"auth_error"`) | +| `Code` | `*string` | No | `nil` | Error code (e.g., `"429"`, `"401"`) | +| `StatusCode` | `*int` | No | `nil` | HTTP status code | + +### Latency + +| Field | Type | Required | Default | Description | +| ------ | --------------- | ----------- | ------- | ------------------------------------------------------------------ | +| `Type` | `string` | Yes | - | Latency type: `"fixed"` or `"uniform"` | +| `Min` | `time.Duration` | Yes | - | Minimum/exact latency (use `time.Millisecond`, NOT raw int) | +| `Max` | `time.Duration` | Conditional | - | Maximum latency (required for `"uniform"`, use `time.Millisecond`) | + +**⚠️ Important**: Use Go's `time.Duration` constants, not raw integers: + +- ✅ Correct: `100 * time.Millisecond` +- ❌ Wrong: `100` (this would be 100 nanoseconds, barely noticeable) + +### SizeRange + +| Field | Type | Required | Default | Description | +| ----- | ----- | -------- | ------- | ----------------------------- | +| `Min` | `int` | Yes | - | Minimum request size in bytes | +| `Max` | `int` | Yes | - | Maximum request size in bytes | + +### Usage + +| Field | Type | Required | Default | Description | +| ------------------ | ----- | -------- | ------- | ---------------------------------- | +| `PromptTokens` | `int` | No | `10` | Number of tokens in the prompt | +| `CompletionTokens` | `int` | No | `20` | Number of tokens in the completion | +| `TotalTokens` | `int` | No | `30` | Total tokens (prompt + completion) | + +## Advanced Features + +### Template Variables + +Use templates to create dynamic responses: + +```go +Response{ + Type: mocker.ResponseTypeSuccess, + Content: &mocker.SuccessResponse{ + MessageTemplate: stringPtr("Hello from {{provider}} using model {{model}}!"), + }, +} +``` + +**Available Variables:** + +- `{{provider}}` - Provider name (e.g., "openai", "anthropic") +- `{{model}}` - Model name (e.g., "gpt-4", "claude-3") +- `{{faker.*}}` - Fake data generation (see [Faker Support](#faker-support) section for full list) + +### Weighted Response Selection + +Configure multiple responses with different weights: + +```go +Responses: []mocker.Response{ + { + Type: mocker.ResponseTypeSuccess, + Weight: 0.8, // 80% chance + Content: &mocker.SuccessResponse{ + Message: "Success response", + }, + }, + { + Type: mocker.ResponseTypeError, + Weight: 0.2, // 20% chance + Error: &mocker.ErrorResponse{ + Message: "Simulated error", + StatusCode: intPtr(500), + }, + }, +} +``` + +### Latency Simulation + +Add realistic delays to responses: + +```go +// Fixed latency +Latency: &mocker.Latency{ + Type: mocker.LatencyTypeFixed, + Min: 100 * time.Millisecond, +} + +// Variable latency +Latency: &mocker.Latency{ + Type: mocker.LatencyTypeUniform, + Min: 50 * time.Millisecond, + Max: 200 * time.Millisecond, +} +``` + +**⚠️ Critical**: Always use `time.Duration` constants (e.g., `time.Millisecond`), never raw integers. Raw integers are interpreted as nanoseconds and will be barely noticeable. + +### Regex Message Matching + +Match specific message content: + +```go +Conditions: mocker.Conditions{ + MessageRegex: stringPtr(`(?i).*error.*|.*help.*`), // Case-insensitive match for "error" or "help" +} +``` + +### Request Size Filtering + +Match requests by size: + +```go +Conditions: mocker.Conditions{ + RequestSize: &mocker.SizeRange{ + Min: 100, // Minimum 100 bytes + Max: 1000, // Maximum 1000 bytes + }, +} +``` + +## Faker Support + +The Mocker plugin includes comprehensive fake data generation capabilities using the [jaswdr/faker](https://github.com/jaswdr/faker) library. This allows you to create realistic mock responses with dynamic, fake data that changes on each request. + +### Using Faker in Templates + +Faker variables can be used in the `MessageTemplate` field using the `{{faker.method}}` syntax: + +```go +Response{ + Type: mocker.ResponseTypeSuccess, + Content: &mocker.SuccessResponse{ + MessageTemplate: stringPtr(`Hello {{faker.first_name}}! Your email {{faker.email}} has been verified. +Your account ID is {{faker.uuid}} and your phone number {{faker.phone}} is on file.`), + }, +} +``` + +### Available Faker Methods + +#### Personal Information + +- `{{faker.name}}` - Full name (e.g., "John Smith") +- `{{faker.first_name}}` - First name (e.g., "John") +- `{{faker.last_name}}` - Last name (e.g., "Smith") +- `{{faker.email}}` - Email address (e.g., "john123@example.com") +- `{{faker.phone}}` - Phone number (e.g., "+1-555-123-4567") + +#### Location + +- `{{faker.address}}` - Full address (e.g., "123 Main St") +- `{{faker.city}}` - City name (e.g., "New York") +- `{{faker.state}}` - State name (e.g., "California") +- `{{faker.zip_code}}` - Postal code (e.g., "12345") + +#### Business + +- `{{faker.company}}` - Company name (e.g., "Tech Solutions Inc.") +- `{{faker.job_title}}` - Job title (e.g., "Software Engineer") + +#### Text and Lorem Ipsum + +- `{{faker.word}}` - Single word (e.g., "example") +- `{{faker.sentence}}` - Sentence with default 8 words +- `{{faker.sentence:5}}` - Sentence with 5 words +- `{{faker.lorem_ipsum}}` - Lorem ipsum text with default 10 words +- `{{faker.lorem_ipsum:20}}` - Lorem ipsum text with 20 words + +#### Identifiers and Data + +- `{{faker.uuid}}` - UUID v4 (e.g., "123e4567-e89b-12d3-a456-426614174000") +- `{{faker.hex_color}}` - Hex color code (e.g., "#FF5733") + +#### Numbers and Dates + +- `{{faker.integer}}` - Random integer between 1-100 +- `{{faker.integer:10,50}}` - Random integer between 10-50 +- `{{faker.float}}` - Random float between 0-100 (2 decimal places) +- `{{faker.float:1,10}}` - Random float between 1-10 +- `{{faker.boolean}}` - Random boolean (true/false) +- `{{faker.date}}` - Date in YYYY-MM-DD format +- `{{faker.datetime}}` - Datetime in YYYY-MM-DD HH:MM:SS format + +### Faker Examples + +#### Customer Support Simulation + +```go +{ + Name: "customer-support-faker", + Enabled: true, + Priority: 100, + Probability: 1.0, + Conditions: mocker.Conditions{ + MessageRegex: stringPtr(`(?i).*support.*|.*help.*`), + }, + Responses: []mocker.Response{ + { + Type: mocker.ResponseTypeSuccess, + Content: &mocker.SuccessResponse{ + MessageTemplate: stringPtr(`Hello {{faker.first_name}}! + +Thank you for contacting {{faker.company}} support. I've reviewed your account and here are the details: + +**Account Information:** +- Name: {{faker.name}} +- Email: {{faker.email}} +- Phone: {{faker.phone}} +- Account ID: {{faker.uuid}} +- Address: {{faker.address}}, {{faker.city}}, {{faker.state}} {{faker.zip_code}} + +**Support Ticket:** #{{faker.integer:10000,99999}} +**Priority:** {{faker.boolean}} +**Estimated Resolution:** {{faker.date}} + +How can I help you today?`), + Usage: &mocker.Usage{ + PromptTokens: 25, + CompletionTokens: 150, + TotalTokens: 175, + }, + }, + }, + }, +} +``` + +#### E-commerce Order Confirmation + +```go +{ + Name: "ecommerce-order-faker", + Enabled: true, + Priority: 100, + Probability: 1.0, + Conditions: mocker.Conditions{ + MessageRegex: stringPtr(`(?i).*order.*|.*purchase.*`), + }, + Responses: []mocker.Response{ + { + Type: mocker.ResponseTypeSuccess, + Content: &mocker.SuccessResponse{ + MessageTemplate: stringPtr(`🎉 Order Confirmed! + +**Order Details:** +- Order ID: {{faker.uuid}} +- Customer: {{faker.name}} +- Email: {{faker.email}} +- Total: ${{faker.float:10,500}} +- Items: {{faker.integer:1,5}} items + +**Shipping Address:** +{{faker.address}} +{{faker.city}}, {{faker.state}} {{faker.zip_code}} + +**Estimated Delivery:** {{faker.date}} +**Tracking Number:** {{faker.integer:100000000,999999999}} + +Thank you for shopping with {{faker.company}}!`), + }, + }, + }, +} +``` + +#### User Profile Generation + +```go +{ + Name: "user-profile-faker", + Enabled: true, + Priority: 100, + Probability: 1.0, + Conditions: mocker.Conditions{ + MessageRegex: stringPtr(`(?i).*profile.*|.*user.*info.*`), + }, + Responses: []mocker.Response{ + { + Type: mocker.ResponseTypeSuccess, + Content: &mocker.SuccessResponse{ + MessageTemplate: stringPtr(`**User Profile Generated:** + +**Personal Information:** +- Full Name: {{faker.name}} +- Email: {{faker.email}} +- Phone: {{faker.phone}} +- Preferred Color: {{faker.hex_color}} + +**Professional Details:** +- Company: {{faker.company}} +- Job Title: {{faker.job_title}} +- Work Phone: {{faker.phone}} + +**Address:** +{{faker.address}} +{{faker.city}}, {{faker.state}} {{faker.zip_code}} + +**Account Settings:** +- User ID: {{faker.uuid}} +- Account Created: {{faker.date}} +- Email Notifications: {{faker.boolean}} +- SMS Alerts: {{faker.boolean}} + +**Bio:** {{faker.lorem_ipsum:25}}`), + }, + }, + }, +} +``` + +### Faker with Weighted Responses + +You can combine faker with weighted response selection for even more realistic scenarios: + +```go +{ + Name: "mixed-faker-responses", + Enabled: true, + Priority: 100, + Probability: 1.0, + Responses: []mocker.Response{ + { + Type: mocker.ResponseTypeSuccess, + Weight: 0.7, // 70% positive responses + Content: &mocker.SuccessResponse{ + MessageTemplate: stringPtr(`Great news, {{faker.first_name}}! Your request has been approved. +Reference number: {{faker.uuid}}. +Contact us at {{faker.phone}} if you have questions.`), + }, + }, + { + Type: mocker.ResponseTypeSuccess, + Weight: 0.2, // 20% neutral responses + Content: &mocker.SuccessResponse{ + MessageTemplate: stringPtr(`Hello {{faker.first_name}}, your request is being processed. +Ticket ID: {{faker.integer:1000,9999}}. +Expected completion: {{faker.date}}.`), + }, + }, + { + Type: mocker.ResponseTypeError, + Weight: 0.1, // 10% error responses + Error: &mocker.ErrorResponse{ + Message: fmt.Sprintf("Account validation failed for user %s. Please contact support.", "{{faker.email}}"), + Type: stringPtr("validation_error"), + Code: stringPtr("VAL_001"), + }, + }, + }, +} +``` + +### Important Notes + +- **Dynamic Generation**: Faker values are generated fresh on each request, ensuring unique responses +- **Performance**: Faker generation is highly optimized and adds minimal overhead +- **Parameters**: Some faker methods support parameters (e.g., `{{faker.sentence:10}}` for 10 words) +- **Reliability**: Uses the established [jaswdr/faker](https://github.com/jaswdr/faker) library with zero dependencies +- **Template Mixing**: You can freely mix faker variables with regular template variables like `{{provider}}` and `{{model}}` + +## Examples + +### Development Environment Mock + +```go +config := mocker.MockerConfig{ + Enabled: true, + DefaultBehavior: mocker.DefaultBehaviorPassthrough, + Rules: []mocker.MockRule{ + { + Name: "dev-openai-mock", + Enabled: true, + Priority: 100, + Probability: 1.0, + Conditions: mocker.Conditions{ + Providers: []string{"openai"}, + Models: []string{"gpt-4", "gpt-4-turbo"}, + }, + Responses: []mocker.Response{ + { + Type: mocker.ResponseTypeSuccess, + Content: &mocker.SuccessResponse{ + MessageTemplate: stringPtr("Development mock response from {{model}} ({{provider}})"), + Usage: &mocker.Usage{ + PromptTokens: 20, + CompletionTokens: 30, + TotalTokens: 50, + }, + }, + }, + }, + }, + }, +} +``` + +### Error Simulation for Testing + +```go +config := mocker.MockerConfig{ + Enabled: true, + Rules: []mocker.MockRule{ + { + Name: "rate-limit-simulation", + Enabled: true, + Priority: 200, + Probability: 0.1, // 10% of requests + Conditions: mocker.Conditions{ + Providers: []string{"openai", "anthropic"}, + }, + Responses: []mocker.Response{ + { + Type: mocker.ResponseTypeError, + AllowFallbacks: boolPtr(true), // Allow fallback providers + Error: &mocker.ErrorResponse{ + Message: "Rate limit exceeded. Please try again later.", + Type: stringPtr("rate_limit"), + Code: stringPtr("429"), + StatusCode: intPtr(429), + }, + }, + }, + Latency: &mocker.Latency{ + Type: mocker.LatencyTypeFixed, + Min: 500 * time.Millisecond, // Simulate slow error response + }, + }, + }, +} +``` + +### A/B Testing Different Responses + +```go +config := mocker.MockerConfig{ + Enabled: true, + Rules: []mocker.MockRule{ + { + Name: "ab-test-responses", + Enabled: true, + Priority: 100, + Probability: 1.0, + Conditions: mocker.Conditions{ + MessageRegex: stringPtr(`(?i).*greeting.*|.*hello.*`), + }, + Responses: []mocker.Response{ + { + Type: mocker.ResponseTypeSuccess, + Weight: 0.5, // 50% - Version A + Content: &mocker.SuccessResponse{ + Message: "Hello! How can I help you today?", + CustomFields: map[string]interface{}{ + "ab_test_version": "A", + "response_style": "formal", + }, + }, + }, + { + Type: mocker.ResponseTypeSuccess, + Weight: 0.5, // 50% - Version B + Content: &mocker.SuccessResponse{ + Message: "Hey there! What's up?", + CustomFields: map[string]interface{}{ + "ab_test_version": "B", + "response_style": "casual", + }, + }, + }, + }, + }, + }, +} +``` + +### Provider-Specific Behavior + +```go +config := mocker.MockerConfig{ + Enabled: true, + Rules: []mocker.MockRule{ + { + Name: "openai-success", + Enabled: true, + Priority: 100, + Probability: 0.9, // 90% success rate + Conditions: mocker.Conditions{ + Providers: []string{"openai"}, + }, + Responses: []mocker.Response{ + { + Type: mocker.ResponseTypeSuccess, + Content: &mocker.SuccessResponse{ + Message: "OpenAI mock response - high reliability", + }, + }, + }, + }, + { + Name: "anthropic-mixed", + Enabled: true, + Priority: 100, + Probability: 1.0, + Conditions: mocker.Conditions{ + Providers: []string{"anthropic"}, + }, + Responses: []mocker.Response{ + { + Type: mocker.ResponseTypeSuccess, + Weight: 0.7, // 70% success + Content: &mocker.SuccessResponse{ + Message: "Anthropic mock response", + }, + }, + { + Type: mocker.ResponseTypeError, + Weight: 0.3, // 30% error + AllowFallbacks: boolPtr(true), + Error: &mocker.ErrorResponse{ + Message: "Anthropic service temporarily unavailable", + StatusCode: intPtr(503), + }, + }, + }, + }, + }, +} +``` + +## Statistics and Monitoring + +### Getting Statistics + +```go +// Get current statistics +stats := plugin.GetStats() +fmt.Printf("Total Requests: %d\n", stats.TotalRequests) +fmt.Printf("Mocked Requests: %d\n", stats.MockedRequests) +fmt.Printf("Success Responses: %d\n", stats.ResponsesGenerated) +fmt.Printf("Error Responses: %d\n", stats.ErrorsGenerated) + +// Per-rule statistics +for ruleName, hits := range stats.RuleHits { + fmt.Printf("Rule '%s': %d hits\n", ruleName, hits) +} +``` + +### Statistics Structure + +```go +type MockStats struct { + TotalRequests int64 `json:"total_requests"` // Total requests processed + MockedRequests int64 `json:"mocked_requests"` // Requests that matched rules + RuleHits map[string]int64 `json:"rule_hits"` // Per-rule hit counts + ErrorsGenerated int64 `json:"errors_generated"` // Error responses generated + ResponsesGenerated int64 `json:"responses_generated"` // Success responses generated +} +``` + +### Monitoring Example + +```go +// Periodic monitoring +go func() { + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + stats := plugin.GetStats() + log.Printf("Mocker Stats - Total: %d, Mocked: %d, Success: %d, Errors: %d", + stats.TotalRequests, stats.MockedRequests, + stats.ResponsesGenerated, stats.ErrorsGenerated) + } + } +}() +``` + +## Performance + +The Mocker plugin has been extensively optimized for high-throughput scenarios, including benchmarking and load testing. Here are the key performance characteristics and optimizations: + +### 🚀 **Key Optimizations** + +#### 1. **Pre-compiled Regex Patterns** + +- All regex patterns are compiled once during plugin initialization +- **Before**: `regexp.Compile()` on every request (~1000x slower) +- **After**: Pre-compiled patterns with direct matching + +#### 2. **Atomic Counters for Statistics** + +- Statistics use `sync/atomic` operations instead of mutex locks +- **Before**: Mutex lock/unlock for every counter increment +- **After**: Lock-free atomic operations + +#### 3. **Optimized String Operations** + +- Fast-path string matching and content extraction +- Efficient template processing with `strings.NewReplacer` +- Minimal memory allocations in hot paths + +#### 4. **Smart Memory Management** + +- Pre-allocated data structures +- Reduced map allocations +- Efficient string building for multi-message content + +### 🧪 **Running Benchmarks** + +Since performance varies significantly across different hardware configurations, you should run benchmarks on your specific system to get accurate measurements: + +```bash +# Run all benchmarks with memory allocation stats +go test -bench=. -benchmem + +# Run specific benchmark scenarios +go test -bench=BenchmarkMockerPlugin_PreHook_SimpleRule -benchmem +go test -bench=BenchmarkMockerPlugin_PreHook_RegexRule -benchmem +go test -bench=BenchmarkMockerPlugin_PreHook_NoMatch -benchmem + +# Run with CPU profiling for detailed analysis +go test -bench=. -cpuprofile=cpu.prof + +# Run with memory profiling +go test -bench=. -memprofile=mem.prof + +# Run benchmarks multiple times for statistical accuracy +go test -bench=. -count=5 +``` + +### 📊 **Understanding Benchmark Output** + +The benchmark output will show: + +- **Operations per second**: How many operations can be performed per second +- **Nanoseconds per operation**: Average time per operation +- **Bytes per operation**: Memory allocated per operation +- **Allocations per operation**: Number of memory allocations per operation + +Example output format: + +```text +BenchmarkMockerPlugin_PreHook_SimpleRule-8 5000000 250 ns/op 400 B/op 5 allocs/op +``` + +### 📈 **Performance Reference** + +As a reference, here are results from testing on a system with 16GB RAM: + +```text +BenchmarkMockerPlugin_PreHook_SimpleRule 6,306,205 ops 189.6 ns/op 416 B/op 5 allocs/op +BenchmarkMockerPlugin_PreHook_RegexRule 712,371 ops 1637 ns/op 420 B/op 5 allocs/op +BenchmarkMockerPlugin_PreHook_MultipleRules 5,604,916 ops 214.1 ns/op 416 B/op 5 allocs/op +BenchmarkMockerPlugin_PreHook_NoMatch 155,663,086 ops 7.7 ns/op 0 B/op 0 allocs/op +BenchmarkMockerPlugin_PreHook_Template 864,408 ops 1351 ns/op 1688 B/op 19 allocs/op +``` + +**Note**: Your results may vary based on your hardware configuration. Run the benchmarks yourself for accurate measurements on your system. + +### 🎯 **Performance Characteristics** + +Based on the optimizations implemented, you can expect: + +#### **Ultra-Fast No-Match Path** + +- Minimal overhead when no rules match +- Perfect for production with selective mocking +- Zero allocations when plugin is disabled + +#### **High-Speed Simple Rules** + +- Fast provider/model string matching +- Suitable for high-frequency benchmarking +- Minimal memory allocations + +#### **Efficient Regex Matching** + +- Pre-compiled patterns (much faster than runtime compilation) +- Good performance for pattern-based mocking +- Scales well with multiple regex rules + +#### **Multiple Rule Evaluation** + +- Priority-based early termination +- Performance doesn't degrade significantly with rule count +- Optimized rule traversal + +### ⚡ **Configuration for Maximum Performance** + +#### For **Benchmarking** (Maximum Speed): + +```go +config := mocker.MockerConfig{ + Enabled: true, + Rules: []mocker.MockRule{ + { + Name: "benchmark-rule", + Enabled: true, + Priority: 100, + Probability: 1.0, // Always match for consistent results + Conditions: mocker.Conditions{ + Providers: []string{"openai"}, // Simple string match (fastest) + }, + Responses: []mocker.Response{ + { + Type: mocker.ResponseTypeSuccess, + Content: &mocker.SuccessResponse{ + Message: "Benchmark response", // Static message (no templates) + Usage: &mocker.Usage{ // Pre-defined usage + PromptTokens: 10, + CompletionTokens: 20, + TotalTokens: 30, + }, + }, + }, + }, + }, + }, +} +``` + +#### For **Production** (Minimal Overhead): + +```go +config := mocker.MockerConfig{ + Enabled: true, + DefaultBehavior: mocker.DefaultBehaviorPassthrough, // Fast no-match path + Rules: []mocker.MockRule{ + // Only critical error simulation rules + { + Name: "rate-limit-sim", + Enabled: true, + Probability: 0.01, // 1% activation rate + Conditions: mocker.Conditions{ + Providers: []string{"openai"}, // Simple conditions only + }, + // ... error response + }, + }, +} +``` + +### 🔧 **Performance Tuning Tips** + +#### 1. **Rule Optimization** + +```go +// ✅ FAST: Simple string matching +Conditions: mocker.Conditions{ + Providers: []string{"openai", "anthropic"}, + Models: []string{"gpt-4", "claude-3"}, +} + +// ⚠️ SLOWER: Regex patterns (but still fast with pre-compilation) +Conditions: mocker.Conditions{ + MessageRegex: stringPtr(`(?i)error|fail`), +} + +// ❌ AVOID: Complex regex patterns +Conditions: mocker.Conditions{ + MessageRegex: stringPtr(`^.*complex.*nested.*(pattern|match).*$`), +} +``` + +#### 2. **Response Optimization** + +```go +// ✅ FAST: Static responses +Content: &mocker.SuccessResponse{ + Message: "Static response", + Usage: &predefinedUsage, // Reuse objects +} + +// ⚠️ MODERATE: Simple templates +Content: &mocker.SuccessResponse{ + MessageTemplate: stringPtr("Response from {{provider}}"), // Minimal variables +} + +// ❌ AVOID: Complex templates with many variables +Content: &mocker.SuccessResponse{ + MessageTemplate: stringPtr("Complex {{var1}} {{var2}} {{var3}} template"), +} +``` + +#### 3. **Rule Priority** + +```go +// ✅ Put most common rules first (higher priority) +Rules: []mocker.MockRule{ + {Priority: 100, /* most common conditions */}, + {Priority: 50, /* less common conditions */}, + {Priority: 10, /* rare conditions */}, +} +``` + +### 📈 **Monitoring Performance** + +Track performance metrics in your application: + +```go +func monitorMockerPerformance(plugin *mocker.MockerPlugin) { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + + lastStats := plugin.GetStats() + lastTime := time.Now() + + for { + select { + case <-ticker.C: + currentStats := plugin.GetStats() + currentTime := time.Now() + + duration := currentTime.Sub(lastTime) + requests := currentStats.TotalRequests - lastStats.TotalRequests + + rps := float64(requests) / duration.Seconds() + mockRate := float64(currentStats.MockedRequests) / float64(currentStats.TotalRequests) * 100 + + log.Printf("Mocker Performance: %.0f req/s, %.1f%% mock rate", rps, mockRate) + + lastStats = currentStats + lastTime = currentTime + } + } +} +``` + +**🏆 The Mocker plugin is optimized for high-throughput scenarios and adds minimal overhead to your application.** + +## Best Practices + +### 1. Rule Organization + +- **Use descriptive names**: `"rate-limit-openai"` instead of `"rule1"` +- **Set appropriate priorities**: Critical rules should have higher priority +- **Group related rules**: Keep similar functionality together + +### 2. Development vs Production + +```go +// Development - High mock rate +config := mocker.MockerConfig{ + Enabled: true, + DefaultBehavior: mocker.DefaultBehaviorSuccess, // Mock everything by default +} + +// Production - Selective mocking +config := mocker.MockerConfig{ + Enabled: true, + DefaultBehavior: mocker.DefaultBehaviorPassthrough, // Pass through by default + Rules: []mocker.MockRule{ + // Only specific error simulation rules + }, +} +``` + +### 3. Error Handling + +- **Use appropriate fallback settings**: Allow fallbacks for temporary errors +- **Provide meaningful error messages**: Help with debugging +- **Set realistic status codes**: Match actual provider behavior + +### 4. Performance Considerations + +- **Limit regex complexity**: Simple patterns perform better +- **Use probability wisely**: Don't mock 100% in production +- **Monitor statistics**: Watch for unexpected behavior + +### 5. Testing + +```go +func TestYourAppWithMocking(t *testing.T) { + // Create predictable mock responses for testing + plugin, _ := mocker.NewMockerPlugin(mocker.MockerConfig{ + Enabled: true, + Rules: []mocker.MockRule{ + { + Name: "test-success", + Enabled: true, + Probability: 1.0, // Always trigger for consistent tests + Responses: []mocker.Response{ + { + Type: mocker.ResponseTypeSuccess, + Content: &mocker.SuccessResponse{ + Message: "Predictable test response", + }, + }, + }, + }, + }, + }) + + // Use plugin in your tests... +} +``` + +## Troubleshooting + +### Common Issues + +#### 1. Plugin Not Triggering + +**Problem**: Requests pass through to real providers instead of being mocked. + +**Solutions**: + +- Check `Enabled: true` in config +- Verify rule conditions match your requests +- Check rule `Probability` (should be > 0) +- Ensure rule is `Enabled: true` + +#### 2. Configuration Validation Errors + +**Problem**: `NewMockerPlugin()` returns validation errors. + +**Common Issues**: + +```go +// ❌ Missing rule name +{ + Name: "", // Error: rule name required +} + +// ❌ Invalid probability +{ + Probability: 1.5, // Error: must be 0.0-1.0 +} + +// ❌ Invalid response type +{ + Type: "invalid", // Error: must be "success" or "error" +} + +// ❌ Missing response content +{ + Type: mocker.ResponseTypeSuccess, + // Error: Content required for success type +} +``` + +#### 3. Statistics Not Updating + +**Problem**: `GetStats()` shows zero values. + +**Solutions**: + +- Ensure rules are actually matching (check conditions) +- Verify plugin is enabled +- Call `GetStats()` before `Cleanup()` (cleanup clears stats) + +#### 4. Regex Not Matching + +**Problem**: `MessageRegex` conditions not working. + +**Solutions**: + +```go +// ❌ Invalid regex +MessageRegex: stringPtr("[invalid"), // Syntax error + +// ✅ Valid regex patterns +MessageRegex: stringPtr(`(?i)hello`), // Case-insensitive +MessageRegex: stringPtr(`error|fail|problem`), // Multiple options +MessageRegex: stringPtr(`\d+`), // Numbers only +``` + +#### 5. Unexpected Fallback Behavior + +**Problem**: Errors don't trigger fallbacks as expected. + +**Solutions**: + +```go +// Control fallback behavior explicitly +Response{ + Type: mocker.ResponseTypeError, + AllowFallbacks: boolPtr(true), // Explicitly allow fallbacks + // or + AllowFallbacks: boolPtr(false), // Explicitly block fallbacks + // or + AllowFallbacks: nil, // Default behavior (allow) +} +``` + +#### 6. Latency Not Working + +**Problem**: Latency simulation has no effect or causes errors. + +**Common Issue**: Using raw integers instead of `time.Duration` values. + +**Solutions**: + +```go +// ❌ WRONG: Raw integers (these are nanoseconds, barely noticeable) +Latency: &mocker.Latency{ + Type: mocker.LatencyTypeFixed, + Min: 100, // 100 nanoseconds = 0.0001ms + Max: 500, // 500 nanoseconds = 0.0005ms +} + +// ✅ CORRECT: Use time.Duration constants +Latency: &mocker.Latency{ + Type: mocker.LatencyTypeFixed, + Min: 100 * time.Millisecond, // 100ms +} + +// ✅ CORRECT: Various duration examples +Latency: &mocker.Latency{ + Type: mocker.LatencyTypeUniform, + Min: 50 * time.Millisecond, // 50ms + Max: 200 * time.Millisecond, // 200ms +} + +// ✅ CORRECT: Other duration units +Min: 1 * time.Second, // 1 second +Min: 500 * time.Microsecond, // 500 microseconds +Min: 2500 * time.Nanosecond, // 2500 nanoseconds (rarely used) +``` + +### Debug Mode + +Enable debug logging to troubleshoot issues: + +```go +client, err := bifrost.Init(schemas.BifrostConfig{ + Account: &account, + Plugins: []schemas.Plugin{plugin}, + Logger: bifrost.NewDefaultLogger(schemas.LogLevelDebug), // Enable debug logs +}) +``` + +### Validation Testing + +Test your configuration before deployment: + +```go +func validateMockerConfig(config mocker.MockerConfig) error { + _, err := mocker.NewMockerPlugin(config) + return err +} + +// Test in your code +if err := validateMockerConfig(yourConfig); err != nil { + log.Fatalf("Invalid mocker configuration: %v", err) +} +``` + +--- + +**Need help?** Check the [Bifrost documentation](../../docs/plugins.md) or open an issue on GitHub. + +``` + +``` diff --git a/plugins/mocker/benchmark_test.go b/plugins/mocker/benchmark_test.go new file mode 100644 index 0000000000..8568062eee --- /dev/null +++ b/plugins/mocker/benchmark_test.go @@ -0,0 +1,296 @@ +package mocker + +import ( + "context" + "strconv" + "testing" + + bifrost "github.com/maximhq/bifrost/core" + "github.com/maximhq/bifrost/core/schemas" +) + +// BenchmarkMockerPlugin_PreHook_SimpleRule benchmarks simple rule matching +func BenchmarkMockerPlugin_PreHook_SimpleRule(b *testing.B) { + plugin, err := NewMockerPlugin(MockerConfig{ + Enabled: true, + Rules: []MockRule{ + { + Name: "simple-rule", + Enabled: true, + Priority: 100, + Probability: 1.0, + Conditions: Conditions{ + Providers: []string{"openai"}, + }, + Responses: []Response{ + { + Type: ResponseTypeSuccess, + Content: &SuccessResponse{ + Message: "Benchmark response", + }, + }, + }, + }, + }, + }) + if err != nil { + b.Fatal(err) + } + + req := &schemas.BifrostRequest{ + Provider: schemas.OpenAI, + Model: "gpt-4", + Input: schemas.RequestInput{ + ChatCompletionInput: &[]schemas.BifrostMessage{ + { + Role: schemas.ModelChatMessageRoleUser, + Content: schemas.MessageContent{ + ContentStr: bifrost.Ptr("Hello, benchmark test"), + }, + }, + }, + }, + } + + ctx := context.Background() + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + _, _, _ = plugin.PreHook(&ctx, req) + } +} + +// BenchmarkMockerPlugin_PreHook_RegexRule benchmarks regex rule matching +func BenchmarkMockerPlugin_PreHook_RegexRule(b *testing.B) { + plugin, err := NewMockerPlugin(MockerConfig{ + Enabled: true, + Rules: []MockRule{ + { + Name: "regex-rule", + Enabled: true, + Priority: 100, + Probability: 1.0, + Conditions: Conditions{ + MessageRegex: bifrost.Ptr(`(?i).*hello.*`), + }, + Responses: []Response{ + { + Type: ResponseTypeSuccess, + Content: &SuccessResponse{ + Message: "Regex matched response", + }, + }, + }, + }, + }, + }) + if err != nil { + b.Fatal(err) + } + + req := &schemas.BifrostRequest{ + Provider: schemas.OpenAI, + Model: "gpt-4", + Input: schemas.RequestInput{ + ChatCompletionInput: &[]schemas.BifrostMessage{ + { + Role: schemas.ModelChatMessageRoleUser, + Content: schemas.MessageContent{ + ContentStr: bifrost.Ptr("Hello, this should match the regex pattern"), + }, + }, + }, + }, + } + + ctx := context.Background() + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + _, _, _ = plugin.PreHook(&ctx, req) + } +} + +// BenchmarkMockerPlugin_PreHook_MultipleRules benchmarks multiple rule evaluation +func BenchmarkMockerPlugin_PreHook_MultipleRules(b *testing.B) { + rules := make([]MockRule, 10) + for i := 0; i < 10; i++ { + rules[i] = MockRule{ + Name: "rule-" + strconv.Itoa(i), + Enabled: true, + Priority: 100 - i, // Descending priority + Probability: 1.0, + Conditions: Conditions{ + Models: []string{"gpt-" + strconv.Itoa(i)}, + }, + Responses: []Response{ + { + Type: ResponseTypeSuccess, + Content: &SuccessResponse{ + Message: "Response from rule " + strconv.Itoa(i), + }, + }, + }, + } + } + + // Add a matching rule at the end + rules = append(rules, MockRule{ + Name: "matching-rule", + Enabled: true, + Priority: 50, + Probability: 1.0, + Conditions: Conditions{ + Models: []string{"gpt-4"}, + }, + Responses: []Response{ + { + Type: ResponseTypeSuccess, + Content: &SuccessResponse{ + Message: "Matching rule response", + }, + }, + }, + }) + + plugin, err := NewMockerPlugin(MockerConfig{ + Enabled: true, + Rules: rules, + }) + if err != nil { + b.Fatal(err) + } + + req := &schemas.BifrostRequest{ + Provider: schemas.OpenAI, + Model: "gpt-4", + Input: schemas.RequestInput{ + ChatCompletionInput: &[]schemas.BifrostMessage{ + { + Role: schemas.ModelChatMessageRoleUser, + Content: schemas.MessageContent{ + ContentStr: bifrost.Ptr("Test message"), + }, + }, + }, + }, + } + + ctx := context.Background() + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + _, _, _ = plugin.PreHook(&ctx, req) + } +} + +// BenchmarkMockerPlugin_PreHook_NoMatch benchmarks when no rules match +func BenchmarkMockerPlugin_PreHook_NoMatch(b *testing.B) { + plugin, err := NewMockerPlugin(MockerConfig{ + Enabled: true, + DefaultBehavior: DefaultBehaviorPassthrough, + Rules: []MockRule{ + { + Name: "non-matching-rule", + Enabled: true, + Priority: 100, + Probability: 1.0, + Conditions: Conditions{ + Providers: []string{"anthropic"}, // Won't match OpenAI + }, + Responses: []Response{ + { + Type: ResponseTypeSuccess, + Content: &SuccessResponse{ + Message: "This won't match", + }, + }, + }, + }, + }, + }) + if err != nil { + b.Fatal(err) + } + + req := &schemas.BifrostRequest{ + Provider: schemas.OpenAI, // Different from rule condition + Model: "gpt-4", + Input: schemas.RequestInput{ + ChatCompletionInput: &[]schemas.BifrostMessage{ + { + Role: schemas.ModelChatMessageRoleUser, + Content: schemas.MessageContent{ + ContentStr: bifrost.Ptr("Test message"), + }, + }, + }, + }, + } + + ctx := context.Background() + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + _, _, _ = plugin.PreHook(&ctx, req) + } +} + +// BenchmarkMockerPlugin_PreHook_Template benchmarks template processing +func BenchmarkMockerPlugin_PreHook_Template(b *testing.B) { + plugin, err := NewMockerPlugin(MockerConfig{ + Enabled: true, + Rules: []MockRule{ + { + Name: "template-rule", + Enabled: true, + Priority: 100, + Probability: 1.0, + Conditions: Conditions{}, // Match all + Responses: []Response{ + { + Type: ResponseTypeSuccess, + Content: &SuccessResponse{ + MessageTemplate: bifrost.Ptr("Hello from {{provider}} using model {{model}}!"), + }, + }, + }, + }, + }, + }) + if err != nil { + b.Fatal(err) + } + + req := &schemas.BifrostRequest{ + Provider: schemas.OpenAI, + Model: "gpt-4", + Input: schemas.RequestInput{ + ChatCompletionInput: &[]schemas.BifrostMessage{ + { + Role: schemas.ModelChatMessageRoleUser, + Content: schemas.MessageContent{ + ContentStr: bifrost.Ptr("Test message"), + }, + }, + }, + }, + } + + ctx := context.Background() + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + _, _, _ = plugin.PreHook(&ctx, req) + } +} diff --git a/plugins/mocker/go.mod b/plugins/mocker/go.mod new file mode 100644 index 0000000000..017c189dd5 --- /dev/null +++ b/plugins/mocker/go.mod @@ -0,0 +1,37 @@ +module github.com/maximhq/bifrost/plugins/mocker + +go 1.24.1 + +require ( + github.com/jaswdr/faker/v2 v2.5.0 + github.com/maximhq/bifrost/core v1.1.5 +) + +require ( + cloud.google.com/go/compute/metadata v0.3.0 // indirect + github.com/andybalholm/brotli v1.1.1 // indirect + github.com/aws/aws-sdk-go-v2 v1.36.3 // indirect + github.com/aws/aws-sdk-go-v2/config v1.29.14 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.67 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 // indirect + github.com/aws/smithy-go v1.22.3 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/mark3labs/mcp-go v0.32.0 // indirect + github.com/spf13/cast v1.7.1 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasthttp v1.60.0 // indirect + github.com/yosida95/uritemplate/v3 v3.0.2 // indirect + golang.org/x/net v0.39.0 // indirect + golang.org/x/oauth2 v0.30.0 // indirect + golang.org/x/text v0.24.0 // indirect +) diff --git a/plugins/mocker/go.sum b/plugins/mocker/go.sum new file mode 100644 index 0000000000..73629fd310 --- /dev/null +++ b/plugins/mocker/go.sum @@ -0,0 +1,76 @@ +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM= +github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= +github.com/aws/aws-sdk-go-v2/config v1.29.14 h1:f+eEi/2cKCg9pqKBoAIwRGzVb70MRKqWX4dg1BDcSJM= +github.com/aws/aws-sdk-go-v2/config v1.29.14/go.mod h1:wVPHWcIFv3WO89w0rE10gzf17ZYy+UVS1Geq8Iei34g= +github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9Bc4+D7nZua0KGYOM= +github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 h1:1XuUZ8mYJw9B6lzAkXhqHlJd/XvaX32evhproijJEZY= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.19/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= +github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k= +github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jaswdr/faker/v2 v2.5.0 h1:KUYfnleIZMSHNp/q+rDk7XEuqUUL5FhfT19iTTFqF5o= +github.com/jaswdr/faker/v2 v2.5.0/go.mod h1:ROK8xwQV0hYOLDUtxCQgHGcl10jbVzIvqHxcIDdwY2Q= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mark3labs/mcp-go v0.32.0 h1:fgwmbfL2gbd67obg57OfV2Dnrhs1HtSdlY/i5fn7MU8= +github.com/mark3labs/mcp-go v0.32.0/go.mod h1:rXqOudj/djTORU/ThxYx8fqEVj/5pvTuuebQ2RC7uk4= +github.com/maximhq/bifrost/core v1.1.5 h1:Nm9XlS9Nso+pn+U5/btsJD8qRDYGQ1BBOjgqWT3PYSc= +github.com/maximhq/bifrost/core v1.1.5/go.mod h1:yMRCncTgKYBIrECSRVxMbY3BL8CjLbipJlc644jryxc= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= +github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.60.0 h1:kBRYS0lOhVJ6V+bYN8PqAHELKHtXqwq9zNMLKx1MBsw= +github.com/valyala/fasthttp v1.60.0/go.mod h1:iY4kDgV3Gc6EqhRZ8icqcmlG6bqhcDXfuHgTO4FXCvc= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= +github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= +golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= +golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/plugins/mocker/main.go b/plugins/mocker/main.go new file mode 100644 index 0000000000..1b32ff09a1 --- /dev/null +++ b/plugins/mocker/main.go @@ -0,0 +1,1083 @@ +package mocker + +import ( + "context" + "fmt" + "maps" + "math/rand" + "regexp" + "sort" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/jaswdr/faker/v2" + "github.com/maximhq/bifrost/core/schemas" +) + +const ( + PluginName = "bifrost-mocker" +) + +// Constants for type checking and validation +const ( + // Response types + ResponseTypeSuccess = "success" + ResponseTypeError = "error" + + // Default behaviors + DefaultBehaviorPassthrough = "passthrough" + DefaultBehaviorError = "error" + DefaultBehaviorSuccess = "success" + + // Latency types + LatencyTypeFixed = "fixed" + LatencyTypeUniform = "uniform" +) + +// compiledRule represents a rule with pre-compiled regex and normalized weights for performance +type compiledRule struct { + MockRule + compiledRegex *regexp.Regexp // Pre-compiled regex for fast matching + normalizedWeights []float64 // Pre-calculated normalized weights for fast response selection +} + +// MockerPlugin provides comprehensive request/response mocking capabilities +type MockerPlugin struct { + config MockerConfig + rules []MockRule + compiledRules []compiledRule // Pre-compiled rules for performance + mu sync.RWMutex + faker faker.Faker // Use jaswdr/faker library + + // Atomic counters for high-performance statistics tracking + totalRequests int64 + mockedRequests int64 + responsesGenerated int64 + errorsGenerated int64 + + // Rule hits tracking (still needs mutex for map access) + ruleHitsMu sync.RWMutex + ruleHits map[string]int64 +} + +// MockerConfig defines the overall configuration for the mocker plugin +type MockerConfig struct { + Enabled bool `json:"enabled"` // Enable/disable the mocker plugin + GlobalLatency *Latency `json:"global_latency"` // Global latency settings applied to all rules (can be overridden per rule) + Rules []MockRule `json:"rules"` // List of mock rules to be evaluated in priority order + DefaultBehavior string `json:"default_behavior"` // Action when no rules match: "passthrough", "error", or "success" +} + +// MockRule defines a single mocking rule with conditions and responses +// Rules are evaluated in priority order (higher numbers = higher priority) +type MockRule struct { + Name string `json:"name"` // Unique rule name for identification and statistics tracking + Enabled bool `json:"enabled"` // Enable/disable this rule (disabled rules are skipped) + Priority int `json:"priority"` // Higher priority rules are checked first (higher numbers = higher priority) + Conditions Conditions `json:"conditions"` // Conditions that must match for this rule to apply + Responses []Response `json:"responses"` // Possible responses (selected using weighted random selection) + Latency *Latency `json:"latency"` // Rule-specific latency override (overrides global latency if set) + Probability float64 `json:"probability"` // Probability of rule activation (0.0=never, 1.0=always, 0=disabled) +} + +// Conditions define when a mock rule should be applied +// All specified conditions must match for the rule to trigger +type Conditions struct { + Providers []string `json:"providers"` // Match specific providers (e.g., ["openai", "anthropic"]) + Models []string `json:"models"` // Match specific models (e.g., ["gpt-4", "claude-3"]) + MessageRegex *string `json:"message_regex"` // Regex pattern to match against message content + RequestSize *SizeRange `json:"request_size"` // Request size constraints in bytes +} + +// Response defines a mock response configuration +// Either Content (for success) or Error (for error) should be set, not both +type Response struct { + Type string `json:"type"` // Response type: "success" or "error" + Weight float64 `json:"weight"` // Weight for random selection (higher = more likely) + Content *SuccessResponse `json:"content"` // Success response content (required if Type="success") + Error *ErrorResponse `json:"error"` // Error response content (required if Type="error") + AllowFallbacks *bool `json:"allow_fallbacks"` // Control fallback behavior for errors (nil=true, false=no fallbacks) +} + +// SuccessResponse defines mock success response content +// Either Message or MessageTemplate should be set (MessageTemplate takes precedence) +type SuccessResponse struct { + Message string `json:"message"` // Static response message + Model *string `json:"model"` // Override model name in response (optional) + Usage *Usage `json:"usage"` // Token usage info (optional, defaults applied if nil) + FinishReason *string `json:"finish_reason"` // Completion reason (optional, defaults to "stop") + MessageTemplate *string `json:"message_template"` // Template with variables like {{model}}, {{provider}} (overrides Message) + CustomFields map[string]interface{} `json:"custom_fields"` // Additional fields stored in response metadata +} + +// ErrorResponse defines mock error response content +type ErrorResponse struct { + Message string `json:"message"` // Error message to return + Type *string `json:"type"` // Error type (e.g., "rate_limit", "auth_error") + Code *string `json:"code"` // Error code (e.g., "429", "401") + StatusCode *int `json:"status_code"` // HTTP status code for the error +} + +// Latency defines latency simulation settings +type Latency struct { + Min time.Duration `json:"min"` // Minimum latency as time.Duration (e.g., 100*time.Millisecond, NOT raw int) + Max time.Duration `json:"max"` // Maximum latency as time.Duration (e.g., 500*time.Millisecond, NOT raw int) + Type string `json:"type"` // Latency type: "fixed" or "uniform" +} + +// SizeRange defines request size constraints in bytes +type SizeRange struct { + Min int `json:"min"` // Minimum request size in bytes + Max int `json:"max"` // Maximum request size in bytes +} + +// Usage defines token usage information +type Usage struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` +} + +// MockStats tracks plugin statistics and rule execution counts +type MockStats struct { + TotalRequests int64 `json:"total_requests"` // Total number of requests processed + MockedRequests int64 `json:"mocked_requests"` // Number of requests that were mocked (rules matched) + RuleHits map[string]int64 `json:"rule_hits"` // Rule name -> hit count mapping + ErrorsGenerated int64 `json:"errors_generated"` // Number of error responses generated + ResponsesGenerated int64 `json:"responses_generated"` // Number of success responses generated +} + +// NewMockerPlugin creates a new mocker plugin instance with sensible defaults +// Returns an error if required configuration is invalid or missing +func NewMockerPlugin(config MockerConfig) (*MockerPlugin, error) { + // Validate configuration + if err := validateConfig(config); err != nil { + return nil, fmt.Errorf("invalid mocker plugin configuration: %w", err) + } + + // Apply defaults if not set + if config.DefaultBehavior == "" { + config.DefaultBehavior = DefaultBehaviorPassthrough // Default to passthrough if no rules match + } + + // If no rules provided, create a simple catch-all rule for quick testing + if len(config.Rules) == 0 && config.Enabled { + config.Rules = []MockRule{ + { + Name: "default-mock", + Enabled: true, + Priority: 1, + Conditions: Conditions{}, // Empty conditions = match all requests + Probability: 1.0, // Always activate + Responses: []Response{ + { + Type: ResponseTypeSuccess, + Weight: 1.0, + Content: &SuccessResponse{ + Message: "This is a mock response from the Mocker plugin", + }, + }, + }, + }, + } + } + + plugin := &MockerPlugin{ + config: config, + rules: config.Rules, + ruleHits: make(map[string]int64), + faker: faker.New(), // Initialize faker + } + + // Pre-compile all regex patterns for performance + if err := plugin.compileRules(); err != nil { + return nil, fmt.Errorf("failed to compile rules: %w", err) + } + + return plugin, nil +} + +// compileRules pre-compiles all regex patterns and calculates normalized weights for performance +func (p *MockerPlugin) compileRules() error { + p.compiledRules = make([]compiledRule, 0, len(p.rules)) + + for _, rule := range p.rules { + compiled := compiledRule{MockRule: rule} + + // Pre-compile regex if present + if rule.Conditions.MessageRegex != nil { + regex, err := regexp.Compile(*rule.Conditions.MessageRegex) + if err != nil { + return fmt.Errorf("invalid regex in rule '%s': %w", rule.Name, err) + } + compiled.compiledRegex = regex + } + + // Pre-calculate normalized weights for fast response selection + compiled.normalizedWeights = p.calculateNormalizedWeights(rule.Responses) + + p.compiledRules = append(p.compiledRules, compiled) + } + + // Sort compiled rules by priority (higher first) + p.sortCompiledRulesByPriority() + + return nil +} + +// calculateNormalizedWeights pre-calculates normalized cumulative weights for fast response selection +func (p *MockerPlugin) calculateNormalizedWeights(responses []Response) []float64 { + if len(responses) == 0 { + return nil + } + + if len(responses) == 1 { + return []float64{1.0} // Single response always gets 100% probability + } + + // Calculate total weight, applying default weight of 1.0 if not specified + totalWeight := 0.0 + for _, response := range responses { + weight := response.Weight + if weight == 0 { + weight = 1.0 // Default weight + } + totalWeight += weight + } + + // Calculate normalized cumulative weights for O(1) selection + normalizedWeights := make([]float64, len(responses)) + cumulativeWeight := 0.0 + + for i, response := range responses { + weight := response.Weight + if weight == 0 { + weight = 1.0 // Default weight + } + cumulativeWeight += weight / totalWeight // Normalize to [0, 1] + normalizedWeights[i] = cumulativeWeight + } + + // Ensure the last weight is exactly 1.0 to handle floating point precision issues + if len(normalizedWeights) > 0 { + normalizedWeights[len(normalizedWeights)-1] = 1.0 + } + + return normalizedWeights +} + +// validateConfig validates the mocker plugin configuration +func validateConfig(config MockerConfig) error { + // Validate default behavior + if config.DefaultBehavior != "" { + switch config.DefaultBehavior { + case DefaultBehaviorPassthrough, DefaultBehaviorError, DefaultBehaviorSuccess: + // Valid + default: + return fmt.Errorf("invalid default_behavior '%s', must be one of: %s, %s, %s", + config.DefaultBehavior, DefaultBehaviorPassthrough, DefaultBehaviorError, DefaultBehaviorSuccess) + } + } + + // Validate global latency if provided + if config.GlobalLatency != nil { + if err := validateLatency(*config.GlobalLatency); err != nil { + return fmt.Errorf("invalid global_latency: %w", err) + } + } + + // Validate each rule + for i, rule := range config.Rules { + if err := validateRule(rule); err != nil { + return fmt.Errorf("invalid rule at index %d (%s): %w", i, rule.Name, err) + } + } + + return nil +} + +// validateRule validates a single mock rule +func validateRule(rule MockRule) error { + // Rule name is required + if rule.Name == "" { + return fmt.Errorf("rule name is required") + } + + // Priority should be reasonable (allow negative for low priority) + if rule.Priority < -1000 || rule.Priority > 1000 { + return fmt.Errorf("priority %d is out of reasonable range (-1000 to 1000)", rule.Priority) + } + + // Probability must be between 0 and 1 + if rule.Probability < 0 || rule.Probability > 1 { + return fmt.Errorf("probability %.2f must be between 0.0 and 1.0", rule.Probability) + } + + // At least one response is required + if len(rule.Responses) == 0 { + return fmt.Errorf("at least one response is required") + } + + // Validate rule-specific latency if provided + if rule.Latency != nil { + if err := validateLatency(*rule.Latency); err != nil { + return fmt.Errorf("invalid rule latency: %w", err) + } + } + + // Validate conditions + if err := validateConditions(rule.Conditions); err != nil { + return fmt.Errorf("invalid conditions: %w", err) + } + + // Validate each response + for i, response := range rule.Responses { + if err := validateResponse(response); err != nil { + return fmt.Errorf("invalid response at index %d: %w", i, err) + } + } + + return nil +} + +// validateLatency validates latency configuration +func validateLatency(latency Latency) error { + // Type is required + if latency.Type == "" { + return fmt.Errorf("latency type is required") + } + + // Validate type + switch latency.Type { + case LatencyTypeFixed, LatencyTypeUniform: + // Valid + default: + return fmt.Errorf("invalid latency type '%s', must be one of: %s, %s", + latency.Type, LatencyTypeFixed, LatencyTypeUniform) + } + + // Min latency should be non-negative + if latency.Min < 0 { + return fmt.Errorf("minimum latency cannot be negative") + } + + // For uniform type, max should be >= min + if latency.Type == LatencyTypeUniform { + if latency.Max < latency.Min { + return fmt.Errorf("maximum latency (%v) cannot be less than minimum latency (%v)", latency.Max, latency.Min) + } + } + + return nil +} + +// validateConditions validates rule conditions +func validateConditions(conditions Conditions) error { + // Validate regex if provided + if conditions.MessageRegex != nil { + _, err := regexp.Compile(*conditions.MessageRegex) + if err != nil { + return fmt.Errorf("invalid message regex '%s': %w", *conditions.MessageRegex, err) + } + } + + // Validate request size range if provided + if conditions.RequestSize != nil { + if conditions.RequestSize.Min < 0 { + return fmt.Errorf("request size minimum cannot be negative") + } + if conditions.RequestSize.Max < conditions.RequestSize.Min { + return fmt.Errorf("request size maximum (%d) cannot be less than minimum (%d)", + conditions.RequestSize.Max, conditions.RequestSize.Min) + } + } + + return nil +} + +// validateResponse validates a response configuration +func validateResponse(response Response) error { + // Type is required + if response.Type == "" { + return fmt.Errorf("response type is required") + } + + // Validate type + switch response.Type { + case ResponseTypeSuccess, ResponseTypeError: + // Valid + default: + return fmt.Errorf("invalid response type '%s', must be one of: %s, %s", + response.Type, ResponseTypeSuccess, ResponseTypeError) + } + + // Weight should be non-negative + if response.Weight < 0 { + return fmt.Errorf("response weight cannot be negative") + } + + // Validate response content based on type + if response.Type == ResponseTypeSuccess { + if response.Content == nil { + return fmt.Errorf("success response must have content") + } + if err := validateSuccessResponse(*response.Content); err != nil { + return fmt.Errorf("invalid success content: %w", err) + } + } else if response.Type == ResponseTypeError { + if response.Error == nil { + return fmt.Errorf("error response must have error content") + } + if err := validateErrorResponse(*response.Error); err != nil { + return fmt.Errorf("invalid error content: %w", err) + } + } + + return nil +} + +// validateSuccessResponse validates success response content +func validateSuccessResponse(content SuccessResponse) error { + // Either Message or MessageTemplate must be provided + if content.Message == "" && (content.MessageTemplate == nil || *content.MessageTemplate == "") { + return fmt.Errorf("either message or message_template is required") + } + + // If usage is provided, validate it + if content.Usage != nil { + if content.Usage.PromptTokens < 0 || content.Usage.CompletionTokens < 0 || content.Usage.TotalTokens < 0 { + return fmt.Errorf("token counts cannot be negative") + } + } + + return nil +} + +// validateErrorResponse validates error response content +func validateErrorResponse(errorContent ErrorResponse) error { + // Message is required + if errorContent.Message == "" { + return fmt.Errorf("error message is required") + } + + // Status code should be reasonable if provided + if errorContent.StatusCode != nil { + if *errorContent.StatusCode < 100 || *errorContent.StatusCode > 599 { + return fmt.Errorf("status code %d is out of valid HTTP range (100-599)", *errorContent.StatusCode) + } + } + + return nil +} + +// GetName returns the plugin name +func (p *MockerPlugin) GetName() string { + return PluginName +} + +// PreHook intercepts requests and applies mocking rules based on configuration +// This is called before the actual provider request and can short-circuit the flow +func (p *MockerPlugin) PreHook(ctx *context.Context, req *schemas.BifrostRequest) (*schemas.BifrostRequest, *schemas.PluginShortCircuit, error) { + // Skip processing if plugin is disabled + if !p.config.Enabled { + return req, nil, nil + } + + // Track total request count using atomic operation (no lock needed) + atomic.AddInt64(&p.totalRequests, 1) + + // Find the first matching rule based on priority order + rule := p.findMatchingCompiledRule(req) + if rule == nil { + // No rules matched, handle according to default behavior + return p.handleDefaultBehavior(req) + } + + // Check if rule should activate based on probability (0.0 = never, 1.0 = always) + if rule.Probability > 0 && rand.Float64() > rule.Probability { + // Rule didn't activate due to probability, continue with normal flow + return req, nil, nil + } + + // Apply artificial latency simulation if configured + if latency := p.getLatency(&rule.MockRule); latency != nil { + delay := p.calculateLatency(latency) + time.Sleep(delay) + } + + // Select a response from the rule's possible responses using pre-calculated weights + response := p.selectResponse(rule) + if response == nil { + // No valid response configuration, continue with normal flow + return req, nil, nil + } + + // Update statistics using atomic operations and minimal locking + atomic.AddInt64(&p.mockedRequests, 1) + + // Rule hits still need a mutex since it's a map, but we minimize lock time + p.ruleHitsMu.Lock() + p.ruleHits[rule.Name]++ + p.ruleHitsMu.Unlock() + + // Generate appropriate mock response based on type + if response.Type == ResponseTypeSuccess { + return p.generateSuccessShortCircuit(req, response) + } else if response.Type == ResponseTypeError { + return p.generateErrorShortCircuit(req, response) + } + + // Fallback: continue with normal flow if response type is unrecognized + return req, nil, nil +} + +// PostHook processes responses after provider calls +func (p *MockerPlugin) PostHook(ctx *context.Context, result *schemas.BifrostResponse, err *schemas.BifrostError) (*schemas.BifrostResponse, *schemas.BifrostError, error) { + return result, err, nil +} + +// Cleanup performs plugin cleanup and frees memory +// IMPORTANT: Call GetStats() before Cleanup() if you need the statistics, +// as this method clears all statistics data to free memory +func (p *MockerPlugin) Cleanup() error { + p.mu.Lock() + defer p.mu.Unlock() + + // Clear all statistics to free memory using atomic operations + atomic.StoreInt64(&p.totalRequests, 0) + atomic.StoreInt64(&p.mockedRequests, 0) + atomic.StoreInt64(&p.responsesGenerated, 0) + atomic.StoreInt64(&p.errorsGenerated, 0) + + // Clear rule hits map + p.ruleHitsMu.Lock() + p.ruleHits = make(map[string]int64) + p.ruleHitsMu.Unlock() + + // Clear rules to free memory + p.rules = nil + p.compiledRules = nil + + return nil +} + +// findMatchingCompiledRule finds the first rule that matches the request using pre-compiled rules +func (p *MockerPlugin) findMatchingCompiledRule(req *schemas.BifrostRequest) *compiledRule { + for i := range p.compiledRules { + rule := &p.compiledRules[i] + if !rule.Enabled { + continue + } + + if p.matchesConditionsFast(req, &rule.Conditions, rule.compiledRegex) { + return rule + } + } + return nil +} + +// matchesConditionsFast checks if request matches rule conditions with optimized performance +func (p *MockerPlugin) matchesConditionsFast(req *schemas.BifrostRequest, conditions *Conditions, compiledRegex *regexp.Regexp) bool { + // Check providers - optimized string comparison + if len(conditions.Providers) > 0 { + providerStr := string(req.Provider) + found := false + for _, provider := range conditions.Providers { + if providerStr == provider { + found = true + break + } + } + if !found { + return false + } + } + + // Check models - direct string comparison + if len(conditions.Models) > 0 { + found := false + for _, model := range conditions.Models { + if req.Model == model { + found = true + break + } + } + if !found { + return false + } + } + + // Check message regex using pre-compiled regex (major performance improvement) + if compiledRegex != nil { + // Extract message content from request (cached if possible) + messageContent := p.extractMessageContentFast(req) + if !compiledRegex.MatchString(messageContent) { + return false + } + } + + // Check request size - only calculate if needed + if conditions.RequestSize != nil { + size := p.calculateRequestSizeFast(req) + if size < conditions.RequestSize.Min || size > conditions.RequestSize.Max { + return false + } + } + + // All conditions matched + return true +} + +// extractMessageContentFast extracts message content with optimized performance +func (p *MockerPlugin) extractMessageContentFast(req *schemas.BifrostRequest) string { + // Handle text completion input + if req.Input.TextCompletionInput != nil { + return *req.Input.TextCompletionInput + } + + // Handle chat completion input - optimized for common cases + if req.Input.ChatCompletionInput != nil { + messages := *req.Input.ChatCompletionInput + if len(messages) == 0 { + return "" + } + + // Fast path for single message + if len(messages) == 1 { + if messages[0].Content.ContentStr != nil { + return *messages[0].Content.ContentStr + } + return "" + } + + // Multiple messages - use string builder for efficiency + var builder strings.Builder + for i, message := range messages { + if message.Content.ContentStr != nil { + if i > 0 { + builder.WriteByte(' ') + } + builder.WriteString(*message.Content.ContentStr) + } + } + return builder.String() + } + + return "" +} + +// calculateRequestSizeFast calculates request size with minimal overhead +func (p *MockerPlugin) calculateRequestSizeFast(req *schemas.BifrostRequest) int { + // Approximate size calculation to avoid expensive JSON marshaling + size := len(req.Model) + len(string(req.Provider)) + + // Add input size + if req.Input.TextCompletionInput != nil { + size += len(*req.Input.TextCompletionInput) + } + + if req.Input.ChatCompletionInput != nil { + for _, message := range *req.Input.ChatCompletionInput { + if message.Content.ContentStr != nil { + size += len(*message.Content.ContentStr) + } + size += 50 // Approximate overhead for message structure + } + } + + return size +} + +// generateSuccessShortCircuit creates a success response short-circuit with optimized allocations +func (p *MockerPlugin) generateSuccessShortCircuit(req *schemas.BifrostRequest, response *Response) (*schemas.BifrostRequest, *schemas.PluginShortCircuit, error) { + if response.Content == nil { + return req, nil, nil + } + + content := response.Content + message := content.Message + + // Apply message template if provided + if content.MessageTemplate != nil { + message = p.applyTemplate(*content.MessageTemplate, req) + } + + // Apply defaults for token usage if not provided + var usage schemas.LLMUsage + if content.Usage != nil { + usage = schemas.LLMUsage{ + PromptTokens: p.getOrDefault(content.Usage.PromptTokens, 10), + CompletionTokens: p.getOrDefault(content.Usage.CompletionTokens, 20), + TotalTokens: p.getOrDefault(content.Usage.TotalTokens, content.Usage.PromptTokens+content.Usage.CompletionTokens), + } + } else { + // Default usage when none specified + usage = schemas.LLMUsage{ + PromptTokens: 10, + CompletionTokens: 20, + TotalTokens: 30, + } + } + + // Get finish reason with minimal allocation + var finishReason *string + if content.FinishReason != nil { + finishReason = content.FinishReason + } else { + // Use a static string to avoid allocation + static := "stop" + finishReason = &static + } + + // Create mock response with proper structure + mockResponse := &schemas.BifrostResponse{ + Model: req.Model, + Usage: usage, + Choices: []schemas.BifrostResponseChoice{ + { + Index: 0, + Message: schemas.BifrostMessage{ + Role: schemas.ModelChatMessageRoleAssistant, + Content: schemas.MessageContent{ + ContentStr: &message, + }, + }, + FinishReason: finishReason, + }, + }, + ExtraFields: schemas.BifrostResponseExtraFields{ + Provider: req.Provider, + }, + } + + // Override model if specified + if content.Model != nil { + mockResponse.Model = *content.Model + } + + // Only create raw response map if there are custom fields (avoid allocation) + if len(content.CustomFields) > 0 { + rawResponse := make(map[string]interface{}, len(content.CustomFields)+1) + + // Add custom fields + for key, value := range content.CustomFields { + rawResponse[key] = value + } + + // Add mock metadata + rawResponse["mock_rule"] = "success" + mockResponse.ExtraFields.RawResponse = rawResponse + } + + // Increment success response counter using atomic operation + atomic.AddInt64(&p.responsesGenerated, 1) + + return req, &schemas.PluginShortCircuit{ + Response: mockResponse, + }, nil +} + +// generateErrorShortCircuit creates an error response short-circuit with optimized performance +func (p *MockerPlugin) generateErrorShortCircuit(req *schemas.BifrostRequest, response *Response) (*schemas.BifrostRequest, *schemas.PluginShortCircuit, error) { + if response.Error == nil { + return req, nil, nil + } + + errorContent := response.Error + allowFallbacks := response.AllowFallbacks + + // Create mock error + mockError := &schemas.BifrostError{ + Error: schemas.ErrorField{ + Message: errorContent.Message, + }, + AllowFallbacks: allowFallbacks, + } + + // Set error type + if errorContent.Type != nil { + mockError.Error.Type = errorContent.Type + } + + // Set error code + if errorContent.Code != nil { + mockError.Error.Code = errorContent.Code + } + + // Set status code + if errorContent.StatusCode != nil { + mockError.StatusCode = errorContent.StatusCode + } + + // Increment error counter using atomic operation + atomic.AddInt64(&p.errorsGenerated, 1) + + return req, &schemas.PluginShortCircuit{ + Error: mockError, + }, nil +} + +// selectResponse selects a response using pre-calculated normalized weights for optimal performance +func (p *MockerPlugin) selectResponse(rule *compiledRule) *Response { + responses := rule.Responses + normalizedWeights := rule.normalizedWeights + + if len(responses) == 0 { + return nil + } + + if len(responses) == 1 { + return &responses[0] + } + + // Fast O(log n) binary search using pre-calculated cumulative weights + randomValue := rand.Float64() + + // Binary search for the selected response + left, right := 0, len(normalizedWeights)-1 + for left < right { + mid := (left + right) / 2 + if randomValue <= normalizedWeights[mid] { + right = mid + } else { + left = mid + 1 + } + } + + return &responses[left] +} + +// getLatency returns the applicable latency configuration +func (p *MockerPlugin) getLatency(rule *MockRule) *Latency { + if rule.Latency != nil { + return rule.Latency + } + return p.config.GlobalLatency +} + +// calculateLatency calculates the actual delay based on latency configuration +func (p *MockerPlugin) calculateLatency(latency *Latency) time.Duration { + switch latency.Type { + case LatencyTypeFixed: + return latency.Min + case LatencyTypeUniform: + if latency.Max <= latency.Min { + return latency.Min + } + // Calculate random duration between Min and Max + diff := latency.Max - latency.Min + return latency.Min + time.Duration(rand.Float64()*float64(diff)) + default: + // Default to fixed latency + return latency.Min + } +} + +// handleDefaultBehavior handles requests when no rules match +func (p *MockerPlugin) handleDefaultBehavior(req *schemas.BifrostRequest) (*schemas.BifrostRequest, *schemas.PluginShortCircuit, error) { + switch p.config.DefaultBehavior { + case DefaultBehaviorError: + return req, &schemas.PluginShortCircuit{ + Error: &schemas.BifrostError{ + Error: schemas.ErrorField{ + Message: "Mock plugin default error", + }, + }, + }, nil + case DefaultBehaviorSuccess: + finishReason := "stop" + return req, &schemas.PluginShortCircuit{ + Response: &schemas.BifrostResponse{ + Model: req.Model, + Usage: schemas.LLMUsage{ + PromptTokens: 5, + CompletionTokens: 10, + TotalTokens: 15, + }, + Choices: []schemas.BifrostResponseChoice{ + { + Index: 0, + Message: schemas.BifrostMessage{ + Role: schemas.ModelChatMessageRoleAssistant, + Content: schemas.MessageContent{ + ContentStr: func() *string { s := "Mock plugin default response"; return &s }(), + }, + }, + FinishReason: &finishReason, + }, + }, + ExtraFields: schemas.BifrostResponseExtraFields{ + Provider: req.Provider, + }, + }, + }, nil + default: // DefaultBehaviorPassthrough + return req, nil, nil + } +} + +// Helper functions + +// sortCompiledRulesByPriority sorts rules by priority (descending) +func (p *MockerPlugin) sortCompiledRulesByPriority() { + sort.Slice(p.compiledRules, func(i, j int) bool { + return p.compiledRules[i].Priority > p.compiledRules[j].Priority + }) +} + +// applyTemplate applies template variables with optimized string operations including faker support +func (p *MockerPlugin) applyTemplate(template string, req *schemas.BifrostRequest) string { + // Fast path: no template variables + if !strings.Contains(template, "{{") { + return template + } + + result := template + + // Replace basic variables first + replacer := strings.NewReplacer( + "{{provider}}", string(req.Provider), + "{{model}}", req.Model, + ) + result = replacer.Replace(result) + + // Handle faker variables with regex for more complex patterns + fakerRegex := regexp.MustCompile(`\{\{faker\.([^}]+)\}\}`) + result = fakerRegex.ReplaceAllStringFunc(result, func(match string) string { + // Extract the faker method name + submatch := fakerRegex.FindStringSubmatch(match) + if len(submatch) < 2 { + return match // Return original if no match + } + + fakerMethod := submatch[1] + return p.generateFakerValue(fakerMethod) + }) + + return result +} + +// generateFakerValue generates fake data based on the faker method name +func (p *MockerPlugin) generateFakerValue(method string) string { + // Parse method with potential parameters (e.g., "lorem_ipsum:20" for 20 words) + parts := strings.Split(method, ":") + baseMethod := parts[0] + + switch baseMethod { + case "name": + return p.faker.Person().Name() + case "first_name": + return p.faker.Person().FirstName() + case "last_name": + return p.faker.Person().LastName() + case "email": + return p.faker.Internet().Email() + case "phone": + return p.faker.Phone().Number() + case "address": + return p.faker.Address().Address() + case "city": + return p.faker.Address().City() + case "state": + return p.faker.Address().State() + case "zip_code": + return p.faker.Address().PostCode() + case "company": + return p.faker.Company().Name() + case "job_title": + return p.faker.Company().JobTitle() + case "lorem_ipsum": + wordCount := 10 // default + if len(parts) > 1 { + if count, err := fmt.Sscanf(parts[1], "%d", &wordCount); err != nil || count != 1 { + wordCount = 10 + } + } + return p.faker.Lorem().Sentence(wordCount) + case "uuid": + return p.faker.UUID().V4() + case "hex_color": + return p.faker.Color().Hex() + case "integer": + min, max := 1, 100 // defaults + if len(parts) > 1 { + params := strings.Split(parts[1], ",") + if len(params) >= 2 { + if _, err := fmt.Sscanf(params[0], "%d", &min); err != nil { + min = 1 // fallback to default on parse error + } + if _, err := fmt.Sscanf(params[1], "%d", &max); err != nil { + max = 100 // fallback to default on parse error + } + } + } + return fmt.Sprintf("%d", p.faker.IntBetween(min, max)) + case "float": + min, max := 0, 100 // defaults as integers + if len(parts) > 1 { + params := strings.Split(parts[1], ",") + if len(params) >= 2 { + if _, err := fmt.Sscanf(params[0], "%d", &min); err != nil { + min = 0 // fallback to default on parse error + } + if _, err := fmt.Sscanf(params[1], "%d", &max); err != nil { + max = 100 // fallback to default on parse error + } + } + } + return fmt.Sprintf("%.2f", p.faker.Float64(2, min, max)) + case "boolean": + return fmt.Sprintf("%t", p.faker.Bool()) + case "date": + return p.faker.Time().Time(time.Now()).Format("2006-01-02") + case "datetime": + return p.faker.Time().Time(time.Now()).Format("2006-01-02 15:04:05") + case "word": + return p.faker.Lorem().Word() + case "sentence": + wordCount := 8 // default + if len(parts) > 1 { + if count, err := fmt.Sscanf(parts[1], "%d", &wordCount); err != nil || count != 1 { + wordCount = 8 + } + } + return p.faker.Lorem().Sentence(wordCount) + default: + // Return the original placeholder if method is not recognized + return fmt.Sprintf("{{faker.%s}}", method) + } +} + +// getOrDefault returns value or default if 0 +func (p *MockerPlugin) getOrDefault(value, defaultValue int) int { + if value == 0 { + return defaultValue + } + return value +} + +// GetStats returns current plugin statistics +// IMPORTANT: Call this method before Cleanup() if you need the statistics, +// as Cleanup() clears all statistics data to free memory +func (p *MockerPlugin) GetStats() MockStats { + p.mu.RLock() + defer p.mu.RUnlock() + + // Create a deep copy using atomic reads for counters + statsCopy := MockStats{ + TotalRequests: atomic.LoadInt64(&p.totalRequests), + MockedRequests: atomic.LoadInt64(&p.mockedRequests), + ErrorsGenerated: atomic.LoadInt64(&p.errorsGenerated), + ResponsesGenerated: atomic.LoadInt64(&p.responsesGenerated), + RuleHits: make(map[string]int64), + } + + // Copy rule hits map (still needs lock) + p.ruleHitsMu.RLock() + maps.Copy(statsCopy.RuleHits, p.ruleHits) + p.ruleHitsMu.RUnlock() + + return statsCopy +} diff --git a/plugins/mocker/plugin_test.go b/plugins/mocker/plugin_test.go new file mode 100644 index 0000000000..2112df26c6 --- /dev/null +++ b/plugins/mocker/plugin_test.go @@ -0,0 +1,538 @@ +package mocker + +import ( + "context" + "testing" + + bifrost "github.com/maximhq/bifrost/core" + "github.com/maximhq/bifrost/core/schemas" +) + +// BaseAccount implements the schemas.Account interface for testing purposes. +// It provides mock implementations of the required methods to test the Mocker plugin +// with a basic OpenAI configuration. +type BaseAccount struct{} + +// GetConfiguredProviders returns a list of supported providers for testing. +func (baseAccount *BaseAccount) GetConfiguredProviders() ([]schemas.ModelProvider, error) { + return []schemas.ModelProvider{schemas.OpenAI, schemas.Anthropic}, nil +} + +// GetKeysForProvider returns a dummy API key configuration for testing. +// Since we're testing the mocker plugin, these keys should never be used +// as the plugin intercepts requests before they reach the actual providers. +func (baseAccount *BaseAccount) GetKeysForProvider(providerKey schemas.ModelProvider) ([]schemas.Key, error) { + return []schemas.Key{ + { + Value: "dummy-api-key-for-testing", // Dummy key + Models: []string{"gpt-4", "gpt-4-turbo", "claude-3"}, + Weight: 1.0, + }, + }, nil +} + +// GetConfigForProvider returns default provider configuration for testing. +func (baseAccount *BaseAccount) GetConfigForProvider(providerKey schemas.ModelProvider) (*schemas.ProviderConfig, error) { + return &schemas.ProviderConfig{ + NetworkConfig: schemas.DefaultNetworkConfig, + ConcurrencyAndBufferSize: schemas.DefaultConcurrencyAndBufferSize, + }, nil +} + +// TestMockerPlugin_GetName tests the plugin name +func TestMockerPlugin_GetName(t *testing.T) { + plugin, err := NewMockerPlugin(MockerConfig{}) + if err != nil { + t.Fatalf("Expected no error creating plugin, got: %v", err) + } + if plugin.GetName() != PluginName { + t.Errorf("Expected '%s', got '%s'", PluginName, plugin.GetName()) + } +} + +// TestMockerPlugin_Disabled tests that disabled plugin doesn't interfere +func TestMockerPlugin_Disabled(t *testing.T) { + config := MockerConfig{ + Enabled: false, + } + plugin, err := NewMockerPlugin(config) + if err != nil { + t.Fatalf("Expected no error creating plugin, got: %v", err) + } + + account := BaseAccount{} + client, err := bifrost.Init(schemas.BifrostConfig{ + Account: &account, + Plugins: []schemas.Plugin{plugin}, + Logger: bifrost.NewDefaultLogger(schemas.LogLevelError), + }) + if err != nil { + t.Fatalf("Error initializing Bifrost: %v", err) + } + defer client.Cleanup() + + // This should pass through to the real provider (but will fail due to dummy key) + _, bifrostErr := client.ChatCompletionRequest(context.Background(), &schemas.BifrostRequest{ + Provider: schemas.OpenAI, + Model: "gpt-4", + Input: schemas.RequestInput{ + ChatCompletionInput: &[]schemas.BifrostMessage{ + { + Role: schemas.ModelChatMessageRoleUser, + Content: schemas.MessageContent{ + ContentStr: bifrost.Ptr("Hello, test message"), + }, + }, + }, + }, + }) + + // Should get an authentication error from OpenAI, not a mock response + // This proves the plugin is disabled and not intercepting requests + if bifrostErr == nil { + t.Error("Expected error from real provider with dummy API key") + } +} + +// TestMockerPlugin_DefaultMockRule tests the default catch-all rule +func TestMockerPlugin_DefaultMockRule(t *testing.T) { + config := MockerConfig{ + Enabled: true, // No rules provided, should create default rule + } + plugin, err := NewMockerPlugin(config) + if err != nil { + t.Fatalf("Expected no error creating plugin, got: %v", err) + } + + account := BaseAccount{} + client, err := bifrost.Init(schemas.BifrostConfig{ + Account: &account, + Plugins: []schemas.Plugin{plugin}, + Logger: bifrost.NewDefaultLogger(schemas.LogLevelError), + }) + if err != nil { + t.Fatalf("Error initializing Bifrost: %v", err) + } + defer client.Cleanup() + + response, bifrostErr := client.ChatCompletionRequest(context.Background(), &schemas.BifrostRequest{ + Provider: schemas.OpenAI, + Model: "gpt-4", + Input: schemas.RequestInput{ + ChatCompletionInput: &[]schemas.BifrostMessage{ + { + Role: schemas.ModelChatMessageRoleUser, + Content: schemas.MessageContent{ + ContentStr: bifrost.Ptr("Hello, test message"), + }, + }, + }, + }, + }) + + if bifrostErr != nil { + t.Fatalf("Expected no error, got: %v", bifrostErr) + } + if response == nil { + t.Fatal("Expected response") + } + if len(response.Choices) == 0 { + t.Fatal("Expected at least one choice") + } + if response.Choices[0].Message.Content.ContentStr == nil { + t.Fatal("Expected content string") + } + if *response.Choices[0].Message.Content.ContentStr != "This is a mock response from the Mocker plugin" { + t.Errorf("Expected default mock message, got: %s", *response.Choices[0].Message.Content.ContentStr) + } +} + +// TestMockerPlugin_CustomSuccessRule tests custom success response +func TestMockerPlugin_CustomSuccessRule(t *testing.T) { + config := MockerConfig{ + Enabled: true, + Rules: []MockRule{ + { + Name: "openai-success", + Enabled: true, + Priority: 100, + Probability: 1.0, + Conditions: Conditions{ + Providers: []string{"openai"}, + }, + Responses: []Response{ + { + Type: ResponseTypeSuccess, + Content: &SuccessResponse{ + Message: "Custom OpenAI mock response", + Usage: &Usage{ + PromptTokens: 15, + CompletionTokens: 25, + TotalTokens: 40, + }, + }, + }, + }, + }, + }, + } + plugin, err := NewMockerPlugin(config) + if err != nil { + t.Fatalf("Expected no error creating plugin, got: %v", err) + } + + account := BaseAccount{} + client, err := bifrost.Init(schemas.BifrostConfig{ + Account: &account, + Plugins: []schemas.Plugin{plugin}, + Logger: bifrost.NewDefaultLogger(schemas.LogLevelError), + }) + if err != nil { + t.Fatalf("Error initializing Bifrost: %v", err) + } + defer client.Cleanup() + + response, bifrostErr := client.ChatCompletionRequest(context.Background(), &schemas.BifrostRequest{ + Provider: schemas.OpenAI, + Model: "gpt-4", + Input: schemas.RequestInput{ + ChatCompletionInput: &[]schemas.BifrostMessage{ + { + Role: schemas.ModelChatMessageRoleUser, + Content: schemas.MessageContent{ + ContentStr: bifrost.Ptr("Hello, test message"), + }, + }, + }, + }, + }) + + if bifrostErr != nil { + t.Fatalf("Expected no error, got: %v", bifrostErr) + } + if response == nil { + t.Fatal("Expected response") + } + if len(response.Choices) == 0 { + t.Fatal("Expected at least one choice") + } + if response.Choices[0].Message.Content.ContentStr == nil { + t.Fatal("Expected content string") + } + if *response.Choices[0].Message.Content.ContentStr != "Custom OpenAI mock response" { + t.Errorf("Expected custom message, got: %s", *response.Choices[0].Message.Content.ContentStr) + } + if response.Usage.TotalTokens != 40 { + t.Errorf("Expected 40 total tokens, got %d", response.Usage.TotalTokens) + } +} + +// TestMockerPlugin_ErrorResponse tests error response generation +func TestMockerPlugin_ErrorResponse(t *testing.T) { + allowFallbacks := false + config := MockerConfig{ + Enabled: true, + Rules: []MockRule{ + { + Name: "rate-limit-error", + Enabled: true, + Priority: 100, + Probability: 1.0, + Conditions: Conditions{ + Providers: []string{"openai"}, + }, + Responses: []Response{ + { + Type: ResponseTypeError, + AllowFallbacks: &allowFallbacks, + Error: &ErrorResponse{ + Message: "Rate limit exceeded", + Type: bifrost.Ptr("rate_limit"), + Code: bifrost.Ptr("429"), + StatusCode: bifrost.Ptr(429), + }, + }, + }, + }, + }, + } + plugin, err := NewMockerPlugin(config) + if err != nil { + t.Fatalf("Expected no error creating plugin, got: %v", err) + } + + account := BaseAccount{} + client, err := bifrost.Init(schemas.BifrostConfig{ + Account: &account, + Plugins: []schemas.Plugin{plugin}, + Logger: bifrost.NewDefaultLogger(schemas.LogLevelError), + }) + if err != nil { + t.Fatalf("Error initializing Bifrost: %v", err) + } + defer client.Cleanup() + + _, bifrostErr := client.ChatCompletionRequest(context.Background(), &schemas.BifrostRequest{ + Provider: schemas.OpenAI, + Model: "gpt-4", + Input: schemas.RequestInput{ + ChatCompletionInput: &[]schemas.BifrostMessage{ + { + Role: schemas.ModelChatMessageRoleUser, + Content: schemas.MessageContent{ + ContentStr: bifrost.Ptr("Hello, test message"), + }, + }, + }, + }, + }) + + if bifrostErr == nil { + t.Fatal("Expected error response") + } + if bifrostErr.Error.Message != "Rate limit exceeded" { + t.Errorf("Expected 'Rate limit exceeded', got: %s", bifrostErr.Error.Message) + } + if bifrostErr.StatusCode == nil || *bifrostErr.StatusCode != 429 { + t.Errorf("Expected status code 429, got: %v", bifrostErr.StatusCode) + } +} + +// TestMockerPlugin_MessageTemplate tests template variable substitution +func TestMockerPlugin_MessageTemplate(t *testing.T) { + config := MockerConfig{ + Enabled: true, + Rules: []MockRule{ + { + Name: "template-test", + Enabled: true, + Priority: 100, + Probability: 1.0, + Conditions: Conditions{}, // Match all + Responses: []Response{ + { + Type: ResponseTypeSuccess, + Content: &SuccessResponse{ + MessageTemplate: bifrost.Ptr("Hello from {{provider}} using model {{model}}"), + }, + }, + }, + }, + }, + } + plugin, err := NewMockerPlugin(config) + if err != nil { + t.Fatalf("Expected no error creating plugin, got: %v", err) + } + + account := BaseAccount{} + client, err := bifrost.Init(schemas.BifrostConfig{ + Account: &account, + Plugins: []schemas.Plugin{plugin}, + Logger: bifrost.NewDefaultLogger(schemas.LogLevelError), + }) + if err != nil { + t.Fatalf("Error initializing Bifrost: %v", err) + } + defer client.Cleanup() + + response, bifrostErr := client.ChatCompletionRequest(context.Background(), &schemas.BifrostRequest{ + Provider: schemas.Anthropic, + Model: "claude-3", + Input: schemas.RequestInput{ + ChatCompletionInput: &[]schemas.BifrostMessage{ + { + Role: schemas.ModelChatMessageRoleUser, + Content: schemas.MessageContent{ + ContentStr: bifrost.Ptr("Hello, test message"), + }, + }, + }, + }, + }) + + if bifrostErr != nil { + t.Fatalf("Expected no error, got: %v", bifrostErr) + } + if response == nil { + t.Fatal("Expected response") + } + if len(response.Choices) == 0 { + t.Fatal("Expected at least one choice") + } + if response.Choices[0].Message.Content.ContentStr == nil { + t.Fatal("Expected content string") + } + expectedMessage := "Hello from anthropic using model claude-3" + if *response.Choices[0].Message.Content.ContentStr != expectedMessage { + t.Errorf("Expected '%s', got: %s", expectedMessage, *response.Choices[0].Message.Content.ContentStr) + } +} + +// TestMockerPlugin_Statistics tests plugin statistics tracking +func TestMockerPlugin_Statistics(t *testing.T) { + config := MockerConfig{ + Enabled: true, + Rules: []MockRule{ + { + Name: "stats-test", + Enabled: true, + Priority: 100, + Probability: 1.0, + Conditions: Conditions{}, // Match all + Responses: []Response{ + { + Type: ResponseTypeSuccess, + Content: &SuccessResponse{ + Message: "Stats test response", + }, + }, + }, + }, + }, + } + plugin, err := NewMockerPlugin(config) + if err != nil { + t.Fatalf("Expected no error creating plugin, got: %v", err) + } + + account := BaseAccount{} + client, err := bifrost.Init(schemas.BifrostConfig{ + Account: &account, + Plugins: []schemas.Plugin{plugin}, + Logger: bifrost.NewDefaultLogger(schemas.LogLevelError), + }) + if err != nil { + t.Fatalf("Error initializing Bifrost: %v", err) + } + defer client.Cleanup() + + // Make multiple requests + for i := 0; i < 3; i++ { + _, _ = client.ChatCompletionRequest(context.Background(), &schemas.BifrostRequest{ + Provider: schemas.OpenAI, + Model: "gpt-4", + Input: schemas.RequestInput{ + ChatCompletionInput: &[]schemas.BifrostMessage{ + { + Role: schemas.ModelChatMessageRoleUser, + Content: schemas.MessageContent{ + ContentStr: bifrost.Ptr("Hello, test message"), + }, + }, + }, + }, + }) + } + + // Check statistics + stats := plugin.GetStats() + if stats.TotalRequests != 3 { + t.Errorf("Expected 3 total requests, got %d", stats.TotalRequests) + } + if stats.MockedRequests != 3 { + t.Errorf("Expected 3 mocked requests, got %d", stats.MockedRequests) + } + if stats.ResponsesGenerated != 3 { + t.Errorf("Expected 3 responses generated, got %d", stats.ResponsesGenerated) + } + if stats.RuleHits["stats-test"] != 3 { + t.Errorf("Expected 3 hits for 'stats-test' rule, got %d", stats.RuleHits["stats-test"]) + } +} + +// TestMockerPlugin_ValidationErrors tests configuration validation +func TestMockerPlugin_ValidationErrors(t *testing.T) { + tests := []struct { + name string + config MockerConfig + expectError bool + }{ + { + name: "invalid default behavior", + config: MockerConfig{ + Enabled: true, + DefaultBehavior: "invalid", + }, + expectError: true, + }, + { + name: "missing rule name", + config: MockerConfig{ + Enabled: true, + Rules: []MockRule{ + { + Name: "", // Missing name + Enabled: true, + Responses: []Response{ + { + Type: ResponseTypeSuccess, + Content: &SuccessResponse{ + Message: "test", + }, + }, + }, + }, + }, + }, + expectError: true, + }, + { + name: "invalid probability", + config: MockerConfig{ + Enabled: true, + Rules: []MockRule{ + { + Name: "test", + Enabled: true, + Probability: 1.5, // Invalid probability > 1 + Responses: []Response{ + { + Type: ResponseTypeSuccess, + Content: &SuccessResponse{ + Message: "test", + }, + }, + }, + }, + }, + }, + expectError: true, + }, + { + name: "valid configuration", + config: MockerConfig{ + Enabled: true, + DefaultBehavior: DefaultBehaviorPassthrough, + Rules: []MockRule{ + { + Name: "valid-rule", + Enabled: true, + Probability: 0.5, + Responses: []Response{ + { + Type: ResponseTypeSuccess, + Content: &SuccessResponse{ + Message: "Valid response", + }, + }, + }, + }, + }, + }, + expectError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := NewMockerPlugin(tt.config) + if tt.expectError && err == nil { + t.Error("Expected error but got none") + } + if !tt.expectError && err != nil { + t.Errorf("Expected no error but got: %v", err) + } + }) + } +} diff --git a/tests/transports-integrations/README.md b/tests/transports-integrations/README.md index 9d36cb82f1..ddb6760f8f 100644 --- a/tests/transports-integrations/README.md +++ b/tests/transports-integrations/README.md @@ -6,7 +6,7 @@ Production-ready end-to-end test suite for testing AI integrations through Bifro The Bifrost integration tests use a centralized configuration system that routes all AI integration requests through Bifrost as a gateway/proxy: -``` +```text ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ │ Test Client │───▶│ Bifrost Gateway │───▶│ AI Integration │ │ │ │ localhost:8080 │ │ (OpenAI, etc.) │ @@ -50,7 +50,7 @@ Our test suite covers 11 comprehensive scenarios for each integration: ## 📁 Directory Structure -``` +```text transports-integrations/ ├── config.yml # Central configuration file ├── requirements.txt # Python dependencies @@ -437,7 +437,7 @@ pytest tests/integrations/test_google.py::TestGoogleIntegration::test_03_single_ #### Quick Reference: Test Categories -``` +```text Test 01: Simple Chat - Basic single-message conversations Test 02: Multi-turn Conversation - Conversation history and context Test 03: Single Tool Call - Basic function calling