Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions core/bifrost.go
Original file line number Diff line number Diff line change
Expand Up @@ -6099,10 +6099,13 @@ func (bifrost *Bifrost) getKeysForBatchAndFileOps(ctx *schemas.BifrostContext, p
// Model filtering logic:
// - If model is nil or empty → include all keys (no model filter)
// - If model is specified:
// - If key.Models is empty → include key (supports all models)
// - If key.Models is ["*"] → include key (supports all models)
// - If key.Models is empty → exclude key (deny-by-default)
// - If key.Models is non-empty → only include if model is in list
if model != nil && *model != "" && len(k.Models) > 0 {
if !slices.Contains(k.Models, *model) {
if model != nil && *model != "" {
if slices.Contains(k.Models, "*") {
// wildcard: allow all models
} else if len(k.Models) == 0 || !slices.Contains(k.Models, *model) {
continue
}
}
Expand Down Expand Up @@ -6195,7 +6198,8 @@ func (bifrost *Bifrost) selectKeyFromProviderForModel(ctx *schemas.BifrostContex
continue
}
hasValue := strings.TrimSpace(key.Value.GetValue()) != "" || CanProviderKeyValueBeEmpty(baseProviderType)
modelSupported := (len(key.Models) == 0 && hasValue) || (slices.Contains(key.Models, model) && hasValue)
// ["*"] = allow all models; [] = deny all; specific list = allow only listed
modelSupported := hasValue && (slices.Contains(key.Models, "*") || slices.Contains(key.Models, model))
// Additional deployment checks for Azure, Bedrock and Vertex
deploymentSupported := true
if baseProviderType == schemas.Azure && key.AzureKeyConfig != nil {
Expand Down
1 change: 1 addition & 0 deletions core/changelog.md
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
- feat: add DisableAutoToolInject to MCPToolManagerConfig to suppress automatic MCP tool injection per request
- feat: add BifrostContextKeyMCPAddedTools to context to track MCP tools added to the request
- refactor: standardize empty array conventions in bifrost. Empty array means deny all, ["*"] means allow all for models/tools/keys.
1 change: 0 additions & 1 deletion core/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ require (
github.com/bytedance/sonic v1.15.0
github.com/fasthttp/websocket v1.5.12
github.com/google/uuid v1.6.0
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674
github.com/hajimehoshi/go-mp3 v0.3.4
github.com/klauspost/compress v1.18.2
github.com/mark3labs/mcp-go v0.43.2
Expand Down
2 changes: 1 addition & 1 deletion core/providers/mistral/mistral.go
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ func (provider *MistralProvider) listModelsByKey(ctx *schemas.BifrostContext, ke
}

// Create final response
response := mistralResponse.ToBifrostListModelsResponse(key.Models)
response := mistralResponse.ToBifrostListModelsResponse(key.Models, request.Unfiltered)

response.ExtraFields.Latency = latency.Milliseconds()

Expand Down
6 changes: 3 additions & 3 deletions core/providers/mistral/models.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import (
"github.com/maximhq/bifrost/core/schemas"
)

func (response *MistralListModelsResponse) ToBifrostListModelsResponse(allowedModels []string) *schemas.BifrostListModelsResponse {
func (response *MistralListModelsResponse) ToBifrostListModelsResponse(allowedModels []string, unfiltered bool) *schemas.BifrostListModelsResponse {
if response == nil {
return nil
}
Expand All @@ -17,7 +17,7 @@ func (response *MistralListModelsResponse) ToBifrostListModelsResponse(allowedMo

includedModels := make(map[string]bool)
for _, model := range response.Data {
if len(allowedModels) > 0 && !slices.Contains(allowedModels, model.ID) {
if !unfiltered && len(allowedModels) > 0 && !slices.Contains(allowedModels, model.ID) {
continue
Comment thread
coderabbitai[bot] marked this conversation as resolved.
}
bifrostResponse.Data = append(bifrostResponse.Data, schemas.Model{
Expand All @@ -32,7 +32,7 @@ func (response *MistralListModelsResponse) ToBifrostListModelsResponse(allowedMo
}

// Backfill allowed models that were not in the response
if len(allowedModels) > 0 {
if !unfiltered && len(allowedModels) > 0 {
for _, allowedModel := range allowedModels {
if !includedModels[allowedModel] {
bifrostResponse.Data = append(bifrostResponse.Data, schemas.Model{
Expand Down
23 changes: 11 additions & 12 deletions core/schemas/transcriptions.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,17 +31,17 @@ type TranscriptionInput struct {
}

type TranscriptionParameters struct {
Language *string `json:"language,omitempty"`
Prompt *string `json:"prompt,omitempty"`
ResponseFormat *string `json:"response_format,omitempty"` // Default is "json"
Temperature *float64 `json:"temperature,omitempty"` // Sampling temperature (0.0-1.0)
TimestampGranularities []string `json:"timestamp_granularities,omitempty"` // "word" and/or "segment"; requires response_format=verbose_json
Include []string `json:"include,omitempty"` // Additional response info (e.g., logprobs)
Format *string `json:"file_format,omitempty"` // Type of file, not required in openai, but required in gemini
MaxLength *int `json:"max_length,omitempty"` // Maximum length of the transcription used by HuggingFace
MinLength *int `json:"min_length,omitempty"` // Minimum length of the transcription used by HuggingFace
MaxNewTokens *int `json:"max_new_tokens,omitempty"` // Maximum new tokens to generate used by HuggingFace
MinNewTokens *int `json:"min_new_tokens,omitempty"` // Minimum new tokens to generate used by HuggingFace
Language *string `json:"language,omitempty"`
Prompt *string `json:"prompt,omitempty"`
ResponseFormat *string `json:"response_format,omitempty"` // Default is "json"
Temperature *float64 `json:"temperature,omitempty"` // Sampling temperature (0.0-1.0)
TimestampGranularities []string `json:"timestamp_granularities,omitempty"` // "word" and/or "segment"; requires response_format=verbose_json
Include []string `json:"include,omitempty"` // Additional response info (e.g., logprobs)
Format *string `json:"file_format,omitempty"` // Type of file, not required in openai, but required in gemini
MaxLength *int `json:"max_length,omitempty"` // Maximum length of the transcription used by HuggingFace
MinLength *int `json:"min_length,omitempty"` // Minimum length of the transcription used by HuggingFace
MaxNewTokens *int `json:"max_new_tokens,omitempty"` // Maximum new tokens to generate used by HuggingFace
MinNewTokens *int `json:"min_new_tokens,omitempty"` // Minimum new tokens to generate used by HuggingFace

// Elevenlabs-specific fields
AdditionalFormats []TranscriptionAdditionalFormat `json:"additional_formats,omitempty"`
Expand Down Expand Up @@ -132,4 +132,3 @@ type BifrostTranscriptionStreamResponse struct {
Usage *TranscriptionUsage `json:"usage,omitempty"`
ExtraFields BifrostResponseExtraFields `json:"extra_fields"`
}

22 changes: 12 additions & 10 deletions docs/features/governance/routing.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,9 @@ Virtual Keys can be restricted to use only specific provider/models. When provid

**Model Validation:**
When you configure provider restrictions on a Virtual Key, Bifrost validates that the requested model is allowed for the selected provider:
- **Explicit `allowed_models`**: If you specify models in the provider config, only those models are permitted
- **Empty `allowed_models`**: Bifrost uses the **Model Catalog** (populated from pricing data + list models API) to determine which models the provider supports
- **`allowed_models: ["*"]`**: Allow all models supported by the provider (uses the Model Catalog for validation).
- **Empty `allowed_models`**: **Deny all** models (deny-by-default).
- **Explicit model list**: Only those specific models are permitted.
- **Model Catalog Sync**: On startup and provider updates, Bifrost calls each provider's list models API. If this fails, you'll see a warning: `{"level":"warn","message":"failed to list models for provider <name>: failed to execute HTTP request to provider API"}`

<Note>
Expand Down Expand Up @@ -80,18 +81,18 @@ curl -X POST http://localhost:8080/v1/chat/completions \
Weights are automatically normalized to a sum 1.0 based on the weights of all providers available on the VK for the given model.
</Info>

**Example with Empty `allowed_models` (using Model Catalog):**
**Example with Wildcard `allowed_models` (allow all via Model Catalog):**
```json
{
"provider_configs": [
{
"provider": "openai",
"allowed_models": [], // Uses Model Catalog
"allowed_models": ["*"], // Allow all — uses Model Catalog for validation
"weight": 0.5
},
{
"provider": "anthropic",
"allowed_models": [], // Uses Model Catalog
"allowed_models": ["*"], // Allow all — uses Model Catalog for validation
"weight": 0.5
}
]
Expand Down Expand Up @@ -149,7 +150,8 @@ curl -X POST http://localhost:8080/v1/chat/completions \
3. In **Provider Configurations** section, add the provider you want to restrict the VK to
4. **Allowed Models**:
- **Specify models**: Enter specific models (e.g., `["gpt-4o", "gpt-4o-mini"]`) to explicitly whitelist only those models
- **Leave blank**: Uses the Model Catalog to determine which models this provider supports (populated from pricing data and the provider's list models API)
- **`["*"]`**: Allow all models (uses the Model Catalog for validation).
- **Leave blank**: Deny all models (deny-by-default).
5. Add the weight you want to give to this provider
6. Click on the **Save** button
</Tab>
Expand Down Expand Up @@ -231,7 +233,7 @@ Virtual Key Restrictions:
│ └── Restricted Keys: [key-dev-002, key-test-003] ← Dev + test keys
└── vk-unrestricted
├── Allowed Models: [all models]
└── Restricted Keys: [] ← Can use ANY available key
└── Restricted Keys: ["*"] ← Can use ANY available key
```

**Request Behavior:**
Expand Down Expand Up @@ -290,7 +292,7 @@ curl -X PUT http://localhost:8080/api/governance/virtual-keys/{vk_id} \
"provider_configs": [
{
"provider": "openai",
"allowed_keys": [
"key_ids": [
"key-prod-001"
]
Comment thread
coderabbitai[bot] marked this conversation as resolved.
}
Expand Down Expand Up @@ -325,10 +327,10 @@ If you see warnings like this in your Bifrost logs during startup or provider up
**What this means:**
- Bifrost attempted to call the provider's list models API to populate the Model Catalog
- The request failed (network issue, provider unavailable, incorrect credentials, etc.)
- If your Virtual Key has `allowed_models: []` (empty) for this provider, model validation will fall back to the pricing data only
- If your Virtual Key has `allowed_models: []` (empty) for this provider, **all models will be denied**. Use `["*"]` to allow all models.

**How to fix:**
1. Check that the provider is correctly configured and accessible
2. Verify network connectivity to the provider's API
3. Ensure API credentials are valid
4. Consider using explicit `allowed_models` instead of relying on the Model Catalog for critical providers
4. Use `allowed_models: ["*"]` to allow all models, or specify an explicit list for critical providers
12 changes: 8 additions & 4 deletions examples/configs/partial/config.json
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@
{
"name": "openai-key-1",
"value": "sk-123",
"weight": 1
"weight": 1,
"models": ["*"]
}
]
},
Expand All @@ -29,7 +30,8 @@
{
"name": "anthropic-key-1",
"value": "sk-456",
"weight": 1
"weight": 1,
"models": ["*"]
}
]
},
Expand All @@ -38,12 +40,14 @@
{
"name": "bedrock-key-1",
"value": "ak-123",
"weight": 1
"weight": 1,
"models": ["*"]
},
{
"name": "bedrock-key-2",
"value": "ak-456",
"weight": 1
"weight": 1,
"models": ["*"]
}
]
}
Expand Down
16 changes: 1 addition & 15 deletions examples/configs/withconfigstore/config.json
Original file line number Diff line number Diff line change
Expand Up @@ -22,21 +22,7 @@
"provider_configs": [
{
"provider": "azure",
"keys":[{
"key_id":"8c52039e-38c6-48b2-8016-0bd884b7befb",
"value":"abc",
"name":"azure-key-1",
"weight": 0.5,
"azure_key_config":{
"endpoint":"https://api.azure.com",
"api_version":"2024-09-01",
"deployments":{
"gpt-4.1-2025-04-14":"gpt-4.1-2025-04-14",
"gpt-4.1-mini-2025-04-14":"gpt-4.1-mini-2025-04-14",
"gpt-4.1-nano-2025-04-14":"gpt-4.1-nano-2025-04-14"
}
}
}],
"key_ids": ["*"],
"allowed_models": [
"gpt-4.1-2025-04-14",
"gpt-4.1-mini-2025-04-14",
Expand Down
3 changes: 2 additions & 1 deletion examples/configs/withlogstore/config.json
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@
{
"name": "openai-key-1",
"value": "sk-proj-abc",
"weight": 1
"weight": 1,
"models": ["*"]
}
]
}
Expand Down
11 changes: 8 additions & 3 deletions examples/configs/withpostgresmcpclientsinconfig/config.json
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,9 @@
"provider_configs": [
{
"provider": "openai",
"weight": 1.0
"weight": 1.0,
"allowed_models": ["*"],
"key_ids": ["*"]
}
]
},
Expand All @@ -109,7 +111,9 @@
"provider_configs": [
{
"provider": "openai",
"weight": 1.0
"weight": 1.0,
"allowed_models": ["*"],
"key_ids": ["*"]
}
]
}
Expand All @@ -130,7 +134,8 @@
{
"name": "openai-primary",
"value": "env.OPENAI_API_KEY",
"weight": 1
"weight": 1,
"models": ["*"]
}
]
}
Expand Down
Loading
Loading