feat: removes enable_litellm_fallback option for more granular settings

.github/workflows/configs/withpostgresmcpclientsinconfig/config.json

@@ -7,7 +7,6 @@
     ],
     "disable_content_logging": false,
     "drop_excess_requests": false,
-    "enable_litellm_fallbacks": false,
     "enable_logging": true,
     "enforce_auth_on_inference": true,
     "initial_pool_size": 300,

.github/workflows/scripts/run-migration-tests.sh

@@ -542,8 +542,8 @@ VALUES ('migration-test-lock', 'holder-migration-test-001', $future, $now)
 ON CONFLICT DO NOTHING;
 
 -- config_client (global client configuration)
-INSERT INTO config_client (id, drop_excess_requests, prometheus_labels_json, allowed_origins_json, allowed_headers_json, header_filter_config_json, initial_pool_size, enable_logging, disable_content_logging, disable_db_pings_in_health, log_retention_days, enforce_governance_header, allow_direct_keys, max_request_body_size_mb, mcp_agent_depth, mcp_tool_execution_timeout, mcp_code_mode_binding_level, mcp_tool_sync_interval, enable_litellm_fallbacks, config_hash, created_at, updated_at)
-VALUES (1, false, '["provider", "model"]', '["*"]', '["Authorization"]', '{}', 300, true, false, false, 365, true, false, true, 100, 10, 30, 'server', 10, false, 'client-config-hash-001', $now, $now)
+INSERT INTO config_client (id, drop_excess_requests, prometheus_labels_json, allowed_origins_json, allowed_headers_json, header_filter_config_json, initial_pool_size, enable_logging, disable_content_logging, disable_db_pings_in_health, log_retention_days, enforce_governance_header, allow_direct_keys, max_request_body_size_mb, mcp_agent_depth, mcp_tool_execution_timeout, mcp_code_mode_binding_level, mcp_tool_sync_interval, compat_convert_text_to_chat, compat_convert_chat_to_responses, compat_should_drop_params, compat_should_convert_params, config_hash, created_at, updated_at)
+VALUES (1, false, '["provider", "model"]', '["*"]', '["Authorization"]', '{}', 300, true, false, false, 365, true, false, true, 100, 10, 30, 'server', 10, false, false, false, 'client-config-hash-001', $now, $now)
 ON CONFLICT DO NOTHING;
 
 -- governance_config (key-value config table)
@@ -3509,4 +3509,4 @@ main() {
   exit $exit_code
 }
 
-main "$@"
+main "$@"

.github/workflows/scripts/test-docker-image.sh

@@ -212,8 +212,7 @@ cat > "$CONFIG_FILE" << 'CONFIGEOF'
     "enable_logging": true,
     "enforce_governance_header": false,
     "allow_direct_keys": false,
-    "max_request_body_size_mb": 100,
-    "enable_litellm_fallbacks": false
+    "max_request_body_size_mb": 100
   },
   "encryption_key": ""
 }

.github/workflows/scripts/validate-helm-config-fields.sh

@@ -164,7 +164,11 @@ bifrost:
     enforceGovernanceHeader: true
     allowDirectKeys: true
     maxRequestBodySizeMb: 50
-    enableLitellmFallbacks: true
+    compat:
+      convertTextToChat: true
+      convertChatToResponses: true
+      shouldDropParams: true
+      shouldConvertParams: true
     prometheusLabels:
       - "team"
       - "env"
@@ -200,7 +204,9 @@ assert_field_value 'client.log_retention_days' '.client.log_retention_days' '30'
 assert_field_value 'client.enforce_governance_header' '.client.enforce_governance_header' 'true'
 assert_field_value 'client.allow_direct_keys' '.client.allow_direct_keys' 'true'
 assert_field_value 'client.max_request_body_size_mb' '.client.max_request_body_size_mb' '50'
-assert_field_value 'client.enable_litellm_fallbacks' '.client.enable_litellm_fallbacks' 'true'
+assert_field_value 'client.compat.convert_text_to_chat' '.client.compat.convert_text_to_chat' 'true'
+assert_field_value 'client.compat.convert_chat_to_responses' '.client.compat.convert_chat_to_responses' 'true'
+assert_field_value 'client.compat.should_drop_params' '.client.compat.should_drop_params' 'true'
 assert_field 'client.prometheus_labels' '.client.prometheus_labels'
 assert_field 'client.header_filter_config.allowlist' '.client.header_filter_config.allowlist'
 assert_field 'client.header_filter_config.denylist' '.client.header_filter_config.denylist'
@@ -1194,4 +1200,4 @@ if [ "$TESTS_FAILED" -gt 0 ]; then
 else
   echo -e "${GREEN}✅ All config.json field validations passed!${NC}"
   exit 0
-fi
+fi

docs/features/litellm-compat.mdx

@@ -9,8 +9,10 @@ icon: "train"
 The LiteLLM compatibility plugin provides two transformations:
 
 1. **Text-to-Chat Conversion** - Automatically converts text completion requests to chat completion format for models that only support chat APIs
+2. **Chat-to-Responses Conversion** - Automatically converts chat completion requests to responses format for models that only support responses APIs
+3. **Drop Unsupported Params** - Automatically drops unsupported parameters if the model doesn't support them
 
-When either transformation is applied, responses include `extra_fields.litellm_compat: true`.
+When either transformation is applied, responses include `extra_fields.converted_request_type: <transformed_request_type>`. If request parameters are dropped, the keys are added in `extra_fields.dropped_compat_plugin_params`.
 
 ---
 
@@ -55,6 +57,36 @@ F --> G
 - `object: "chat.completion"` → `object: "text_completion"`
 - Usage statistics and metadata are preserved
 
+## 2. Chat-to-Responses Conversion
+
+Some AI models (like OpenAI o1-pro) only support the responses API and don't support native chat completion endpoints. LiteLLM compatibility mode automatically handles this by:
+
+1. Checking if the model supports chat completion natively (using the model catalog)
+2. If not supported, converting your chat message to responses API format
+3. Calling the responses endpoint internally
+4. Transforming the response back to chat completion format
+
+<Note>
+**Smart Conversion**: The conversion only happens when the model doesn't support chat completions natively. If a model has native chat completion support (like OpenAI's gpt-4 models), Bifrost uses the chat completion endpoint directly without any conversion.
+</Note>
+
+This allows you to use a unified chat completion interface across all providers, even those that only support responses API.
+
+## How It Works
+
+When LiteLLM compatibility is enabled and you make a chat completion request, Bifrost first checks if the model supports chat completion:
+
+```mermaid
+flowchart LR
+A[Chat Completion Request] --> B{Model Supports Chat Completion?}
+B -->|Yes| C[Call Chat Completion API]
+B -->|No| D[Convert to Responses Message]
+D --> E[Call Responses API]
+E --> F[Transform Response]
+C --> G[Chat Completion Response]
+F --> G
+```
+
 ## Enabling LiteLLM Compatibility
 
 <Tabs group="litellm-compat">
@@ -63,7 +95,10 @@ F --> G
 
 1. Open the Bifrost dashboard
 2. Navigate to **Settings** → **Client Configuration**
-3. Enable **LiteLLM Fallbacks**
+3. Expand **LiteLLM Compat** and enable the features you need:
+   - **Convert Text to Chat** — converts text completion requests to chat for models that only support chat
+   - **Convert Chat to Responses** — converts chat completion requests to responses for models that only support responses
+   - **Drop Unsupported Params** — drops unsupported parameters based on model catalog allowlist
 4. Save your configuration
 
 </Tab>
@@ -73,7 +108,11 @@ F --> G
 ```json
 {
   "client_config": {
-    "enable_litellm_fallbacks": true
+    "compat": {
+      "convert_text_to_chat": true,
+      "convert_chat_to_responses": true,
+      "should_drop_params": true
+    }
   }
 }
 ```
@@ -84,9 +123,9 @@ F --> G
 
 ## Supported Providers
 
-LiteLLM compatibility mode works with any provider that supports chat completions but lacks native text completion support:
+Text completion to chat completion conversion works with any provider that supports chat completions but lacks native text completion support:
 
-| Provider | Native Text Completion | LiteLLM Fallback |
+| Provider | Native Text Completion | With Fallback |
 |----------|----------------------|------------------|
 | OpenAI (GPT-4, GPT-3.5-turbo) | No | Yes |
 | Anthropic (Claude) | No | Yes |
@@ -95,6 +134,12 @@ LiteLLM compatibility mode works with any provider that supports chat completion
 | Mistral | No | Yes |
 | Bedrock | Varies by model | Yes |
 
+Chat completion to responses conversion works with any provider that supports responses but lacks native chat completion support:
+
+| Provider | Native Chat Completion | With Fallback |
+|----------|----------------------|------------------|
+| OpenAI (o1-pro) | No | Yes |
+
 ## Behavior Details
 
 **Model Capability Detection:**
@@ -117,22 +162,31 @@ LiteLLM compatibility mode works with any provider that supports chat completion
 | Response | `choices[0].message.content` | `choices[0].text` |
 | Response | `object: "chat.completion"` | `object: "text_completion"` |
 
+### Transformation 2: Chat-to-Responses Conversion
+
+**Applies to:** Chat completion requests on responses-only models
+
+| Phase | Original | Transformed |
+|-------|----------|-------------|
+| Request | Chat message with `role: "user"` | Responses input with `role: "user"` |
+| Request | `chat_completion` request type | `responses` request type |
 
 ### Metadata Set on Transformed Responses
 
 When either transformation is applied:
 
-- `extra_fields.litellm_compat`: Set to `true`
-- `extra_fields.provider`: The provider that handled the request
 - `extra_fields.request_type`: Reflects the original request type
 - `extra_fields.original_model_requested`: The originally requested model
 - `extra_fields.resolved_model_used`: The actual provider API identifier used (equals original_model_requested when no alias mapping exists)
 
 ### Error Handling
 
 When errors occur on transformed requests:
-- `extra_fields.litellm_compat` is set to `true`
 - Original request type and model are preserved in error metadata
+- `extra_fields.converted_request_type`: Set to type of request that was converted to (i.e., `chat_completion` or `responses`)
+- `extra_fields.provider`: The provider that handled the request
+- `extra_fields.original_model_requested`: The originally requested model
+- `extra_fields.dropped_compat_plugin_params`: If any unsupported parameters were dropped, the keys are added here
 
 ## What's Preserved
 
@@ -145,7 +199,7 @@ When errors occur on transformed requests:
 
 **Good Use Cases:**
 - Migrating from LiteLLM to Bifrost without code changes
-- Maintaining backward compatibility with text completion interfaces
+- Maintaining backward compatibility with text completion interfaces or chat completion interfaces
 - Using a unified API across providers with different capabilities
 
 **Consider Alternatives When:**
@@ -157,4 +211,4 @@ When errors occur on transformed requests:
 
 - [Fallbacks](/features/fallbacks) - Automatic provider failover
 - [Drop-in Replacement](/features/drop-in-replacement) - Use existing SDKs with Bifrost
-- [LiteLLM Integration](/integrations/litellm-sdk) - Using LiteLLM SDK with Bifrost
+- [LiteLLM Integration](/integrations/litellm-sdk) - Using LiteLLM SDK with Bifrost

docs/openapi/openapi.json

@@ -133221,9 +133221,15 @@
                           "type": "integer",
                           "description": "Maximum request body size in MB"
                         },
-                        "enable_litellm_fallbacks": {
-                          "type": "boolean",
-                          "description": "Whether LiteLLM fallbacks are enabled"
+                        "compat": {
+                          "type": "object",
+                          "description": "Compat plugin configuration",
+                          "properties": {
+                            "convert_text_to_chat": { "type": "boolean", "description": "Convert text completion requests to chat" },
+                            "convert_chat_to_responses": { "type": "boolean", "description": "Convert chat completion requests to responses" },
+                            "should_drop_params": { "type": "boolean", "description": "Drop unsupported parameters based on model catalog" },
+                            "should_convert_params": { "type": "boolean", "description": "Converts model parameter values that are not supported by the model.", "default": true }
+                          }
                         },
                         "log_retention_days": {
                           "type": "integer",
@@ -133537,9 +133543,15 @@
                         "type": "integer",
                         "description": "Maximum request body size in MB"
                       },
-                      "enable_litellm_fallbacks": {
-                        "type": "boolean",
-                        "description": "Whether LiteLLM fallbacks are enabled"
+                      "compat": {
+                        "type": "object",
+                        "description": "Compat plugin configuration",
+                        "properties": {
+                          "convert_text_to_chat": { "type": "boolean", "description": "Convert text completion requests to chat" },
+                          "convert_chat_to_responses": { "type": "boolean", "description": "Convert chat completion requests to responses" },
+                          "should_drop_params": { "type": "boolean", "description": "Drop unsupported parameters based on model catalog" },
+                          "should_convert_params": { "type": "boolean", "description": "Converts model parameter values that are not supported by the model.", "default": true }
+                        }
                       },
                       "log_retention_days": {
                         "type": "integer",
@@ -205784,9 +205796,15 @@
                 "type": "integer",
                 "description": "Maximum request body size in MB"
               },
-              "enable_litellm_fallbacks": {
-                "type": "boolean",
-                "description": "Whether LiteLLM fallbacks are enabled"
+              "compat": {
+                "type": "object",
+                "description": "Compat plugin configuration",
+                "properties": {
+                  "convert_text_to_chat": { "type": "boolean", "description": "Convert text completion requests to chat" },
+                  "convert_chat_to_responses": { "type": "boolean", "description": "Convert chat completion requests to responses" },
+                  "should_drop_params": { "type": "boolean", "description": "Drop unsupported parameters based on model catalog" },
+                  "should_convert_params": { "type": "boolean", "description": "Converts model parameter values that are not supported by the model.", "default": true }
+                }
               },
               "log_retention_days": {
                 "type": "integer",
@@ -205999,9 +206017,15 @@
                 "type": "integer",
                 "description": "Maximum request body size in MB"
               },
-              "enable_litellm_fallbacks": {
-                "type": "boolean",
-                "description": "Whether LiteLLM fallbacks are enabled"
+              "compat": {
+                "type": "object",
+                "description": "Compat plugin configuration",
+                "properties": {
+                  "convert_text_to_chat": { "type": "boolean", "description": "Convert text completion requests to chat" },
+                  "convert_chat_to_responses": { "type": "boolean", "description": "Convert chat completion requests to responses" },
+                  "should_drop_params": { "type": "boolean", "description": "Drop unsupported parameters based on model catalog" },
+                  "should_convert_params": { "type": "boolean", "description": "Converts model parameter values that are not supported by the model.", "default": true }
+                }
               },
               "log_retention_days": {
                 "type": "integer",
@@ -224498,4 +224522,4 @@
       }
     }
   }
-}
+}

docs/openapi/schemas/management/config.yaml

@@ -44,9 +44,19 @@ ClientConfig:
     max_request_body_size_mb:
       type: integer
       description: Maximum request body size in MB
-    enable_litellm_fallbacks:
-      type: boolean
-      description: Whether LiteLLM fallbacks are enabled
+    compat:
+      type: object
+      description: Compat plugin configuration
+      properties:
+        convert_text_to_chat:
+          type: boolean
+          description: Convert text completion requests to chat
+        convert_chat_to_responses:
+          type: boolean
+          description: Convert chat completion requests to responses
+        should_drop_params:
+          type: boolean
+          description: Drop unsupported parameters based on model catalog
     log_retention_days:
       type: integer
       description: Number of days to retain logs

docs/providers/supported-providers/overview.mdx

@@ -48,7 +48,7 @@ The following table summarizes which operations are supported by each provider v
 
 <Note>
 Some operations are not supported by the downstream provider, and their internal implementation in Bifrost is optional. 🟡
-Like Text completions are not supported by Groq, but Bifrost can emulate them internally using the Chat Completions API. This feature is disabled by default, but it can be enabled by setting the `enable_litellm_fallbacks` flag to `true` in the client configuration.
+Like Text completions are not supported by Groq, but Bifrost can emulate them internally using the Chat Completions API. This feature is disabled by default, but it can be enabled by setting `compat.convert_text_to_chat` to `true` in the client configuration.
 We do not promote using such fallbacks, since text completions and chat completions are fundamentally different. However, this option is available to help users migrating from LiteLLM (which does support these fallbacks).
 </Note>
 

examples/configs/withpostgresmcpclientsinconfig/config.json

@@ -7,7 +7,6 @@
     ],
     "disable_content_logging": false,
     "drop_excess_requests": false,
-    "enable_litellm_fallbacks": false,
     "enable_logging": true,
     "enforce_auth_on_inference": true,
     "initial_pool_size": 300,

examples/configs/withprompushgateway/config.json

@@ -183,8 +183,7 @@
         "enable_logging": true,
         "enforce_auth_on_inference": false,
         "allow_direct_keys": false,
-        "max_request_body_size_mb": 100,
-        "enable_litellm_fallbacks": false
+        "max_request_body_size_mb": 100
     },
     "config_store": {
       "enabled": true,

examples/configs/withvirtualkeys/config.json

@@ -7,7 +7,6 @@
         ],
         "disable_content_logging": false,
         "drop_excess_requests": false,
-        "enable_litellm_fallbacks": false,
         "enable_logging": true,
         "enforce_auth_on_inference": true,
         "initial_pool_size": 300,

examples/dockers/data/config.json

@@ -27,12 +27,14 @@
       "*"
     ],
     "max_request_body_size_mb": 100,
-    "enable_litellm_fallbacks": false
+    "compat": {
+      "should_convert_params": true
+    }
   },
   "framework": {
     "pricing": {
       "pricing_url": "https://getbifrost.ai/datasheet",
       "pricing_sync_interval": 86400
     }
   }
-}
+}