Litellm oss staging 04 21 2026

.github/workflows/test-unit-proxy-db.yml

@@ -215,6 +215,7 @@ jobs:
               tests/proxy_unit_tests/test_google_gemini_proxy_request.py
               tests/proxy_unit_tests/test_get_favicon.py
               tests/proxy_unit_tests/test_get_image.py
+              tests/proxy_unit_tests/test_reducto_ocr_route.py
               tests/proxy_unit_tests/test_ui_path_detection.py
               tests/proxy_unit_tests/test_prompt_test_endpoint.py
               tests/proxy_unit_tests/test_check_batch_cost.py

docs/my-website/docs/providers/reducto.md

@@ -0,0 +1,103 @@
+# Reducto
+
+## Overview
+
+| Property | Details |
+|-------|-------|
+| Description | Reducto parse support over LiteLLM's existing OCR API |
+| Provider Route on LiteLLM | `reducto/` |
+| Supported Operations | `/ocr` |
+| Supported Models | `reducto/parse-v3`, `reducto/parse-legacy` |
+| Link to Provider Doc | [Reducto ↗](https://platform.reducto.ai/) |
+
+Reducto is exposed through LiteLLM's OCR surface, so this provider uses `litellm.ocr()` and `litellm.aocr()`.
+
+## Quick Start
+
+### LiteLLM SDK
+
+```python showLineNumbers title="SDK Usage"
+import litellm
+import os
+
+os.environ["REDUCTO_API_KEY"] = "your-api-key"
+
+response = litellm.ocr(
+    model="reducto/parse-v3",
+    document={"type": "file", "file": "document.pdf"},
+)
+
+for page in response.pages:
+    print(page.markdown)
+```
+
+You can also override credentials per call with `api_key=` and `api_base=`.
+
+### LiteLLM Proxy
+
+```yaml showLineNumbers title="proxy_config.yaml"
+model_list:
+  - model_name: reducto-parse
+    litellm_params:
+      model: reducto/parse-v3
+      api_key: os.environ/REDUCTO_API_KEY
+    model_info:
+      mode: ocr
+```
+
+## Parse V3
+
+`reducto/parse-v3` maps to Reducto's current parse API and accepts:
+
+- `formatting`
+- `retrieval`
+- `settings`
+
+```python showLineNumbers title="Parse V3"
+response = await litellm.aocr(
+    model="reducto/parse-v3",
+    document={"type": "file", "file": "document.pdf"},
+    formatting={"table_output_format": "html"},
+    retrieval={"chunk_mode": "section"},
+    settings={"ocr_system": "standard"},
+)
+```
+
+## Parse Legacy
+
+`reducto/parse-legacy` keeps the legacy request shape and accepts `enhance`.
+
+```python showLineNumbers title="Parse Legacy"
+response = litellm.ocr(
+    model="reducto/parse-legacy",
+    document={"type": "file", "file": "document.pdf"},
+    enhance={"agentic": [{"type": "table"}]},
+)
+```
+
+## Upload Behavior
+
+- `document={"type":"file","file":...}` is auto-converted by LiteLLM into a data URI, then uploaded to Reducto's `/upload` endpoint before `/parse`.
+- `document_url="reducto://..."` is passed through directly and skips upload.
+- Plain `http(s)` document URLs are rejected for Reducto. Upload the file first or pass a local file to LiteLLM.
+- Image files also work through `type="file"`; LiteLLM normalizes them to `image_url` data URIs before the Reducto upload step.
+
+## Cost Tracking
+
+Reducto returns OCR usage in credits. LiteLLM supports credit-priced OCR models via `ocr_cost_per_credit`.
+
+If you want spend tracking, register pricing for your deployment:
+
+```python showLineNumbers title="Register OCR Credit Pricing"
+import litellm
+
+litellm.register_model(
+    {
+        "reducto/parse-v3": {
+            "litellm_provider": "reducto",
+            "mode": "ocr",
+            "ocr_cost_per_credit": 0.003,
+        }
+    }
+)
+```

docs/my-website/docs/proxy/config_settings.md

@@ -418,6 +418,8 @@ router_settings:
 | AIOHTTP_KEEPALIVE_TIMEOUT | Keep-alive timeout for aiohttp connections in seconds. **Default is 120**
 | AIOHTTP_TRUST_ENV | Flag to enable aiohttp trust environment. When this is set to True, aiohttp will respect HTTP(S)_PROXY env vars. **Default is False**
 | AIOHTTP_TTL_DNS_CACHE | DNS cache time-to-live for aiohttp in seconds. **Default is 300**
+| AKTO_GUARDRAIL_API_BASE | Base URL for the Akto Guardrail API (e.g. `http://localhost:9090`). Used by the Akto guardrail integration.
+| AKTO_API_KEY | API key for authenticating with the Akto Guardrail service.
 | ALLOWED_EMAIL_DOMAINS | List of email domains allowed for access
 | APSCHEDULER_COALESCE | Whether to combine multiple pending executions of a job into one. **Default is False**
 | APSCHEDULER_MAX_INSTANCES | Maximum number of concurrent instances of each job. **Default is 1**
@@ -1020,6 +1022,10 @@ router_settings:
 | REQUEST_TIMEOUT | Timeout in seconds for requests. Default is 6000
 | ROOT_REDIRECT_URL | URL to redirect root path (/) to when DOCS_URL is set to something other than "/" (DOCS_URL is "/" by default)
 | ROUTER_MAX_FALLBACKS | Maximum number of fallbacks for router. Default is 5
+| RUBRIK_API_KEY | Bearer token for authenticating with the Rubrik webhook service
+| RUBRIK_BATCH_SIZE | Number of log entries to buffer before flushing to Rubrik. Default is 512
+| RUBRIK_SAMPLING_RATE | Fraction of requests to log to Rubrik (0.0 to 1.0). Default is 1.0
+| RUBRIK_WEBHOOK_URL | Base URL of the Rubrik webhook service for tool blocking and batch logging
 | RUNWAYML_DEFAULT_API_VERSION | Default API version for RunwayML service. Default is "2024-11-06"
 | RUNWAYML_POLLING_TIMEOUT | Timeout in seconds for RunwayML image generation polling. Default is 600 (10 minutes)
 | S3_VECTORS_DEFAULT_DIMENSION | Default vector dimension for S3 Vectors RAG ingestion. Default is 1024

docs/my-website/docs/proxy/guardrails/rubrik.md

@@ -0,0 +1,188 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Rubrik Guardrail
+
+Use Rubrik's tool blocking and logging integration to validate LLM tool calls against an external policy service and batch-log all LLM requests/responses.
+
+**Key features:**
+- **Tool blocking**: Validates tool calls against an external Rubrik service after LLM completion. Blocked tool calls trigger a policy violation response.
+- **Batch logging**: Logs all LLM requests and responses to Rubrik with configurable sampling and batching.
+- **Fail-open**: If the tool blocking service is unavailable, requests are allowed through unchanged.
+
+---
+
+## Quick Start
+
+### 1. Configure `config.yaml`
+
+Credentials can be set directly in the YAML config or via environment variables. The config approach is recommended.
+
+<Tabs>
+<TabItem value="config" label="config.yaml (Recommended)" default>
+
+```yaml
+model_list:
+  - model_name: gpt-4
+    litellm_params:
+      model: openai/gpt-4
+      api_key: os.environ/OPENAI_API_KEY
+
+guardrails:
+  - guardrail_name: "rubrik"
+    litellm_params:
+      guardrail: rubrik
+      mode: "post_call"
+      api_key: "your-rubrik-api-key"
+      api_base: "https://your-rubrik-service.example.com"
+      default_on: true
+```
+
+You can also reference environment variables in the config:
+
+```yaml
+guardrails:
+  - guardrail_name: "rubrik"
+    litellm_params:
+      guardrail: rubrik
+      mode: "post_call"
+      api_key: os.environ/RUBRIK_API_KEY
+      api_base: os.environ/RUBRIK_WEBHOOK_URL
+      default_on: true
+```
+
+</TabItem>
+<TabItem value="env" label="Environment Variables">
+
+As an alternative, you can configure the Rubrik service URL and API key purely through environment variables. When set, these are used as fallbacks if `api_base` / `api_key` are not provided in the config.
+
+```bash
+export RUBRIK_WEBHOOK_URL="https://your-rubrik-service.example.com"
+export RUBRIK_API_KEY="your-rubrik-api-key"
+```
+
+With a minimal config:
+
+```yaml
+model_list:
+  - model_name: gpt-4
+    litellm_params:
+      model: openai/gpt-4
+      api_key: os.environ/OPENAI_API_KEY
+
+guardrails:
+  - guardrail_name: "rubrik"
+    litellm_params:
+      guardrail: rubrik
+      mode: "post_call"
+      default_on: true
+```
+
+</TabItem>
+</Tabs>
+
+### 2. Launch the Proxy
+
+```bash
+litellm --config config.yaml --port 4000
+```
+
+### 3. Test It
+
+```bash
+curl -X POST http://localhost:4000/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "model": "gpt-4",
+    "messages": [{"role": "user", "content": "What is the weather in SF?"}],
+    "tools": [
+      {
+        "type": "function",
+        "function": {
+          "name": "get_weather",
+          "description": "Get the weather for a location",
+          "parameters": {
+            "type": "object",
+            "properties": {
+              "location": {"type": "string"}
+            },
+            "required": ["location"]
+          }
+        }
+      }
+    ]
+  }'
+```
+
+---
+
+## Configuration Reference
+
+### YAML Config Parameters
+
+These are set under `guardrails.[].litellm_params` in your `config.yaml`:
+
+| Parameter | Required | Description |
+|-----------|----------|-------------|
+| `guardrail: rubrik` | Yes | Selects the Rubrik guardrail integration |
+| `mode: "post_call"` | Yes | Run after the LLM response is received |
+| `api_base` | Yes | Rubrik webhook base URL. Can use `os.environ/RUBRIK_WEBHOOK_URL`. Falls back to `RUBRIK_WEBHOOK_URL` env var if omitted. |
+| `api_key` | No | Rubrik API key. Can use `os.environ/RUBRIK_API_KEY`. Falls back to `RUBRIK_API_KEY` env var if omitted. |
+| `default_on` | No | When `true`, the guardrail runs on all requests without needing per-request opt-in |
+
+### Environment Variables
+
+These are optional fallbacks used when `api_base` / `api_key` are not set in the YAML config. `RUBRIK_SAMPLING_RATE` and `RUBRIK_BATCH_SIZE` can only be set via environment variables.
+
+| Variable | Required | Default | Description |
+|----------|----------|---------|-------------|
+| `RUBRIK_WEBHOOK_URL` | Only if `api_base` not in config | — | Base URL of the Rubrik webhook service |
+| `RUBRIK_API_KEY` | No | — | Bearer token for authenticating with the Rubrik service |
+| `RUBRIK_SAMPLING_RATE` | No | `1.0` | Fraction of requests to **log** (0.0 to 1.0). Does not affect tool blocking, which always runs. Set to `0.5` to log ~50% of requests. |
+| `RUBRIK_BATCH_SIZE` | No | `512` | Number of log entries to buffer before flushing. Logs are also flushed on a periodic interval. |
+
+---
+
+## How Tool Blocking Works
+
+1. After the LLM returns a response with tool calls, the Rubrik guardrail sends them to the blocking service at `{api_base}/v1/after_completion/openai/v1`.
+2. The service evaluates each tool call against configured policies and returns the set of **allowed** tool calls.
+3. If any tool calls are blocked, the proxy returns the policy violation explanation as a response instead of the original LLM response.
+4. If the blocking service is unreachable or returns an error, the guardrail **fails open** — the original response is returned unchanged.
+
+### Request/Response format
+
+The guardrail sends a JSON envelope to the blocking service:
+
+```json
+{
+  "request": {
+    "messages": [...],
+    "model": "gpt-4",
+    "proxy_server_request": {...}
+  },
+  "response": {
+    "id": "chatcmpl-...",
+    "object": "chat.completion",
+    "choices": [{
+      "message": {
+        "role": "assistant",
+        "tool_calls": [...]
+      }
+    }]
+  }
+}
+```
+
+The service should return an OpenAI chat completion format response containing only the **allowed** tool calls and an optional `content` field with the blocking explanation.
+
+---
+
+## How Batch Logging Works
+
+All LLM requests (successes and failures) are queued and sent in batches to `{api_base}/v1/litellm/batch`.
+
+- Logs are flushed when the queue reaches `RUBRIK_BATCH_SIZE` (default 512) or on a periodic interval (default 5 seconds). These defaults are inherited from LiteLLM's global settings.
+- Use `RUBRIK_SAMPLING_RATE` to reduce logging volume in high-traffic deployments. Sampling only affects logging — tool blocking always runs regardless of the sampling rate.
+- For Anthropic `/v1/messages` requests, the log ID is normalized to `litellm_call_id` for consistency across tool blocking and logging.

docs/my-website/sidebars.js

@@ -91,8 +91,10 @@ const sidebars = {
           "proxy/guardrails/custom_code_guardrail",
           "proxy/guardrails/prompt_injection",
           "proxy/guardrails/tool_permission",
+          "proxy/guardrails/rubrik",
           "proxy/guardrails/zscaler_ai_guard",
-          "proxy/guardrails/javelin"
+          "proxy/guardrails/javelin",
+          "proxy/guardrails/akto"
         ].sort(),
       ],
     },
@@ -994,6 +996,7 @@ const sidebars = {
         "providers/predibase",
         "providers/pydantic_ai_agent",
         "providers/ragflow",
+        "providers/reducto",
         "providers/recraft",
         "providers/replicate",
         {

litellm/__init__.py

@@ -617,6 +617,7 @@ def identify(event_details):
 aws_polly_models: Set = set()
 gigachat_models: Set = set()
 llamagate_models: Set = set()
+reducto_models: Set = set()
 bedrock_mantle_models: Set = set()
 
 
@@ -882,6 +883,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None):
             gigachat_models.add(key)
         elif value.get("litellm_provider") == "llamagate":
             llamagate_models.add(key)
+        elif value.get("litellm_provider") == "reducto":
+            reducto_models.add(key)
         elif value.get("litellm_provider") == "bedrock_mantle":
             bedrock_mantle_models.add(key)
 
@@ -992,6 +995,7 @@ def add_known_models(model_cost_map: Optional[Dict] = None):
     | ovhcloud_models
     | lemonade_models
     | docker_model_runner_models
+    | reducto_models
     | bedrock_mantle_models
     | set(clarifai_models)
 )
@@ -1097,6 +1101,7 @@ def add_known_models(model_cost_map: Optional[Dict] = None):
     "aws_polly": aws_polly_models,
     "gigachat": gigachat_models,
     "llamagate": llamagate_models,
+    "reducto": reducto_models,
     "bedrock_mantle": bedrock_mantle_models,
 }