🔄 sync: upstream v1.78.5-stable

.circleci/requirements.txt

@@ -1,5 +1,5 @@
 # used by CI/CD testing
-openai==1.81.0
+openai==1.100.1
 python-dotenv
 tiktoken
 importlib_metadata
@@ -14,4 +14,5 @@ google-cloud-iam==2.19.1
 fastapi-sso==0.16.0 
 uvloop==0.21.0
 mcp==1.10.1    # for MCP server  
-semantic_router==0.1.10 # for auto-routing with litellm
+semantic_router==0.1.10 # for auto-routing with litellm
+fastuuid==0.12.0

.devcontainer/devcontainer.json

@@ -11,7 +11,12 @@
 	// },
 
 	// Features to add to the dev container. More info: https://containers.dev/features.
-	// "features": {},
+	"features": {
+		"ghcr.io/devcontainers/features/node:1": {
+			"version": "lts"
+		},
+		"ghcr.io/devcontainers/features/docker-in-docker:2": {}
+	},
 
 	// Configure tool-specific properties.
 	"customizations": {
@@ -30,7 +35,7 @@
 
 	// Use 'forwardPorts' to make a list of ports inside the container available locally.
 	"forwardPorts": [4000],
-
+	
 	"containerEnv": {
 		"LITELLM_LOG": "DEBUG"
 	},
@@ -48,5 +53,5 @@
 	// "remoteUser": "litellm",
 
 	// Use 'postCreateCommand' to run commands after the container is created.
-	"postCreateCommand": "pipx install poetry && poetry install -E extra_proxy -E proxy"
+	"postCreateCommand": "bash ./.devcontainer/post-create.sh"
 }

.devcontainer/post-create.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -e
+
+echo "[post-create] Installing poetry via pip"
+python -m pip install --upgrade pip
+python -m pip install poetry
+
+echo "[post-create] Installing Python dependencies (poetry)"
+poetry install --with dev --extras proxy
+
+echo "[post-create] Generating Prisma client"
+poetry run prisma generate
+
+echo "[post-create] Installing npm dependencies"
+cd ui/litellm-dashboard && npm install --no-audit --no-fund
+
+echo "[post-create] Done"

.github/scripts/scan_keywords.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+import json
+import os
+import sys
+import urllib.request
+import urllib.error
+
+
+def read_event_payload() -> dict:
+    event_path = os.environ.get("GITHUB_EVENT_PATH")
+    if not event_path or not os.path.exists(event_path):
+        return {}
+    with open(event_path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def get_issue_text(event: dict) -> tuple[str, str, int, str, str]:
+    issue = event.get("issue") or {}
+    title = (issue.get("title") or "").strip()
+    body = (issue.get("body") or "").strip()
+    number = issue.get("number") or 0
+    html_url = issue.get("html_url") or ""
+    author = ((issue.get("user") or {}).get("login") or "").strip()
+    return title, body, number, html_url, author
+
+
+def detect_keywords(text: str, keywords: list[str]) -> list[str]:
+    lowered = text.lower()
+    matches = []
+    for keyword in keywords:
+        k = keyword.strip().lower()
+        if not k:
+            continue
+        if k in lowered:
+            matches.append(keyword.strip())
+    # Deduplicate while preserving order
+    seen = set()
+    unique_matches = []
+    for m in matches:
+        if m not in seen:
+            unique_matches.append(m)
+            seen.add(m)
+    return unique_matches
+
+
+def send_webhook(webhook_url: str, payload: dict) -> None:
+    if not webhook_url:
+        return
+    data = json.dumps(payload).encode("utf-8")
+    req = urllib.request.Request(
+        webhook_url,
+        data=data,
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=10) as resp:
+            resp.read()
+    except urllib.error.HTTPError as e:
+        print(f"Webhook HTTP error: {e.code} {e.reason}", file=sys.stderr)
+    except urllib.error.URLError as e:
+        print(f"Webhook URL error: {e.reason}", file=sys.stderr)
+    except Exception as e:
+        print(f"Webhook unexpected error: {e}", file=sys.stderr)
+
+
+def _excerpt(text: str, max_len: int = 400) -> str:
+    if not text:
+        return ""
+
+    # Keep original formatting
+    if len(text) <= max_len:
+        return text
+    return text[: max_len - 1] + "…"
+
+
+
+def main() -> int:
+    event = read_event_payload()
+    if not event:
+        print("::warning::No event payload found; exiting without labeling.")
+        return 0
+
+    # Read issue details
+    title, body, number, html_url, author = get_issue_text(event)
+    combined_text = f"{title}\n\n{body}".strip()
+
+    # Keywords from env or defaults
+    keywords_env = os.environ.get("KEYWORDS", "")
+    default_keywords = ["azure", "openai", "bedrock", "vertexai", "vertex ai", "anthropic"]
+    keywords = [k.strip() for k in keywords_env.split(",")] if keywords_env else default_keywords
+
+    matches = detect_keywords(combined_text, keywords)
+    found = bool(matches)
+
+    # Emit outputs
+    github_output = os.environ.get("GITHUB_OUTPUT")
+    if github_output:
+        with open(github_output, "a", encoding="utf-8") as fh:
+            fh.write(f"found={'true' if found else 'false'}\n")
+            fh.write(f"matches={','.join(matches)}\n")
+
+    # Optional webhook notification
+    webhook_url = os.environ.get("PROVIDER_ISSUE_WEBHOOK_URL", "").strip()
+    if found and webhook_url:
+        repo_full = (event.get("repository") or {}).get("full_name", "")
+        title_part = f"*{title}*" if title else "New issue"
+        author_part = f" by @{author}" if author else ""
+        body_preview = _excerpt(body)
+        preview_block = f"\n{body_preview}" if body_preview else ""
+        payload = {
+            "text": (
+                f"New issue 🚨\n"
+                f"{title_part}\n\n{preview_block}\n"
+                f"<{html_url}|View issue>\n"
+                f"Author: {author}"
+            )
+        }
+        send_webhook(webhook_url, payload)
+
+    # Print a short log line for Actions UI
+    if found:
+        print(f"Detected provider keywords: {', '.join(matches)}")
+    else:
+        print("No provider keywords detected.")
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
+
+

.github/workflows/auto_update_price_and_context_window_file.py

@@ -43,8 +43,8 @@ def write_to_file(file_path, data):
         # Print an error message if writing to file fails
         print("Error updating JSON file:", e)
 
-# Update the existing models and add the missing models
-def transform_remote_data(data):
+# Update the existing models and add the missing models for OpenRouter
+def transform_openrouter_data(data):
     transformed = {}
     for row in data:
         # Add the fields 'max_tokens' and 'input_cost_per_token'
@@ -81,6 +81,34 @@ def transform_remote_data(data):
 
     return transformed
 
+# Update the existing models and add the missing models for Vercel AI Gateway
+def transform_vercel_ai_gateway_data(data):
+    transformed = {}
+    for row in data:
+        obj = {
+            "max_tokens": row["context_window"],
+            "input_cost_per_token": float(row["pricing"]["input"]),
+            "output_cost_per_token": float(row["pricing"]["output"]),
+            'max_output_tokens': row['max_tokens'],
+            'max_input_tokens': row["context_window"],
+        }
+
+        # Handle cache pricing if available
+        if "pricing" in row:
+            if "input_cache_read" in row["pricing"] and row["pricing"]["input_cache_read"] is not None:
+                obj['cache_read_input_token_cost'] = float(f"{float(row['pricing']['input_cache_read']):e}")
+
+            if "input_cache_write" in row["pricing"] and row["pricing"]["input_cache_write"] is not None:
+                obj['cache_creation_input_token_cost'] = float(f"{float(row['pricing']['input_cache_write']):e}")
+
+        mode = "embedding" if "embedding" in row["id"].lower() else "chat"
+
+        obj.update({"litellm_provider": "vercel_ai_gateway", "mode": mode})
+
+        transformed[f'vercel_ai_gateway/{row["id"]}'] = obj
+
+    return transformed
+
 
 # Load local data from a specified file
 def load_local_data(file_path):
@@ -100,22 +128,32 @@ def load_local_data(file_path):
 
 def main():
     local_file_path = "model_prices_and_context_window.json"  # Path to the local data file
-    url = "https://openrouter.ai/api/v1/models"  # URL to fetch remote data
+    openrouter_url = "https://openrouter.ai/api/v1/models"  # URL to fetch OpenRouter data
+    vercel_ai_gateway_url = "https://ai-gateway.vercel.sh/v1/models"  # URL to fetch Vercel AI Gateway data
 
     # Load local data from file
     local_data = load_local_data(local_file_path)
-    # Fetch remote data asynchronously
-    remote_data = asyncio.run(fetch_data(url))
-    # Transform the fetched remote data
-    remote_data = transform_remote_data(remote_data)
-
-    # If both local and remote data are available, synchronize and save
-    if local_data and remote_data:
-        sync_local_data_with_remote(local_data, remote_data)
+
+    # Fetch OpenRouter data
+    openrouter_data = asyncio.run(fetch_data(openrouter_url))
+    # Transform the fetched OpenRouter data
+    openrouter_data = transform_openrouter_data(openrouter_data)
+
+    # Fetch Vercel AI Gateway data
+    vercel_data = asyncio.run(fetch_data(vercel_ai_gateway_url))
+    # Transform the fetched Vercel AI Gateway data
+    vercel_data = transform_vercel_ai_gateway_data(vercel_data)
+
+    # Combine both datasets
+    all_remote_data = {**openrouter_data, **vercel_data}
+
+    # If both local and openrouter data are available, synchronize and save
+    if local_data and all_remote_data:
+        sync_local_data_with_remote(local_data, all_remote_data)
         write_to_file(local_file_path, local_data)
     else:
         print("Failed to fetch model data from either local file or URL.")
 
 # Entry point of the script
 if __name__ == "__main__":
-    main()
+    main()

.github/workflows/interpret_load_test.py

@@ -88,6 +88,7 @@ def get_docker_run_command(release_version):
 
 
 if __name__ == "__main__":
+    return
     csv_file = "load_test_stats.csv"  # Change this to the path of your CSV file
     markdown_table = interpret_results(csv_file)
 

.github/workflows/issue-keyword-labeler.yml

@@ -0,0 +1,64 @@
+name: Issue Keyword Labeler
+
+on:
+  issues:
+    types: 
+        - opened
+
+jobs:
+  scan-and-label:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      contents: read
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Scan for provider keywords
+        id: scan
+        env:
+          PROVIDER_ISSUE_WEBHOOK_URL: ${{ secrets.PROVIDER_ISSUE_WEBHOOK_URL }}
+          KEYWORDS: azure,openai,bedrock,vertexai,vertex ai,anthropic
+        run: python3 .github/scripts/scan_keywords.py
+
+      - name: Ensure label exists
+        if: steps.scan.outputs.found == 'true'
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const labelName = 'llm translation';
+            try {
+              await github.rest.issues.getLabel({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                name: labelName
+              });
+            } catch (error) {
+              if (error.status === 404) {
+                await github.rest.issues.createLabel({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  name: labelName,
+                  color: 'c1ff72',
+                  description: 'Issues related to LLM provider translation/mapping'
+                });
+              } else {
+                throw error;
+              }
+            }
+
+      - name: Add label to the issue
+        if: steps.scan.outputs.found == 'true'
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            await github.rest.issues.addLabels({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              labels: ['llm translation']
+            });
+