From 47160b0720f0db232a11a9f8eddc571708ec8a36 Mon Sep 17 00:00:00 2001 From: yassin Date: Thu, 8 Oct 2026 02:04:43 +0000 Subject: [PATCH 01/38] feat(github_copilot): per-user GitHub OAuth connections for Copilot credentials Related to LIT-9306 --- .../migration.sql | 18 + .../litellm_proxy_extras/schema.prisma | 14 + litellm/__init__.py | 2 + litellm/caching/caching_handler.py | 12 +- litellm/constants.py | 7 + litellm/exceptions.py | 12 + .../litellm_core_utils/get_litellm_params.py | 2 + .../get_llm_provider_logic.py | 28 +- .../pass_through/messages/handler.py | 1 + litellm/llms/github_copilot/authenticator.py | 32 +- .../github_copilot/chat/transformation.py | 20 +- .../embedding/transformation.py | 9 +- .../github_copilot/messages/transformation.py | 20 +- litellm/llms/github_copilot/per_user_auth.py | 620 ++++++++++++ .../responses/transformation.py | 29 +- litellm/main.py | 19 +- litellm/models/credentials.py | 33 + litellm/proxy/auth/auth_utils.py | 6 + litellm/proxy/auth/route_checks.py | 25 +- .../proxy/credential_endpoints/endpoints.py | 304 +++++- .../user_provider_credentials.py | 219 ++++ litellm/proxy/litellm_pre_call_utils.py | 133 +++ litellm/proxy/schema.prisma | 14 + litellm/responses/main.py | 13 +- litellm/router.py | 4 + .../clientside_credential_handler.py | 19 + litellm/router_utils/cooldown_handlers.py | 33 + .../router_utils/fallback_event_handlers.py | 1 + litellm/types/litellm_params.py | 2 + litellm/types/proxy/litellm_pre_call_utils.py | 6 +- litellm/types/utils.py | 6 +- litellm/utils.py | 6 + schema.prisma | 14 + .../endpointaudit/coverage_allowlist.txt | 7 + ...github_copilot_embedding_transformation.py | 31 + ..._github_copilot_messages_transformation.py | 42 + ...github_copilot_responses_transformation.py | 43 +- .../test_github_copilot_transformation.py | 48 + .../llms/github_copilot/test_per_user_auth.py | 954 ++++++++++++++++++ tests/unit/proxy/auth/test_auth_utils.py | 79 ++ tests/unit/proxy/auth/test_route_checks.py | 88 ++ .../credential_endpoints/test_endpoints.py | 537 +++++++++- .../test_user_provider_credentials.py | 149 +++ .../test_model_management_endpoints.py | 185 ++++ .../unit/proxy/test_litellm_pre_call_utils.py | 515 +++++++++- .../router_utils/test_cooldown_handlers.py | 47 +- tests/unit/test_internal_context.py | 1 + tests/unit/types/test_litellm_params.py | 2 + tests/unit/types/test_router.py | 11 +- .../hooks/credentials/useUserConnections.ts | 15 + .../models-and-endpoints/page.test.tsx | 11 +- .../(dashboard)/models-and-endpoints/page.tsx | 5 +- .../panels/LlmCredentialsPanel.test.tsx | 91 ++ .../panels/LlmCredentialsPanel.tsx | 11 +- .../add_model/provider_auth_types.test.ts | 9 + .../add_model/provider_auth_types.ts | 23 + .../add_model/provider_specific_fields.tsx | 5 + .../CredentialModal.integration.test.tsx | 66 ++ .../GithubCopilotConnectModal.test.tsx | 124 +++ .../model_add/GithubCopilotConnectModal.tsx | 136 +++ .../model_add/UserConnectionsPanel.test.tsx | 50 + .../model_add/UserConnectionsPanel.tsx | 130 +++ .../github_copilot_connect_flow.test.ts | 79 ++ .../model_add/github_copilot_connect_flow.ts | 99 ++ .../src/components/networking.tsx | 48 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 242 +++++ 66 files changed, 5460 insertions(+), 106 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20261008120000_user_provider_credentials/migration.sql create mode 100644 litellm/llms/github_copilot/per_user_auth.py create mode 100644 litellm/proxy/credential_endpoints/user_provider_credentials.py create mode 100644 tests/unit/llms/github_copilot/test_per_user_auth.py create mode 100644 tests/unit/proxy/credential_endpoints/test_user_provider_credentials.py create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/credentials/useUserConnections.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/LlmCredentialsPanel.test.tsx create mode 100644 ui/litellm-dashboard/src/components/model_add/GithubCopilotConnectModal.test.tsx create mode 100644 ui/litellm-dashboard/src/components/model_add/GithubCopilotConnectModal.tsx create mode 100644 ui/litellm-dashboard/src/components/model_add/UserConnectionsPanel.test.tsx create mode 100644 ui/litellm-dashboard/src/components/model_add/UserConnectionsPanel.tsx create mode 100644 ui/litellm-dashboard/src/components/model_add/github_copilot_connect_flow.test.ts create mode 100644 ui/litellm-dashboard/src/components/model_add/github_copilot_connect_flow.ts diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261008120000_user_provider_credentials/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261008120000_user_provider_credentials/migration.sql new file mode 100644 index 000000000000..ea4fe9bc0ab9 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261008120000_user_provider_credentials/migration.sql @@ -0,0 +1,18 @@ +-- CreateTable +CREATE TABLE IF NOT EXISTS "LiteLLM_UserProviderCredentials" ( + "id" TEXT NOT NULL, + "user_id" TEXT NOT NULL, + "credential_name" TEXT NOT NULL, + "provider" TEXT NOT NULL, + "credential_b64" TEXT NOT NULL, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "LiteLLM_UserProviderCredentials_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_UserProviderCredentials_user_id_credential_name_key" ON "LiteLLM_UserProviderCredentials"("user_id", "credential_name"); + +-- CreateIndex +CREATE INDEX IF NOT EXISTS "LiteLLM_UserProviderCredentials_credential_name_idx" ON "LiteLLM_UserProviderCredentials"("credential_name"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 038dfdeaca5f..92f2cedf7d6f 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -452,6 +452,20 @@ model LiteLLM_MCPUserCredentials { @@unique([user_id, server_id]) } +// Per-user provider connections (e.g. GitHub Copilot OAuth) keyed by credential name +model LiteLLM_UserProviderCredentials { + id String @id @default(uuid()) + user_id String + credential_name String + provider String + credential_b64 String + created_at DateTime @default(now()) @map("created_at") + updated_at DateTime @default(now()) @updatedAt @map("updated_at") + + @@unique([user_id, credential_name]) + @@index([credential_name]) +} + // Per-user environment variable values for MCP servers. // values_b64 is an encrypted JSON object: {VAR_NAME: "value", ...}. model LiteLLM_MCPUserEnvVars { diff --git a/litellm/__init__.py b/litellm/__init__.py index c0dc61e2911d..d1378a6d7846 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1392,6 +1392,8 @@ def _build_models_by_provider() -> dict: from .llms.custom_httpx.async_client_cleanup import close_litellm_async_clients from .exceptions import ( AuthenticationError, + CallerCredentialAuthenticationError, + CallerCredentialRateLimitError, InvalidRequestError, BadRequestError, ImageFetchError, diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index f1d7d6ebe730..5e91d60f11d0 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -105,11 +105,15 @@ def _drop_logging_obj_from_kwargs(request_kwargs: dict[str, object]) -> dict[str def _is_response_cache_excluded(model: str | None, kwargs: Mapping[str, object]) -> bool: + from litellm.llms.github_copilot.per_user_auth import is_github_copilot_per_user_request + custom_llm_provider: Final = kwargs.get("custom_llm_provider") model_provider: Final = model.split("/", maxsplit=1)[0] if model is not None else None return ( - isinstance(custom_llm_provider, str) and custom_llm_provider in RESPONSE_CACHE_EXCLUDED_PROVIDERS - ) or model_provider in RESPONSE_CACHE_EXCLUDED_PROVIDERS + (isinstance(custom_llm_provider, str) and custom_llm_provider in RESPONSE_CACHE_EXCLUDED_PROVIDERS) + or model_provider in RESPONSE_CACHE_EXCLUDED_PROVIDERS + or is_github_copilot_per_user_request(kwargs) + ) def _is_chat_completion_cached_dict(cached_result: dict) -> bool: @@ -315,6 +319,7 @@ async def async_get_cache( None """ if _is_response_cache_excluded(model=model, kwargs=kwargs): + self.request_kwargs = _drop_logging_obj_from_kwargs(kwargs) return None # Check if caching should be performed BEFORE doing expensive operations @@ -442,6 +447,7 @@ def sync_get_cache( # Check if caching should be performed BEFORE doing expensive kwargs copy if _is_response_cache_excluded(model=model, kwargs=kwargs): + self.request_kwargs = _drop_logging_obj_from_kwargs(kwargs) return CachingHandlerResponse(cached_result=None) if litellm.cache is not None and self._is_call_type_supported_by_cache(original_function=original_function): args = args or () @@ -861,11 +867,11 @@ async def _retrieve_from_cache( new_kwargs.pop("metadata", None) model_value: Final = new_kwargs.get("model") model: Final = model_value if isinstance(model_value, str) else None + self.request_kwargs = _drop_logging_obj_from_kwargs(new_kwargs) if _is_response_cache_excluded(model=model, kwargs=new_kwargs): return None if new_kwargs.get("stream") is True and "cache_key" not in new_kwargs: new_kwargs["cache_key"] = litellm.cache.get_cache_key(**new_kwargs) - self.request_kwargs = _drop_logging_obj_from_kwargs(new_kwargs) cached_result: object | None = None if call_type == CallTypes.aembedding.value: if isinstance(new_kwargs["input"], str): diff --git a/litellm/constants.py b/litellm/constants.py index 290a7b32fe0e..aba2f6fc405b 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -12,6 +12,13 @@ # Replies depend on the caller's delegated identity, which response-cache keys do not include. RESPONSE_CACHE_EXCLUDED_PROVIDERS: Final = frozenset({"microsoft_365_copilot"}) MICROSOFT_365_COPILOT_DEFAULT_TIME_ZONE: Final = "UTC" +GITHUB_COPILOT_PER_USER_AUTH_TYPE: Final = "per_user_oauth" +GITHUB_COPILOT_AUTH_TYPE_KEY: Final = "github_copilot_auth_type" +GITHUB_COPILOT_USER_TOKEN_SAFETY_MARGIN_SECONDS: Final = 60 +GITHUB_COPILOT_USER_CREDENTIAL_CACHE_TTL_SECONDS: Final = 60 +GITHUB_COPILOT_DEVICE_FLOW_CACHE_PREFIX: Final = "github_copilot_device_flow" +USER_PROVIDER_CREDENTIAL_CACHE_PREFIX: Final = "user_provider_credential" +USER_PROVIDER_CREDENTIAL_NOT_CONNECTED: Final = "__not_connected__" DEFER_PYDANTIC_BUILD: Final = os.getenv("DEFER_PYDANTIC_BUILD", "true") in ("true", "1", "on") DEFAULT_HEALTH_CHECK_PROMPT: Final = str(os.getenv("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm")) AZURE_DEFAULT_RESPONSES_API_VERSION: Final = str(os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview")) diff --git a/litellm/exceptions.py b/litellm/exceptions.py index b349b1a1bb54..439eadce13d3 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -172,6 +172,12 @@ def __repr__(self): return _message +class CallerCredentialAuthenticationError(AuthenticationError): + """401 raised when the CALLING user's own stored provider credential is + missing or rejected (e.g. a per-user OAuth connection). Scoped to one + caller's credential, so routers must not cool down the shared deployment.""" + + # raise when invalid models passed, example gpt-8 class NotFoundError(openai.NotFoundError): def __init__( @@ -531,6 +537,12 @@ def __repr__(self): return _message +class CallerCredentialRateLimitError(RateLimitError): + """429 raised when a per-user credential exchange is rate limited by the + provider (e.g. GitHub's token endpoint). Scoped to one caller's + credential, so routers must not cool down the shared deployment.""" + + # sub class of rate limit error - meant to give more granularity for error handling context window exceeded errors class ContextWindowExceededError(BadRequestError): def __init__( diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index 2ddaf3b5bc88..76a717694481 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -75,6 +75,8 @@ "itpm", "otpm", "use_xai_oauth", + "github_copilot_auth_type", + "github_copilot_user_session", PROVIDER_AFFINITY_HEADER_KWARG_KEY, } ) diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 724be5510f57..ced4a6f9118c 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -795,13 +795,29 @@ def _get_openai_compatible_provider_info( api_base = api_base or get_secret("GALADRIEL_API_BASE") or "https://api.galadriel.com/v1" dynamic_api_key = api_key or get_secret_str("GALADRIEL_API_KEY") elif custom_llm_provider == "github_copilot": - ( - api_base, - dynamic_api_key, - custom_llm_provider, - ) = litellm.GithubCopilotConfig().get_openai_compatible_provider_info( - model, api_base, api_key, custom_llm_provider + from litellm.llms.github_copilot.common_utils import DEFAULT_GITHUB_COPILOT_API_BASE + from litellm.llms.github_copilot.per_user_auth import ( + github_copilot_per_user_credential_name, + github_copilot_user_session_from, ) + + user_session: Final = github_copilot_user_session_from(litellm_params) # pyright: ignore[reportUnknownArgumentType] # litellm_params arrives as an untyped request dict + if user_session is not None: + api_base = user_session.api_base + dynamic_api_key = user_session.token + elif github_copilot_per_user_credential_name(litellm_params) is not None: # pyright: ignore[reportUnknownArgumentType] # litellm_params arrives as an untyped request dict + # per-user deployments resolve no token at provider-info time; the + # caller's session supplies base + key per request + api_base = api_base or DEFAULT_GITHUB_COPILOT_API_BASE + dynamic_api_key = None + else: + ( + api_base, + dynamic_api_key, + custom_llm_provider, + ) = litellm.GithubCopilotConfig().get_openai_compatible_provider_info( + model, api_base, api_key, custom_llm_provider + ) elif custom_llm_provider == "chatgpt": ( api_base, diff --git a/litellm/llms/anthropic/pass_through/messages/handler.py b/litellm/llms/anthropic/pass_through/messages/handler.py index 4e7a154be672..d19f0aa4b36e 100644 --- a/litellm/llms/anthropic/pass_through/messages/handler.py +++ b/litellm/llms/anthropic/pass_through/messages/handler.py @@ -467,6 +467,7 @@ def anthropic_messages_handler( custom_llm_provider=custom_llm_provider, api_base=litellm_params.api_base, api_key=litellm_params.api_key, + litellm_params=litellm_params, ) # Store agentic loop params in logging object for agentic hooks diff --git a/litellm/llms/github_copilot/authenticator.py b/litellm/llms/github_copilot/authenticator.py index a58f6b08a896..59e1804f4532 100644 --- a/litellm/llms/github_copilot/authenticator.py +++ b/litellm/llms/github_copilot/authenticator.py @@ -45,6 +45,22 @@ class _AccessTokenPoll(TypedDict): _ACCESS_TOKEN_POLL: Final = TypeAdapter(_AccessTokenPoll) +def github_api_headers( + access_token: str | None = None, +) -> dict[str, str]: # mutable-ok: returned straight into httpx handlers whose headers params require dict + headers: Final = { + "accept": "application/json", + "editor-version": "vscode/1.85.1", + "editor-plugin-version": "copilot/1.155.0", + "user-agent": "GithubCopilot/1.155.0", + "accept-encoding": "gzip,deflate,br", + "content-type": "application/json", + } + if access_token: + headers["authorization"] = f"token {access_token}" + return headers + + class Authenticator: def __init__(self) -> None: """Initialize the GitHub Copilot authenticator with configurable token paths.""" @@ -230,21 +246,7 @@ def _get_github_headers(self, access_token: str | None = None) -> dict[str, str] Returns: Dict[str, str]: Headers for GitHub API requests. """ - headers: Final = { - "accept": "application/json", - "editor-version": "vscode/1.85.1", - "editor-plugin-version": "copilot/1.155.0", - "user-agent": "GithubCopilot/1.155.0", - "accept-encoding": "gzip,deflate,br", - } - - if access_token: - headers["authorization"] = f"token {access_token}" - - if "content-type" not in headers: - headers["content-type"] = "application/json" - - return headers + return github_api_headers(access_token) def _get_device_code(self) -> _DeviceCode: """ diff --git a/litellm/llms/github_copilot/chat/transformation.py b/litellm/llms/github_copilot/chat/transformation.py index b17bd74365cb..b075d025c48b 100644 --- a/litellm/llms/github_copilot/chat/transformation.py +++ b/litellm/llms/github_copilot/chat/transformation.py @@ -17,6 +17,7 @@ GetAPIKeyError, get_copilot_default_headers, ) +from ..per_user_auth import require_github_copilot_user_session if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -109,12 +110,19 @@ def validate_environment( ) # Add Copilot-specific headers (editor-version, user-agent, etc.) - try: - copilot_api_key: Final = self.authenticator.get_api_key() - copilot_headers: Final = get_copilot_default_headers(copilot_api_key) - validated_headers = {**copilot_headers, **validated_headers} - except GetAPIKeyError: - pass # Will be handled later in the request flow + user_session: Final = require_github_copilot_user_session(litellm_params) # pyright: ignore[reportUnknownArgumentType] # litellm_params arrives as an untyped request dict + if user_session is not None: + validated_headers = {**get_copilot_default_headers(user_session.token), **validated_headers} + # the caller's stored session token wins unconditionally, even over a + # caller-supplied api_key or the parent's placeholder Authorization + validated_headers["Authorization"] = f"Bearer {user_session.token}" + else: + try: + copilot_api_key: Final = self.authenticator.get_api_key() + copilot_headers: Final = get_copilot_default_headers(copilot_api_key) + validated_headers = {**copilot_headers, **validated_headers} + except GetAPIKeyError: + pass # Will be handled later in the request flow # Add X-Initiator header based on message roles initiator: Final = self._determine_initiator(messages) diff --git a/litellm/llms/github_copilot/embedding/transformation.py b/litellm/llms/github_copilot/embedding/transformation.py index c1acf6fccaad..b4937912a8d7 100644 --- a/litellm/llms/github_copilot/embedding/transformation.py +++ b/litellm/llms/github_copilot/embedding/transformation.py @@ -26,6 +26,7 @@ GetAPIKeyError, get_copilot_default_headers, ) +from ..per_user_auth import require_github_copilot_user_session if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -61,6 +62,9 @@ def validate_environment( """ Validate environment and set up headers for GitHub Copilot API. """ + user_session: Final = require_github_copilot_user_session(litellm_params) # pyright: ignore[reportUnknownArgumentType] # litellm_params arrives as an untyped request dict + if user_session is not None: + return {**get_copilot_default_headers(user_session.token), **headers} try: # Get GitHub Copilot API key via OAuth api_key = self.authenticator.get_api_key() @@ -101,9 +105,10 @@ def get_complete_url( """ Get the complete URL for GitHub Copilot Embedding API endpoint. """ - # Use provided api_base or fall back to authenticator's base or default + user_session: Final = require_github_copilot_user_session(litellm_params) # pyright: ignore[reportUnknownArgumentType] # litellm_params arrives as an untyped request dict effective_api_base = ( - api_base + (user_session.api_base if user_session is not None else None) + or api_base or self.authenticator.get_api_base() or os.getenv("GITHUB_COPILOT_API_BASE") or DEFAULT_GITHUB_COPILOT_API_BASE diff --git a/litellm/llms/github_copilot/messages/transformation.py b/litellm/llms/github_copilot/messages/transformation.py index b36b437e2e5c..76cb55d3633d 100644 --- a/litellm/llms/github_copilot/messages/transformation.py +++ b/litellm/llms/github_copilot/messages/transformation.py @@ -11,6 +11,7 @@ GetAPIKeyError, get_copilot_default_headers, ) +from ..per_user_auth import require_github_copilot_user_session _MESSAGES_PROXY_API_VERSION: Final = "2026-06-01" @@ -68,9 +69,16 @@ def validate_anthropic_messages_environment( # session, never the caller-supplied api_base. rstrip so a # tenant-specific base with a trailing slash does not yield a # double-slash URL once "/v1/messages" is appended downstream. - dynamic_api_base: Final = (self.authenticator.get_api_base() or DEFAULT_GITHUB_COPILOT_API_BASE).rstrip("/") + user_session: Final = require_github_copilot_user_session(litellm_params) # pyright: ignore[reportUnknownArgumentType] # litellm_params arrives as an untyped request dict + dynamic_api_base: Final = ( + user_session.api_base + if user_session is not None + else (self.authenticator.get_api_base() or DEFAULT_GITHUB_COPILOT_API_BASE) + ).rstrip("/") try: - dynamic_api_key: Final = self.authenticator.get_api_key() + dynamic_api_key: Final = ( + user_session.token if user_session is not None else self.authenticator.get_api_key() + ) except GetAPIKeyError as e: raise AuthenticationError( model=model, @@ -116,7 +124,13 @@ def get_complete_url( reuse it to avoid a second authenticator read, falling back to a fresh resolution only if it was not provided. """ - resolved = (api_base or self.authenticator.get_api_base() or DEFAULT_GITHUB_COPILOT_API_BASE).rstrip("/") + user_session: Final = require_github_copilot_user_session(litellm_params) + resolved = ( + (user_session.api_base if user_session is not None else None) + or api_base + or self.authenticator.get_api_base() + or DEFAULT_GITHUB_COPILOT_API_BASE + ).rstrip("/") if not resolved.endswith("/v1/messages"): resolved = f"{resolved}/v1/messages" return resolved diff --git a/litellm/llms/github_copilot/per_user_auth.py b/litellm/llms/github_copilot/per_user_auth.py new file mode 100644 index 000000000000..1f454f3c41be --- /dev/null +++ b/litellm/llms/github_copilot/per_user_auth.py @@ -0,0 +1,620 @@ +"""Per-user GitHub Copilot OAuth: exchange a caller's stored GitHub token for a +short-lived Copilot token, plus the device-flow helpers the proxy endpoints use +to establish that connection. The shared device login (Authenticator) is never +touched on the per-user path. +""" + +import asyncio +import hashlib +import os +import threading +import time +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import Final, Literal, TypeAlias +from urllib.parse import urlparse + +import httpx +from pydantic import ConfigDict, TypeAdapter, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict + +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.constants import ( + GITHUB_COPILOT_AUTH_TYPE_KEY, + GITHUB_COPILOT_PER_USER_AUTH_TYPE, + GITHUB_COPILOT_USER_TOKEN_SAFETY_MARGIN_SECONDS, +) +from litellm.exceptions import ( + APIConnectionError, + BadRequestError, + CallerCredentialAuthenticationError, + CallerCredentialRateLimitError, + ServiceUnavailableError, +) + +from .authenticator import ( + DEFAULT_GITHUB_ACCESS_TOKEN_URL, + DEFAULT_GITHUB_API_KEY_URL, + DEFAULT_GITHUB_CLIENT_ID, + DEFAULT_GITHUB_DEVICE_CODE_URL, + github_api_headers, +) +from .common_utils import DEFAULT_GITHUB_COPILOT_API_BASE + +GITHUB_COPILOT_USER_SESSION_KWARG_KEY: Final = "github_copilot_user_session" +_LLM_PROVIDER: Final = "github_copilot" +_DEVICE_FLOW_SCOPE: Final = "read:user" +_SESSION_CACHE_MAX_SIZE: Final = 1000 +_SYNC_LOCK_COUNT: Final = 64 + +_SESSION_CACHE: Final = InMemoryCache( + max_size_in_memory=_SESSION_CACHE_MAX_SIZE, + default_ttl=GITHUB_COPILOT_USER_TOKEN_SAFETY_MARGIN_SECONDS, +) +_EXCHANGE_LOCKS: Final = tuple(threading.Lock() for _ in range(_SYNC_LOCK_COUNT)) +_IN_FLIGHT: Final[ # mutable-ok: single-flight dedup registry mutated under its own locking + dict[str, "asyncio.Future[GithubCopilotUserSession]"] +] = {} + + +@with_config(ConfigDict(extra="allow", strict=True)) +class _CopilotTokenPayload(TypedDict): + token: ReadOnly[str] + expires_at: ReadOnly[NotRequired[object]] + endpoints: ReadOnly[NotRequired[object]] + + +@with_config(ConfigDict(extra="allow", strict=True)) +class _DeviceCodePayload(TypedDict): + device_code: ReadOnly[str] + user_code: ReadOnly[str] + verification_uri: ReadOnly[str] + expires_in: ReadOnly[object] + interval: ReadOnly[object] + + +@with_config(ConfigDict(extra="allow", strict=True)) +class _AccessTokenPollPayload(TypedDict): + access_token: ReadOnly[NotRequired[str]] + error: ReadOnly[NotRequired[object]] + error_description: ReadOnly[NotRequired[object]] + interval: ReadOnly[NotRequired[object]] + + +@with_config(ConfigDict(extra="allow", strict=True)) +class _CopilotEndpointsPayload(TypedDict): + api: ReadOnly[NotRequired[object]] + + +@with_config(ConfigDict(extra="allow", strict=True)) +class _GithubUserPayload(TypedDict): + login: ReadOnly[str] + + +_COPILOT_TOKEN: Final = TypeAdapter(_CopilotTokenPayload) +_DEVICE_CODE: Final = TypeAdapter(_DeviceCodePayload) +_ACCESS_TOKEN_POLL: Final = TypeAdapter(_AccessTokenPollPayload) +_GITHUB_USER: Final = TypeAdapter(_GithubUserPayload) +_COPILOT_ENDPOINTS: Final = TypeAdapter(_CopilotEndpointsPayload) +_CACHED_SESSION_ENTRY: Final = TypeAdapter(tuple[str, str, float]) + + +@dataclass(frozen=True, slots=True, repr=False) +class GithubCopilotUserSession: + """A resolved per-user Copilot token plus the validated host it may be sent + to. Only this module constructs it, so ``isinstance`` checks downstream are + unforgeable by request JSON.""" + + token: str = field(repr=False) + api_base: str + + def __repr__(self) -> str: + return f"GithubCopilotUserSession(api_base={self.api_base!r}, token=REDACTED)" + + __str__ = __repr__ + + +@dataclass(frozen=True, slots=True) +class GithubCopilotDeviceFlowStart: + device_code: str = field(repr=False) + user_code: str + verification_uri: str + expires_in: int + interval: int + + +DeviceFlowPollStatus: TypeAlias = Literal["pending", "slow_down", "expired", "denied", "connected"] + + +@dataclass(frozen=True, slots=True) +class GithubCopilotDeviceFlowPoll: + status: DeviceFlowPollStatus + interval: int | None = None + access_token: str | None = field(default=None, repr=False) + + +def validated_copilot_api_base(endpoints_api: object) -> str: + """Only a genuine *.githubcopilot.com HTTPS endpoint may receive a Copilot + token; anything else falls back to the default base.""" + if not isinstance(endpoints_api, str): + return DEFAULT_GITHUB_COPILOT_API_BASE + parsed: Final = urlparse(endpoints_api) + hostname: Final = (parsed.hostname or "").lower() + if ( + parsed.scheme.lower() != "https" + or not (hostname == "githubcopilot.com" or hostname.endswith(".githubcopilot.com")) + or parsed.username is not None + or parsed.password is not None + ): + return DEFAULT_GITHUB_COPILOT_API_BASE + try: + port: Final = parsed.port + except ValueError: + return DEFAULT_GITHUB_COPILOT_API_BASE + if port not in (None, 443): + return DEFAULT_GITHUB_COPILOT_API_BASE + return endpoints_api.rstrip("/") + + +def _session_cache_key(user_id: str, github_token: str) -> str: + return hashlib.sha256(f"{user_id}\x00{github_token}".encode()).hexdigest() + + +def evict_copilot_user_session(user_id: str, github_token: str) -> None: + _SESSION_CACHE.delete_cache(_session_cache_key(user_id, github_token)) + + +def _cached_session(cache_key: str) -> GithubCopilotUserSession | None: + cached: Final[object] = _SESSION_CACHE.get_cache(cache_key) + try: + token, api_base, expires_at = _CACHED_SESSION_ENTRY.validate_python(cached) + except Exception: + return None + if expires_at - GITHUB_COPILOT_USER_TOKEN_SAFETY_MARGIN_SECONDS <= time.time(): + return None + return GithubCopilotUserSession(token=token, api_base=api_base) + + +def _cache_session(cache_key: str, session: GithubCopilotUserSession, expires_at: float) -> None: + ttl: Final = expires_at - GITHUB_COPILOT_USER_TOKEN_SAFETY_MARGIN_SECONDS - time.time() + if ttl <= 0: + return + _SESSION_CACHE.set_cache(cache_key, (session.token, session.api_base, expires_at), ttl=ttl) + + +def _reject_connection(credential_name: str) -> CallerCredentialAuthenticationError: + return CallerCredentialAuthenticationError( + message=( + f"GitHub rejected the GitHub Copilot connection for credential '{credential_name}'. " + f"Reconnect GitHub Copilot for credential '{credential_name}' in the LiteLLM UI (LLM Credentials)" + ), + llm_provider=_LLM_PROVIDER, + model="", + ) + + +def _session_from_response(response: httpx.Response, credential_name: str, cache_key: str) -> GithubCopilotUserSession: + if response.status_code in (401, 403, 404): + _SESSION_CACHE.delete_cache(cache_key) + raise _reject_connection(credential_name) + if response.status_code == 429: + raise CallerCredentialRateLimitError( + message="GitHub rate limited the GitHub Copilot token exchange", + llm_provider=_LLM_PROVIDER, + model="", + ) + if not response.is_success: + raise ServiceUnavailableError( + message=f"GitHub Copilot token exchange failed with status {response.status_code}", + llm_provider=_LLM_PROVIDER, + model="", + ) + try: + payload: Final = _COPILOT_TOKEN.validate_python(response.json()) + except Exception as e: + raise ServiceUnavailableError( + message="GitHub Copilot token exchange returned an invalid response", + llm_provider=_LLM_PROVIDER, + model="", + ) from e + + endpoints: Final[object] = payload.get("endpoints") + endpoints_api: Final[object] = ( + _COPILOT_ENDPOINTS.validate_python(endpoints).get("api") if isinstance(endpoints, Mapping) else None + ) + api_base: Final = validated_copilot_api_base(endpoints_api) + session: Final = GithubCopilotUserSession(token=payload["token"], api_base=api_base) + expires_at: Final = _expires_at_seconds(payload.get("expires_at")) + if expires_at > 0: + _cache_session(cache_key, session, expires_at) + return session + + +def _to_int(raw: object) -> int: + if isinstance(raw, bool): + return 0 + if isinstance(raw, int): + return raw + if isinstance(raw, float): + return int(raw) + if isinstance(raw, str): + try: + return int(raw) + except ValueError: + return 0 + return 0 + + +def _expires_at_seconds(raw: object) -> float: + if isinstance(raw, bool): + return 0.0 + if isinstance(raw, (int, float)): + return float(raw) + if isinstance(raw, str): + try: + return float(raw) + except ValueError: + return 0.0 + return 0.0 + + +def _do_exchange(github_token: str, credential_name: str, cache_key: str) -> GithubCopilotUserSession: + import litellm + + try: + response: Final = litellm.module_level_client.get( + DEFAULT_GITHUB_API_KEY_URL, + headers=github_api_headers(github_token), + ) + except Exception as e: + raise APIConnectionError( + message="GitHub Copilot token exchange request failed", + llm_provider=_LLM_PROVIDER, + model="", + ) from e + return _session_from_response(response, credential_name, cache_key) + + +async def _do_aexchange(github_token: str, credential_name: str, cache_key: str) -> GithubCopilotUserSession: + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + + client: Final = get_async_httpx_client(_LLM_PROVIDER) + try: + response: Final = await client.get( + DEFAULT_GITHUB_API_KEY_URL, + headers=github_api_headers(github_token), + ) + except Exception as e: + raise APIConnectionError( + message="GitHub Copilot token exchange request failed", + llm_provider=_LLM_PROVIDER, + model="", + ) from e + return _session_from_response(response, credential_name, cache_key) + + +def exchange_github_token( + user_id: str, + github_token: str, + credential_name: str, +) -> GithubCopilotUserSession: + cache_key: Final = _session_cache_key(user_id, github_token) + cached: Final = _cached_session(cache_key) + if cached is not None: + return cached + lock: Final = _EXCHANGE_LOCKS[int(cache_key, 16) % _SYNC_LOCK_COUNT] + with lock: + rechecked: Final = _cached_session(cache_key) + if rechecked is not None: + return rechecked + return _do_exchange(github_token, credential_name, cache_key) + + +async def aexchange_github_token( + user_id: str, + github_token: str, + credential_name: str, +) -> GithubCopilotUserSession: + cache_key: Final = _session_cache_key(user_id, github_token) + cached: Final = _cached_session(cache_key) + if cached is not None: + return cached + loop: Final = asyncio.get_running_loop() + in_flight: Final = _IN_FLIGHT.get(cache_key) + if in_flight is not None and not in_flight.done() and in_flight.get_loop() is loop: + return await asyncio.shield(in_flight) + future: Final = loop.create_future() + _IN_FLIGHT[cache_key] = future + try: + session: Final = await _do_aexchange(github_token, credential_name, cache_key) + except BaseException as e: + if not future.done(): + future.set_exception(e) + future.exception() # consume so unawaited waiters do not log "exception was never retrieved" + raise + else: + if not future.done(): + future.set_result(session) + return session + finally: + if _IN_FLIGHT.get(cache_key) is future: + _IN_FLIGHT.pop(cache_key, None) + + +async def astart_device_flow() -> GithubCopilotDeviceFlowStart: + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + + client: Final = get_async_httpx_client(_LLM_PROVIDER) + device_code_url: Final = os.getenv("GITHUB_COPILOT_DEVICE_CODE_URL", DEFAULT_GITHUB_DEVICE_CODE_URL) + client_id: Final = os.getenv("GITHUB_COPILOT_CLIENT_ID", DEFAULT_GITHUB_CLIENT_ID) + try: + response: Final = await client.post( + device_code_url, + headers=github_api_headers(), + json={"client_id": client_id, "scope": _DEVICE_FLOW_SCOPE}, + ) + except httpx.HTTPStatusError as e: + raise ServiceUnavailableError( + message=f"GitHub device flow start failed with status {e.response.status_code}", + llm_provider=_LLM_PROVIDER, + model="", + ) from e + except Exception as e: + raise APIConnectionError( + message="GitHub device flow start request failed", + llm_provider=_LLM_PROVIDER, + model="", + ) from e + try: + payload: Final = _DEVICE_CODE.validate_python(response.json()) + except Exception as e: + raise ServiceUnavailableError( + message="GitHub device flow start returned an invalid response", + llm_provider=_LLM_PROVIDER, + model="", + ) from e + return GithubCopilotDeviceFlowStart( + device_code=payload["device_code"], + user_code=payload["user_code"], + verification_uri=payload["verification_uri"], + expires_in=_to_int(payload["expires_in"]), + interval=_to_int(payload["interval"]), + ) + + +async def apoll_device_flow(device_code: str) -> GithubCopilotDeviceFlowPoll: + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + + client: Final = get_async_httpx_client(_LLM_PROVIDER) + access_token_url: Final = os.getenv("GITHUB_COPILOT_ACCESS_TOKEN_URL", DEFAULT_GITHUB_ACCESS_TOKEN_URL) + client_id: Final = os.getenv("GITHUB_COPILOT_CLIENT_ID", DEFAULT_GITHUB_CLIENT_ID) + try: + response: Final = await client.post( + access_token_url, + headers=github_api_headers(), + json={ + "client_id": client_id, + "device_code": device_code, + "grant_type": "urn:ietf:params:oauth:grant-type:device_code", + }, + ) + except httpx.HTTPStatusError as e: + return _poll_payload_from_response(e.response) + except Exception as e: + raise APIConnectionError( + message="GitHub device flow poll request failed", + llm_provider=_LLM_PROVIDER, + model="", + ) from e + return _poll_payload_from_response(response) + + +def _poll_payload_from_response(response: httpx.Response) -> GithubCopilotDeviceFlowPoll: + if not response.is_success: + raise ServiceUnavailableError( + message=f"GitHub device flow poll failed with status {response.status_code}", + llm_provider=_LLM_PROVIDER, + model="", + ) + try: + payload: Final = _ACCESS_TOKEN_POLL.validate_python(response.json()) + except Exception as e: + raise ServiceUnavailableError( + message="GitHub device flow poll returned an invalid response", + llm_provider=_LLM_PROVIDER, + model="", + ) from e + + access_token: Final = payload.get("access_token") + if isinstance(access_token, str) and access_token: + return GithubCopilotDeviceFlowPoll(status="connected", access_token=access_token) + + error: Final = payload.get("error") + raw_interval: Final = payload.get("interval") + interval: Final = _to_int(raw_interval) or None + match error: + case "authorization_pending": + return GithubCopilotDeviceFlowPoll(status="pending", interval=interval) + case "slow_down": + return GithubCopilotDeviceFlowPoll(status="slow_down", interval=interval) + case "expired_token": + return GithubCopilotDeviceFlowPoll(status="expired", interval=interval) + case "access_denied": + return GithubCopilotDeviceFlowPoll(status="denied", interval=interval) + case _: + raise ServiceUnavailableError( + message="GitHub device flow poll returned an unrecognized error", + llm_provider=_LLM_PROVIDER, + model="", + ) + + +async def afetch_github_login(github_token: str) -> str: + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + + client: Final = get_async_httpx_client(_LLM_PROVIDER) + try: + response: Final = await client.get( + "https://api.github.com/user", + headers=github_api_headers(github_token), + ) + except Exception as e: + raise APIConnectionError( + message="GitHub user lookup request failed", + llm_provider=_LLM_PROVIDER, + model="", + ) from e + if not response.is_success: + raise ServiceUnavailableError( + message=f"GitHub user lookup failed with status {response.status_code}", + llm_provider=_LLM_PROVIDER, + model="", + ) + try: + payload: Final = _GITHUB_USER.validate_python(response.json()) + except Exception as e: + raise ServiceUnavailableError( + message="GitHub user lookup returned an invalid response", + llm_provider=_LLM_PROVIDER, + model="", + ) from e + return payload["login"] + + +async def acheck_copilot_seat(user_id: str, github_token: str, credential_name: str) -> GithubCopilotUserSession: + return await aexchange_github_token( + user_id=user_id, + github_token=github_token, + credential_name=credential_name, + ) + + +def github_copilot_auth_mode(litellm_credential_name: object, auth_type_value: object) -> bool: + """Whether this call runs in per-user mode. A resolved credential's stored + ``credential_values`` are the only authority once a credential is named and + found; the request's own ``github_copilot_auth_type`` can never flip the + mode either way on a named credential.""" + if isinstance(litellm_credential_name, str) and litellm_credential_name: + import litellm + + credential: Final[object] = litellm.CredentialAccessor.find_credential(litellm_credential_name) + if credential is not None: + values: Final[object] = getattr(credential, "credential_values", None) + return isinstance(values, Mapping) and ( + values.get(GITHUB_COPILOT_AUTH_TYPE_KEY) == GITHUB_COPILOT_PER_USER_AUTH_TYPE + ) + return auth_type_value == GITHUB_COPILOT_PER_USER_AUTH_TYPE + if auth_type_value == GITHUB_COPILOT_PER_USER_AUTH_TYPE: + raise BadRequestError( + message=f"{GITHUB_COPILOT_AUTH_TYPE_KEY} {GITHUB_COPILOT_PER_USER_AUTH_TYPE} requires litellm_credential_name", + llm_provider=_LLM_PROVIDER, + model="", + ) + return False + + +def _connect_error(credential_name: str) -> CallerCredentialAuthenticationError: + return CallerCredentialAuthenticationError( + message=(f"Connect GitHub Copilot for credential '{credential_name}' in the LiteLLM UI (LLM Credentials)"), + llm_provider=_LLM_PROVIDER, + model="", + ) + + +def _caller_connection(kwargs: Mapping[str, object], credential_name: str) -> tuple[str, str]: + secret_fields: Final[object] = kwargs.get("secret_fields") + user_id: Final[object] = ( + secret_fields.get("user_provider_credentials_user_id") if isinstance(secret_fields, Mapping) else None + ) + credentials: Final[object] = ( + secret_fields.get("user_provider_credentials") if isinstance(secret_fields, Mapping) else None + ) + token: Final[object] = credentials.get(credential_name) if isinstance(credentials, Mapping) else None + if not isinstance(user_id, str) or not user_id or not isinstance(token, str) or not token: + raise _connect_error(credential_name) + return user_id, token + + +def _params_value(source: object, key: str) -> object: + if isinstance(source, Mapping): + return source.get(key) + return getattr(source, key, None) + + +def github_copilot_per_user_credential_name(litellm_params: object) -> str | None: + """The credential name iff these litellm_params select per-user mode.""" + credential_name: Final[object] = _params_value(litellm_params, "litellm_credential_name") + auth_type: Final[object] = _params_value(litellm_params, GITHUB_COPILOT_AUTH_TYPE_KEY) + if not github_copilot_auth_mode(credential_name, auth_type): + return None + return credential_name if isinstance(credential_name, str) else "" + + +def github_copilot_user_session_from(source: object) -> GithubCopilotUserSession | None: + if source is None: + return None + candidate: Final = ( + source.get(GITHUB_COPILOT_USER_SESSION_KWARG_KEY) + if isinstance(source, Mapping) + else getattr(source, GITHUB_COPILOT_USER_SESSION_KWARG_KEY, None) + ) + return candidate if isinstance(candidate, GithubCopilotUserSession) else None + + +def require_github_copilot_user_session(litellm_params: object) -> GithubCopilotUserSession | None: + """The attached session, or None for shared mode. Per-user params with no + attached session only reach here via paths that skipped the attach step, so + they get the same not-connected 401 instead of the shared device login.""" + session: Final = github_copilot_user_session_from(litellm_params) + if session is not None: + return session + credential_name: Final = github_copilot_per_user_credential_name(litellm_params) + if credential_name is not None: + raise _connect_error(credential_name) + return None + + +def is_github_copilot_per_user_request(kwargs: Mapping[str, object]) -> bool: + """True when this call carries a resolved per-user Copilot session, meaning it + must bypass the shared response cache (cache keys ignore caller identity). + The session kwarg is attached in ``utils.wrapper``/``wrapper_async`` after + ``load_credentials_from_list``, before the cache lookup runs.""" + return isinstance(kwargs.get(GITHUB_COPILOT_USER_SESSION_KWARG_KEY), GithubCopilotUserSession) + + +def attach_github_copilot_user_session( + kwargs: dict[str, object], # mutable-ok: kwargs is the request mutation channel +) -> None: + credential_name_obj: Final[object] = kwargs.get("litellm_credential_name") + auth_type_obj: Final[object] = kwargs.get(GITHUB_COPILOT_AUTH_TYPE_KEY) + if not github_copilot_auth_mode(credential_name_obj, auth_type_obj): + return + if isinstance(kwargs.get(GITHUB_COPILOT_USER_SESSION_KWARG_KEY), GithubCopilotUserSession): + return + credential_name: Final = credential_name_obj if isinstance(credential_name_obj, str) else "" + user_id, github_token = _caller_connection(kwargs, credential_name) + session: Final = exchange_github_token( + user_id=user_id, + github_token=github_token, + credential_name=credential_name, + ) + kwargs[GITHUB_COPILOT_USER_SESSION_KWARG_KEY] = session # rebind-ok: kwargs is the request mutation channel + + +async def aattach_github_copilot_user_session( + kwargs: dict[str, object], # mutable-ok: kwargs is the request mutation channel +) -> None: + credential_name_obj: Final[object] = kwargs.get("litellm_credential_name") + auth_type_obj: Final[object] = kwargs.get(GITHUB_COPILOT_AUTH_TYPE_KEY) + if not github_copilot_auth_mode(credential_name_obj, auth_type_obj): + return + if isinstance(kwargs.get(GITHUB_COPILOT_USER_SESSION_KWARG_KEY), GithubCopilotUserSession): + return + credential_name: Final = credential_name_obj if isinstance(credential_name_obj, str) else "" + user_id, github_token = _caller_connection(kwargs, credential_name) + session: Final = await aexchange_github_token( + user_id=user_id, + github_token=github_token, + credential_name=credential_name, + ) + kwargs[GITHUB_COPILOT_USER_SESSION_KWARG_KEY] = session # rebind-ok: kwargs is the request mutation channel diff --git a/litellm/llms/github_copilot/responses/transformation.py b/litellm/llms/github_copilot/responses/transformation.py index 7bdf253f28f1..e8d8163c326b 100644 --- a/litellm/llms/github_copilot/responses/transformation.py +++ b/litellm/llms/github_copilot/responses/transformation.py @@ -31,6 +31,7 @@ GetAPIKeyError, get_copilot_default_headers, ) +from ..per_user_auth import require_github_copilot_user_session if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -199,22 +200,25 @@ def validate_environment( - copilot-vision-request if vision content detected - User-provided extra_headers (merged with priority) """ + user_session: Final = require_github_copilot_user_session(litellm_params) # pyright: ignore[reportUnknownArgumentType] # litellm_params arrives as an untyped request dict + default_headers: Final = get_copilot_default_headers(user_session.token) if user_session is not None else None try: - # Get GitHub Copilot API key via OAuth - api_key: Final = self.authenticator.get_api_key() + if default_headers is None: + # Get GitHub Copilot API key via OAuth + api_key: Final = self.authenticator.get_api_key() - if not api_key: - raise AuthenticationError( - model=model, - llm_provider="github_copilot", - message="GitHub Copilot API key is required. Please authenticate via OAuth Device Flow.", - ) + if not api_key: + raise AuthenticationError( + model=model, + llm_provider="github_copilot", + message="GitHub Copilot API key is required. Please authenticate via OAuth Device Flow.", + ) # Get default headers (from copilot-api configuration) - default_headers: Final = get_copilot_default_headers(api_key) + copilot_headers: Final = default_headers or get_copilot_default_headers(api_key) # Merge with existing headers (user's extra_headers take priority) - merged_headers: Final = {**default_headers, **headers} + merged_headers: Final = {**copilot_headers, **headers} # Analyze input to determine additional headers input_param: Final = self._get_input_from_params(litellm_params) @@ -249,9 +253,10 @@ def get_complete_url( """ Get the complete URL for GitHub Copilot Responses API endpoint. """ - # Use provided api_base or fall back to authenticator's base or default + user_session: Final = require_github_copilot_user_session(litellm_params) # pyright: ignore[reportUnknownArgumentType] # litellm_params arrives as an untyped request dict effective_api_base = ( - api_base + (user_session.api_base if user_session is not None else None) + or api_base or self.authenticator.get_api_base() or os.getenv("GITHUB_COPILOT_API_BASE") or DEFAULT_GITHUB_COPILOT_API_BASE diff --git a/litellm/main.py b/litellm/main.py index b9f171ba8492..d8913c3c3f37 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -642,10 +642,12 @@ async def acompletion( "enable_json_schema_validation": enable_json_schema_validation, } if custom_llm_provider is None: + _supplemental: Final = {k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS if k in kwargs} _, custom_llm_provider, _, _ = get_llm_provider( model=model, custom_llm_provider=custom_llm_provider, api_base=kwargs.get("api_base") or base_url, + litellm_params=(GenericLiteLLMParams.model_validate(_supplemental) if _supplemental else None), ) fallbacks = fallbacks or litellm.model_fallbacks @@ -2649,10 +2651,14 @@ def _complete_custom_openai( from litellm.llms.github_copilot.common_utils import ( get_copilot_default_headers, ) + from litellm.llms.github_copilot.per_user_auth import ( + require_github_copilot_user_session, + ) - copilot_auth: Final = Authenticator() - copilot_api_key: Final = copilot_auth.get_api_key() - copilot_headers: Final = get_copilot_default_headers(copilot_api_key) + user_session: Final = require_github_copilot_user_session(litellm_params) # pyright: ignore[reportUnknownArgumentType] # litellm_params arrives as an untyped request dict + copilot_headers: Final = get_copilot_default_headers( + user_session.token if user_session is not None else Authenticator().get_api_key() + ) if extra_headers: copilot_headers.update(extra_headers) extra_headers = copilot_headers @@ -5789,6 +5795,8 @@ def completion( *ANTHROPIC_WIF_KWARGS_KEYS, *OPENAI_WIF_KWARGS_KEYS, *OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS, + "github_copilot_auth_type", + "github_copilot_user_session", PROVIDER_AFFINITY_HEADER_KWARG_KEY, ) if key in kwargs @@ -6477,11 +6485,14 @@ def embedding( default_params: Final = [*openai_params, "aembedding", "extra_headers"] non_default_params: Final = filter_out_litellm_params(kwargs, excluding=default_params) + litellm_params_dict: Final = get_litellm_params(**kwargs) + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( model=model, custom_llm_provider=custom_llm_provider, api_base=api_base, api_key=api_key, + litellm_params=GenericLiteLLMParams(**kwargs), ) if dynamic_api_key is not None: @@ -6512,8 +6523,6 @@ def embedding( model_info=kwargs.get("model_info"), ) - litellm_params_dict: Final = get_litellm_params(**kwargs) - logging: Final[LiteLLMLoggingObj] = litellm_logging_obj logging.update_environment_variables( model=model, diff --git a/litellm/models/credentials.py b/litellm/models/credentials.py index b91ced275ff4..9bdd4199e195 100644 --- a/litellm/models/credentials.py +++ b/litellm/models/credentials.py @@ -6,6 +6,7 @@ """ from collections.abc import Mapping +from typing import Literal from pydantic import Field, model_validator @@ -43,3 +44,35 @@ class UpdateCredentialItem(LiteLLMBaseModel): credential_values: Mapping[str, object] | None = None model_id: str | None = None credential_values_to_delete: tuple[str, ...] | None = None + + +class UserProviderConnection(LiteLLMBaseModel): + credential_name: str + provider: str + connected: bool + github_login: str | None = None + connected_at: str | None = None + + +class UserProviderConnectionsResponse(LiteLLMBaseModel): + connections: list[UserProviderConnection] # mutable-ok: pydantic response model field + + +class UserConnectionStartResponse(LiteLLMBaseModel): + user_code: str + verification_uri: str + expires_in: int + interval: int + + +UserConnectionPollStatus = Literal["pending", "slow_down", "expired", "denied", "connected", "no_copilot_seat"] + + +class UserConnectionPollResponse(LiteLLMBaseModel): + status: UserConnectionPollStatus + interval: int | None = None + github_login: str | None = None + + +class UserConnectionDeleteResponse(LiteLLMBaseModel): + status: str diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 8c7965a54d3f..02d5179f8322 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -429,6 +429,12 @@ def _build_banned_observability_params() -> frozenset[str]: # SDK-only field; also rejected outright in is_request_body_safe. "model_list", "vertex_ai_credentials", + # Per-user GitHub Copilot connection slots: the mode is decided by the + # stored credential's values and the caller's connection by the proxy's own + # secret_fields, so a body-supplied value could only spoof either. + "github_copilot_auth_type", + "user_provider_credentials", + "github_copilot_user_session", # Observability credentials, hosts, and project identifiers: derived # from the canonical ``_supported_callback_params`` allowlist so new # integrations are covered automatically. Sorted for stable iteration diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 2c1643c7c423..0ab1fb572c32 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -70,6 +70,16 @@ _AUTH_ENFORCED_PASS_THROUGH_ROUTE_GROUPS: Final = frozenset(("openai_routes", "llm_api_routes")) +# Method-scoped because the generic credential CRUD handlers share these paths: +# the route check cannot admit PATCH/DELETE on a name like "user_connections" +# or "*/user_connection*" without also opening the admin CRUD handlers. +_INTERNAL_USER_CREDENTIAL_CONNECTION_ROUTES: Final = ( + ("GET", "/credentials/user_connections"), + ("POST", "/credentials/{credential_name:path}/user_connection/start"), + ("POST", "/credentials/{credential_name:path}/user_connection/poll"), + ("DELETE", "/credentials/{credential_name:path}/user_connection"), +) + class RouteChecks: @staticmethod @@ -331,7 +341,10 @@ def non_proxy_admin_allowed_routes_check( ) elif ( _user_role == LitellmUserRoles.INTERNAL_USER.value - and RouteChecks.check_route_access(route=route, allowed_routes=LiteLLMRoutes.internal_user_routes.value) + and ( + RouteChecks.check_route_access(route=route, allowed_routes=LiteLLMRoutes.internal_user_routes.value) + or RouteChecks.is_internal_user_credential_connection_route(route=route, request=request) + ) or user_is_org_admin(request_data=request_data, user_object=user_obj) and RouteChecks.check_route_access(route=route, allowed_routes=LiteLLMRoutes.org_admin_allowed_routes.value) or _user_role == LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value @@ -673,6 +686,16 @@ def get_request_method(request: Request | None) -> str | None: _get_request_method = get_request_method + @staticmethod + def is_internal_user_credential_connection_route(route: str, request: Request | None) -> bool: + method: Final = RouteChecks.get_request_method(request) + if method is None: + return False + return any( + method == expected_method and RouteChecks.check_route_access(route=route, allowed_routes=(pattern,)) + for expected_method, pattern in _INTERNAL_USER_CREDENTIAL_CONNECTION_ROUTES + ) + @staticmethod def is_auth_enforced_pass_through_route(route: str, method: str | None = None) -> bool: """ diff --git a/litellm/proxy/credential_endpoints/endpoints.py b/litellm/proxy/credential_endpoints/endpoints.py index 33ebef3b3c62..5128b15c19d1 100644 --- a/litellm/proxy/credential_endpoints/endpoints.py +++ b/litellm/proxy/credential_endpoints/endpoints.py @@ -2,6 +2,7 @@ CRUD endpoints for storing reusable credentials. """ +import json from collections.abc import Mapping from typing import ( Annotated, @@ -13,7 +14,13 @@ from pydantic import TypeAdapter import litellm +from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger +from litellm.constants import ( + GITHUB_COPILOT_AUTH_TYPE_KEY, + GITHUB_COPILOT_DEVICE_FLOW_CACHE_PREFIX, + GITHUB_COPILOT_PER_USER_AUTH_TYPE, +) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.litellm_logging import get_masked_values from litellm.llms.anthropic.wif import ( @@ -22,7 +29,14 @@ UnbuildableIdentitySource, anthropic_internal_issuer_jwks, ) -from litellm.models.credentials import UpdateCredentialItem +from litellm.models.credentials import ( + UpdateCredentialItem, + UserConnectionDeleteResponse, + UserConnectionPollResponse, + UserConnectionStartResponse, + UserProviderConnection, + UserProviderConnectionsResponse, +) from litellm.proxy._types import ( CommonProxyErrors, LitellmUserRoles, @@ -37,15 +51,30 @@ named_credential_wif_fields, stored_credential_provider, ) -from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper +from litellm.proxy.common_utils.encrypt_decrypt_utils import ( + decrypt_value_helper, + encrypt_value_helper, +) from litellm.proxy.utils import handle_exception_on_proxy, jsonify_object from litellm.repositories.base_repository import is_unique_violation from litellm.repositories.credentials_repository import CredentialsRepository from litellm.types.router import server_owned_wif_fields_named from litellm.types.utils import CreateCredentialItem, CredentialItem +from .user_provider_credentials import ( + GithubCopilotUserConnectionPayload, + decode_user_provider_credential, + delete_user_provider_credential, + delete_user_provider_credentials_for_credential, + invalidate_user_provider_credential_cache, + list_user_provider_credentials, + list_user_provider_credentials_for_credential, + upsert_user_provider_credential, +) + router: Final = APIRouter() _CREDENTIAL_DICT_ADAPTER: Final = TypeAdapter(dict[str, object]) +_GITHUB_COPILOT_PROVIDER: Final = "github_copilot" def _reject_non_admin_wif_fields( @@ -418,6 +447,268 @@ async def get_credential_by_model( raise handle_exception_on_proxy(e) +def _authenticated_user_id(user_api_key_dict: UserAPIKeyAuth) -> str: + user_id: Final = user_api_key_dict.user_id + if not isinstance(user_id, str) or not user_id: + raise HTTPException(status_code=401, detail="An authenticated user is required") + return user_id + + +def _per_user_credential_or_404(credential_name: str) -> CredentialItem: + credential: Final = CredentialAccessor.find_credential(credential_name) + values: Final = credential.credential_values if credential is not None else None + if ( + credential is None + or not isinstance(values, Mapping) + or values.get(GITHUB_COPILOT_AUTH_TYPE_KEY) != GITHUB_COPILOT_PER_USER_AUTH_TYPE + ): + raise HTTPException(status_code=404, detail="Credential not found") + return credential + + +def _device_flow_cache_key(user_id: str, credential_name: str) -> str: + pair: Final = json.dumps([user_id, credential_name], separators=(",", ":")) + return f"{GITHUB_COPILOT_DEVICE_FLOW_CACHE_PREFIX}:{pair}" + + +def _per_user_credential_names() -> tuple[str, ...]: + return tuple( + credential.credential_name + for credential in litellm.credential_list + if isinstance(credential.credential_values, Mapping) + and credential.credential_values.get(GITHUB_COPILOT_AUTH_TYPE_KEY) == GITHUB_COPILOT_PER_USER_AUTH_TYPE + ) + + +@router.get( + "/credentials/user_connections", + dependencies=[Depends(user_api_key_auth)], + tags=["credential management"], + response_model=UserProviderConnectionsResponse, +) +async def list_user_connections( + request: Request, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI resolves the dependency from the default +): + """List the calling user's per-user provider connections.""" + from litellm.proxy.proxy_server import prisma_client + + try: + user_id: Final = _authenticated_user_id(user_api_key_dict) + if prisma_client is None: + raise HTTPException( + status_code=500, + detail={"error": CommonProxyErrors.db_not_connected_error.value}, + ) + rows: Final = await list_user_provider_credentials(prisma_client, user_id) + github_logins: Final[dict[str, str]] = {} # mutable-ok: accumulates one entry per credential row + connected_at: Final[dict[str, str]] = {} # mutable-ok: accumulates one entry per credential row + for row in rows: + if row.provider != _GITHUB_COPILOT_PROVIDER: + continue + payload = decode_user_provider_credential(row.credential_b64) + if payload is not None: + github_logins[row.credential_name] = payload.github_login + updated = getattr(row, "updated_at", None) + if updated is not None: + connected_at[row.credential_name] = updated.isoformat() + return UserProviderConnectionsResponse( + connections=[ + UserProviderConnection( + credential_name=name, + provider=_GITHUB_COPILOT_PROVIDER, + connected=name in github_logins, + github_login=github_logins.get(name), + connected_at=connected_at.get(name), + ) + for name in _per_user_credential_names() + ] + ) + except Exception as e: # noqa: BLE001 # endpoint boundary: every failure becomes the proxy error contract + raise handle_exception_on_proxy(e) + + +@router.post( + "/credentials/{credential_name:path}/user_connection/start", + dependencies=[Depends(user_api_key_auth)], + tags=["credential management"], + response_model=UserConnectionStartResponse, +) +@with_service_target("user_provider_connections") +async def start_user_connection( + request: Request, + fastapi_response: Response, + credential_name: str = Path(..., description="The credential name, percent-decoded"), + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI resolves the dependency from the default +): + """Begin a GitHub device flow for the calling user's connection to a per-user credential.""" + from litellm.llms.github_copilot.per_user_auth import astart_device_flow + from litellm.proxy.proxy_server import prisma_client, user_api_key_cache + + try: + user_id: Final = _authenticated_user_id(user_api_key_dict) + _per_user_credential_or_404(credential_name) + if prisma_client is None: + raise HTTPException( + status_code=500, + detail={"error": CommonProxyErrors.db_not_connected_error.value}, + ) + flow: Final = await astart_device_flow() + await user_api_key_cache.async_set_cache( + _device_flow_cache_key(user_id, credential_name), + encrypt_value_helper(flow.device_code), + ttl=flow.expires_in, + ) + return UserConnectionStartResponse( + user_code=flow.user_code, + verification_uri=flow.verification_uri, + expires_in=flow.expires_in, + interval=flow.interval, + ) + except Exception as e: # noqa: BLE001 # endpoint boundary: every failure becomes the proxy error contract + raise handle_exception_on_proxy(e) + + +@router.post( + "/credentials/{credential_name:path}/user_connection/poll", + dependencies=[Depends(user_api_key_auth)], + tags=["credential management"], + response_model=UserConnectionPollResponse, +) +@with_service_target("user_provider_connections") +async def poll_user_connection( + request: Request, + fastapi_response: Response, + credential_name: str = Path(..., description="The credential name, percent-decoded"), + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI resolves the dependency from the default +): + """Poll the device flow once and persist the connection on completion.""" + from litellm.llms.github_copilot.per_user_auth import ( + acheck_copilot_seat, + afetch_github_login, + apoll_device_flow, + ) + from litellm.proxy.proxy_server import prisma_client, user_api_key_cache + + try: + user_id: Final = _authenticated_user_id(user_api_key_dict) + _per_user_credential_or_404(credential_name) + if prisma_client is None: + raise HTTPException( + status_code=500, + detail={"error": CommonProxyErrors.db_not_connected_error.value}, + ) + cache_key: Final = _device_flow_cache_key(user_id, credential_name) + encrypted_device_code: Final[object] = await user_api_key_cache.async_get_cache(cache_key) + device_code: Final = ( + decrypt_value_helper(str(encrypted_device_code), key="device_code") + if isinstance(encrypted_device_code, str) and encrypted_device_code + else None + ) + if not device_code: + return UserConnectionPollResponse(status="expired") + + poll: Final = await apoll_device_flow(device_code) + if poll.status in ("expired", "denied"): + await user_api_key_cache.async_delete_cache(cache_key) + return UserConnectionPollResponse(status=poll.status, interval=poll.interval) + if poll.status in ("pending", "slow_down"): + return UserConnectionPollResponse(status=poll.status, interval=poll.interval) + + github_token: Final = poll.access_token + if not github_token: + await user_api_key_cache.async_delete_cache(cache_key) + raise HTTPException(status_code=502, detail="GitHub device flow completed without a token") + try: + await acheck_copilot_seat(user_id=user_id, github_token=github_token, credential_name=credential_name) + except litellm.CallerCredentialAuthenticationError: + await user_api_key_cache.async_delete_cache(cache_key) + return UserConnectionPollResponse(status="no_copilot_seat") + github_login: Final = await afetch_github_login(github_token) + await upsert_user_provider_credential( + prisma_client, + user_id, + credential_name, + _GITHUB_COPILOT_PROVIDER, + GithubCopilotUserConnectionPayload(access_token=github_token, github_login=github_login), + ) + await user_api_key_cache.async_delete_cache(cache_key) + await invalidate_user_provider_credential_cache(user_api_key_cache, user_id, credential_name) + return UserConnectionPollResponse(status="connected", github_login=github_login) + except litellm.CallerCredentialRateLimitError as e: + raise HTTPException(status_code=429, detail=str(e)) + except Exception as e: # noqa: BLE001 # every remaining failure maps to the proxy error shape + raise handle_exception_on_proxy(e) + + +@router.delete( + "/credentials/{credential_name:path}/user_connection", + dependencies=[Depends(user_api_key_auth)], + tags=["credential management"], + response_model=UserConnectionDeleteResponse, +) +@with_service_target("user_provider_connections") +async def delete_user_connection( + request: Request, + fastapi_response: Response, + credential_name: str = Path(..., description="The credential name, percent-decoded"), + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI resolves the dependency from the default +): + """Disconnect the calling user's stored GitHub token for a per-user credential. Idempotent.""" + from litellm.llms.github_copilot.per_user_auth import evict_copilot_user_session + from litellm.proxy.proxy_server import prisma_client, user_api_key_cache + + try: + user_id: Final = _authenticated_user_id(user_api_key_dict) + _per_user_credential_or_404(credential_name) + if prisma_client is None: + raise HTTPException( + status_code=500, + detail={"error": CommonProxyErrors.db_not_connected_error.value}, + ) + prior: Final = await delete_user_provider_credential(prisma_client, user_id, credential_name) + await invalidate_user_provider_credential_cache(user_api_key_cache, user_id, credential_name) + await user_api_key_cache.async_delete_cache(_device_flow_cache_key(user_id, credential_name)) + if prior is not None: + evict_copilot_user_session(user_id, prior.access_token) + return UserConnectionDeleteResponse(status="disconnected") + except Exception as e: # noqa: BLE001 # endpoint boundary: every failure becomes the proxy error contract + raise handle_exception_on_proxy(e) + + +@with_service_target("user_provider_connections") +async def _purge_user_connections_for_credential(credential_name: str) -> None: + """Drop every user connection under a per-user credential and clear caches. + + Shared cleanup for DELETE (credential removed) and PATCH (renamed or no + longer per-user).""" + from litellm.llms.github_copilot.per_user_auth import evict_copilot_user_session + from litellm.proxy.proxy_server import prisma_client, user_api_key_cache + + if prisma_client is None: + return + try: + prior_rows: Final = await list_user_provider_credentials_for_credential(prisma_client, credential_name) + user_ids: Final = await delete_user_provider_credentials_for_credential(prisma_client, credential_name) + except Exception: # noqa: BLE001 # best-effort purge must not break credential deletion + verbose_proxy_logger.exception( + "_purge_user_connections_for_credential: failed to drop user connections for %s", credential_name + ) + return + access_tokens: Final[dict[str, str]] = {} # mutable-ok: accumulates one entry per deleted row + for row in prior_rows: + decoded = decode_user_provider_credential(row.credential_b64) + if decoded is not None: + access_tokens[row.user_id] = decoded.access_token + for user_id in user_ids: + await invalidate_user_provider_credential_cache(user_api_key_cache, user_id, credential_name) + await user_api_key_cache.async_delete_cache(_device_flow_cache_key(user_id, credential_name)) + token = access_tokens.get(user_id) + if token: + evict_copilot_user_session(user_id, token) + + @router.delete( "/credentials/{credential_name:path}", dependencies=[Depends(user_api_key_auth)], @@ -452,6 +743,7 @@ async def delete_credential( ## DELETE FROM LITELLM ## litellm.credential_list = [cred for cred in litellm.credential_list if cred.credential_name != credential_name] + await _purge_user_connections_for_credential(credential_name) return {"success": True, "message": "Credential deleted successfully"} except Exception as e: raise handle_exception_on_proxy(e) @@ -558,6 +850,14 @@ async def update_credential( # Sync in-memory credential_list (skip if not in memory - e.g., proxy restarted) _sync_in_memory_credential(patch, credential_name, merged_credential.credential_name) + merged_values: Final = merged_credential.credential_values + still_per_user: Final = ( + isinstance(merged_values, Mapping) + and merged_values.get(GITHUB_COPILOT_AUTH_TYPE_KEY) == GITHUB_COPILOT_PER_USER_AUTH_TYPE + ) + if credential.credential_name != credential_name or not still_per_user: + await _purge_user_connections_for_credential(credential_name) + return {"success": True, "message": "Credential updated successfully"} except Exception as e: raise handle_exception_on_proxy(e) diff --git a/litellm/proxy/credential_endpoints/user_provider_credentials.py b/litellm/proxy/credential_endpoints/user_provider_credentials.py new file mode 100644 index 000000000000..100dd09db8ea --- /dev/null +++ b/litellm/proxy/credential_endpoints/user_provider_credentials.py @@ -0,0 +1,219 @@ +"""DB and cache helpers for per-user provider connections. + +A row in ``LiteLLM_UserProviderCredentials`` binds one LiteLLM user to one +admin-named LLM credential (e.g. the user's own GitHub token behind a +``github_copilot`` per-user credential). ``credential_b64`` always stores the +payload encrypted; the request-time cache carries only ciphertext plus a +negative marker so an unconnected user does not hit the DB per request. +""" + +import json +from collections.abc import Mapping, Sequence +from typing import TYPE_CHECKING, Final + +from pydantic import BaseModel, ConfigDict, Field + +from litellm._internal_context import with_service_target +from litellm._logging import verbose_proxy_logger +from litellm.caching.dual_cache import DualCache +from litellm.constants import ( + GITHUB_COPILOT_USER_CREDENTIAL_CACHE_TTL_SECONDS, + USER_PROVIDER_CREDENTIAL_CACHE_PREFIX, + USER_PROVIDER_CREDENTIAL_NOT_CONNECTED, +) +from litellm.proxy.common_utils.encrypt_decrypt_utils import ( + decrypt_value_helper, + encrypt_value_helper, +) +from litellm.repositories.chunked_in import find_many_in +from litellm.repositories.prisma_protocols import TableActions +from litellm.repositories.table_repositories import PrismaTableRepository + +if TYPE_CHECKING: + from prisma import models as prisma_models + + from litellm.proxy.utils import PrismaClient + +_NOT_CONNECTED: Final = USER_PROVIDER_CREDENTIAL_NOT_CONNECTED + + +class GithubCopilotUserConnectionPayload(BaseModel): + model_config = ConfigDict(extra="ignore") + + access_token: str = Field(repr=False) + github_login: str + + +class _UserProviderCredentialsRepository(PrismaTableRepository["prisma_models.LiteLLM_UserProviderCredentials"]): + table_name = "litellm_userprovidercredentials" + + +def _table( + prisma_client: "PrismaClient", +) -> TableActions["prisma_models.LiteLLM_UserProviderCredentials"]: + return _UserProviderCredentialsRepository(prisma_client).table + + +def _cache_key(user_id: str, credential_name: str) -> str: + pair: Final = json.dumps([user_id, credential_name], separators=(",", ":")) + return f"{USER_PROVIDER_CREDENTIAL_CACHE_PREFIX}:{pair}" + + +def _encode(payload: GithubCopilotUserConnectionPayload) -> str: + return encrypt_value_helper(payload.model_dump_json()) + + +def decode_user_provider_credential(stored: str) -> GithubCopilotUserConnectionPayload | None: + decrypted: Final = decrypt_value_helper( + value=stored, + key="user_provider_credential", + exception_type="debug", + return_original_value=False, + ) + if decrypted is None: + return None + try: + return GithubCopilotUserConnectionPayload.model_validate_json(decrypted) + except Exception: + return None + + +async def upsert_user_provider_credential( + prisma_client: "PrismaClient", + user_id: str, + credential_name: str, + provider: str, + payload: GithubCopilotUserConnectionPayload, +) -> None: + await _table(prisma_client).upsert( + where={"user_id_credential_name": {"user_id": user_id, "credential_name": credential_name}}, + data={ + "create": { + "user_id": user_id, + "credential_name": credential_name, + "provider": provider, + "credential_b64": _encode(payload), + }, + "update": {"credential_b64": _encode(payload), "provider": provider}, + }, + ) + + +async def get_user_provider_credential( + prisma_client: "PrismaClient", + user_id: str, + credential_name: str, +) -> GithubCopilotUserConnectionPayload | None: + row: Final = await _table(prisma_client).find_unique( + where={"user_id_credential_name": {"user_id": user_id, "credential_name": credential_name}} + ) + if row is None: + return None + return decode_user_provider_credential(row.credential_b64) + + +async def delete_user_provider_credential( + prisma_client: "PrismaClient", + user_id: str, + credential_name: str, +) -> GithubCopilotUserConnectionPayload | None: + existing: Final = await get_user_provider_credential(prisma_client, user_id, credential_name) + await _table(prisma_client).delete_many(where={"user_id": user_id, "credential_name": credential_name}) + return existing + + +async def delete_user_provider_credentials_for_credential( + prisma_client: "PrismaClient", + credential_name: str, +) -> Sequence[str]: + """Delete every user connection under ``credential_name``; returns the + affected user_ids so callers can invalidate their cache entries.""" + rows: Final = await _table(prisma_client).find_many(where={"credential_name": credential_name}) + user_ids: Final = tuple(dict.fromkeys(row.user_id for row in rows)) + await _table(prisma_client).delete_many(where={"credential_name": credential_name}) + return user_ids + + +async def list_user_provider_credentials( + prisma_client: "PrismaClient", + user_id: str, +) -> Sequence["prisma_models.LiteLLM_UserProviderCredentials"]: + return await _table(prisma_client).find_many(where={"user_id": user_id}) + + +async def list_user_provider_credentials_for_credential( + prisma_client: "PrismaClient", + credential_name: str, +) -> Sequence["prisma_models.LiteLLM_UserProviderCredentials"]: + return await _table(prisma_client).find_many(where={"credential_name": credential_name}) + + +@with_service_target("user_provider_connections") +async def invalidate_user_provider_credential_cache( + cache: DualCache, + user_id: str, + credential_name: str, +) -> None: + await cache.async_delete_cache(_cache_key(user_id, credential_name)) + + +@with_service_target("user_provider_connections") +async def aget_user_provider_tokens( + prisma_client: "PrismaClient", + cache: DualCache, + user_id: str, + credential_names: Sequence[str], +) -> Mapping[str, str]: + """Map each per-user credential name to the caller's stored GitHub token. + + Cached values are the stored ciphertext or the ``_NOT_CONNECTED`` marker; + plaintext tokens only ever live in the returned dict.""" + names: Final = tuple(dict.fromkeys(credential_names)) + if not names: + return {} + cached: dict[str, str] = {} # mutable-ok: accumulates hits and DB reads + misses: list[str] = [] # mutable-ok: accumulates cache misses + for name in names: + value = await cache.async_get_cache(_cache_key(user_id, name)) + if value == _NOT_CONNECTED: + continue + if isinstance(value, str) and value: + cached[name] = value + else: + misses.append(name) + if not misses: + return { + name: token + for name, ciphertext in cached.items() + if (token := _token_from_ciphertext(ciphertext)) is not None + } + + try: + rows: Final = await find_many_in( + _table(prisma_client), + "credential_name", + misses, + where={"user_id": user_id}, + ) + except Exception: + verbose_proxy_logger.exception("aget_user_provider_tokens: DB read failed for user_id=%s", user_id) + return {} + found: dict[str, str] = {} # mutable-ok: accumulates rows + for row in rows: + found[row.credential_name] = row.credential_b64 + for name in misses: + await cache.async_set_cache( + _cache_key(user_id, name), + found.get(name, _NOT_CONNECTED), + ttl=GITHUB_COPILOT_USER_CREDENTIAL_CACHE_TTL_SECONDS, + ) + if name in found: + cached[name] = found[name] + return { + name: token for name, ciphertext in cached.items() if (token := _token_from_ciphertext(ciphertext)) is not None + } + + +def _token_from_ciphertext(ciphertext: str) -> str | None: + payload: Final = decode_user_provider_credential(ciphertext) + return payload.access_token if payload is not None else None diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 03a93016177e..de4c1120492c 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -267,6 +267,7 @@ def parse_cache_control(cache_control): "proxy_server_request", "standard_logging_object", "secret_fields", + "github_copilot_user_session", "mock_response", "mock_tool_calls", "disable_global_guardrails", @@ -2217,6 +2218,9 @@ async def add_litellm_data_to_request( data=data, headers=_headers, user_api_key_dict=user_api_key_dict ) + # Header mappings can overwrite user_id below; the key/JWT identity is the only + # one allowed to load per-user provider credentials. + authenticated_user_id: Final = user_api_key_dict.user_id user_api_key_dict = LiteLLMProxyRequestSetup.add_internal_user_from_user_mapping( general_settings, user_api_key_dict, _headers ) @@ -2595,6 +2599,12 @@ async def add_litellm_data_to_request( llm_router=llm_router, ) + await _resolve_user_provider_credentials_for_request( + data=data, + authenticated_user_id=authenticated_user_id, + llm_router=llm_router, + ) + ## ENFORCED PARAMS CHECK # loop through each enforced param # example enforced_params ['user', 'metadata', 'metadata.generation_name'] @@ -2620,6 +2630,129 @@ async def add_litellm_data_to_request( return data +def _fallback_edges( + llm_router: litellm.Router, + model_group: str, +) -> frozenset[str]: + targets: Final[set[str]] = set() # mutable-ok: accumulates fallback targets + for fallback_list in ( + llm_router.fallbacks, + llm_router.context_window_fallbacks, + llm_router.content_policy_fallbacks, + ): + for entry in fallback_list or (): + if not isinstance(entry, dict): + continue + for key in (model_group, "*"): + raw = entry.get(key) + values = raw if isinstance(raw, list) else ([raw] if raw is not None else []) + targets.update(value for value in values if isinstance(value, str)) + return frozenset(targets) + + +def _fallback_target_groups( + llm_router: litellm.Router, + model_group: str, +) -> frozenset[str]: + """Transitive closure over the router's fallback graph: a chain like + A -> B -> C must still surface C's per-user credential for a request to A.""" + seen: Final[set[str]] = set() # mutable-ok: BFS visited set + frontier: Final[list[str]] = [model_group] # mutable-ok: BFS work list + while frontier: + group = frontier.pop() + for target in _fallback_edges(llm_router, group): + if target not in seen and target != model_group: + seen.add(target) + frontier.append(target) + return frozenset(seen) + + +def _per_user_credential_names_for_groups( + llm_router: litellm.Router, + model_groups: frozenset[str], +) -> tuple[str, ...]: + from litellm.constants import ( + GITHUB_COPILOT_AUTH_TYPE_KEY, + GITHUB_COPILOT_PER_USER_AUTH_TYPE, + ) + + names: Final[list[str]] = [] # mutable-ok: accumulates one name per per-user deployment + for group in model_groups: + for deployment in llm_router.get_model_list(model_name=group) or (): + litellm_params = deployment.get("litellm_params") + credential_name_obj: object = ( + litellm_params.get("litellm_credential_name") + if isinstance(litellm_params, Mapping) + else getattr(litellm_params, "litellm_credential_name", None) + ) + if not isinstance(credential_name_obj, str) or not credential_name_obj or credential_name_obj in names: + continue + credential = CredentialAccessor.find_credential(credential_name_obj) + if credential is None: + continue + values = credential.credential_values + if ( + isinstance(values, Mapping) + and values.get(GITHUB_COPILOT_AUTH_TYPE_KEY) == GITHUB_COPILOT_PER_USER_AUTH_TYPE + ): + names.append(credential_name_obj) + return tuple(names) + + +async def _resolve_user_provider_credentials_for_request( + data: dict[str, object], # mutable-ok: writes resolved credentials into the nested secret_fields dict + authenticated_user_id: str | None, + llm_router: litellm.Router | None, +) -> None: + """Resolve the calling user's per-user provider connections into secret_fields. + + Runs after model alias rewrites so ``data['model']`` is the final requested + model. ``authenticated_user_id`` is the identity established by the validated + API key or JWT, never a header-mapped value, so a caller cannot load another + user's stored token. Never raises for "not connected": shared deployments in + the same group still serve the request, so the provider-side helper is the + one that fails the call with a 401 when per-user mode applies.""" + user_id: Final = authenticated_user_id + model: Final = data.get("model") + if not isinstance(user_id, str) or not user_id or llm_router is None or not isinstance(model, str): + return + model_groups: Final = frozenset({model} | _fallback_target_groups(llm_router, model)) + credential_names: Final = _per_user_credential_names_for_groups(llm_router, model_groups) + if not credential_names: + return + if "litellm_credential_name" in data: + raise HTTPException( + status_code=400, + detail="litellm_credential_name cannot be set in the request body for a model that uses per-user GitHub OAuth", + ) + + from litellm.proxy.proxy_server import prisma_client, user_api_key_cache + from litellm.types.proxy.litellm_pre_call_utils import RedactedDict + + from .credential_endpoints.user_provider_credentials import aget_user_provider_tokens + + if prisma_client is None: + return + try: + tokens: Final = await aget_user_provider_tokens( + prisma_client=prisma_client, + cache=user_api_key_cache, + user_id=user_id, + credential_names=credential_names, + ) + except Exception: # noqa: BLE001 # credential lookup failures degrade to the shared deployment, never break the request + verbose_proxy_logger.exception( + "_resolve_user_provider_credentials_for_request: failed to load user provider credentials" + ) + return + + secret_fields: Final = data.get("secret_fields") + if not isinstance(secret_fields, dict): + return + secret_fields["user_provider_credentials"] = RedactedDict(dict(tokens)) + secret_fields["user_provider_credentials_user_id"] = user_id + + def _warn_stale_team_alias_once(warning_key: str, message: str, *args: str) -> None: if warning_key in _STALE_TEAM_ALIAS_WARNING_KEYS: return diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 038dfdeaca5f..92f2cedf7d6f 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -452,6 +452,20 @@ model LiteLLM_MCPUserCredentials { @@unique([user_id, server_id]) } +// Per-user provider connections (e.g. GitHub Copilot OAuth) keyed by credential name +model LiteLLM_UserProviderCredentials { + id String @id @default(uuid()) + user_id String + credential_name String + provider String + credential_b64 String + created_at DateTime @default(now()) @map("created_at") + updated_at DateTime @default(now()) @updatedAt @map("updated_at") + + @@unique([user_id, credential_name]) + @@index([credential_name]) +} + // Per-user environment variable values for MCP servers. // values_b64 is an encrypted JSON object: {VAR_NAME: "value", ...}. model LiteLLM_MCPUserEnvVars { diff --git a/litellm/responses/main.py b/litellm/responses/main.py index c4632f09723c..3bc50b36e297 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -627,7 +627,9 @@ async def aresponses( # get custom llm provider so we can use this for mapping exceptions if custom_llm_provider is None: _, custom_llm_provider, _, _ = litellm.get_llm_provider( - model=model, api_base=local_vars.get("base_url", None) + model=model, + api_base=local_vars.get("base_url", None), + litellm_params=GenericLiteLLMParams(**kwargs), ) # Update local_vars with detected provider (fixes #19782) local_vars["custom_llm_provider"] = custom_llm_provider @@ -1208,7 +1210,9 @@ def responses( if custom_llm_provider is None: _, custom_llm_provider, _, _ = litellm.get_llm_provider( - model=model, api_base=local_vars.get("base_url", None) + model=model, + api_base=local_vars.get("base_url", None), + litellm_params=GenericLiteLLMParams(**kwargs), ) local_vars["custom_llm_provider"] = custom_llm_provider @@ -2116,7 +2120,9 @@ async def acompact_responses( # get custom llm provider so we can use this for mapping exceptions if custom_llm_provider is None: _, custom_llm_provider, _, _ = litellm.get_llm_provider( - model=model, api_base=local_vars.get("base_url", None) + model=model, + api_base=local_vars.get("base_url", None), + litellm_params=GenericLiteLLMParams(**kwargs), ) # Update local_vars with detected provider (fixes #19782) local_vars["custom_llm_provider"] = custom_llm_provider @@ -2380,6 +2386,7 @@ async def _aresponses_websocket( model=model, api_base=api_base, api_key=api_key, + litellm_params=litellm_params, ) resolved_model: Final = _strip_responses_routing_prefix(provider_model) diff --git a/litellm/router.py b/litellm/router.py index d6730db87482..b78282d15a51 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8607,6 +8607,7 @@ def deployment_callback_on_failure( deployment=deployment_id, time_to_cooldown=_time_to_cooldown, requested_model_group=(get_litellm_metadata_from_kwargs(kwargs) or {}).get("model_group"), + request_kwargs=kwargs, ) # setting deployment_id in cooldown deployments return result @@ -9948,6 +9949,7 @@ def _add_deployment(self, deployment: Deployment) -> Deployment: model=deployment.litellm_params.model, custom_llm_provider=deployment.litellm_params.get("custom_llm_provider", None), api_base=deployment.litellm_params.api_base, + litellm_params=deployment.litellm_params, ) # done reading model["litellm_params"] # Check if provider is supported: either in enum or JSON-configured @@ -11357,6 +11359,7 @@ def _set_model_group_info(self, model_group: str, user_facing_model_group_name: litellm_model, llm_provider, _, _ = litellm.get_llm_provider( model=litellm_params.model, custom_llm_provider=litellm_params.custom_llm_provider, + litellm_params=litellm_params, ) except litellm.exceptions.BadRequestError as e: verbose_router_logger.error("litellm.router.py::get_model_group_info() - %s", e) @@ -12332,6 +12335,7 @@ def _deployment_accepts_param(deployment: DeploymentTypedDict, group: str, param model, custom_llm_provider, _, _ = litellm.get_llm_provider( model=deployment_params.get("model") or group, custom_llm_provider=deployment_params.get("custom_llm_provider"), + litellm_params=LiteLLM_Params.model_validate(deployment_params), ) supported: Final = litellm.get_supported_openai_params( model=model, diff --git a/litellm/router_utils/clientside_credential_handler.py b/litellm/router_utils/clientside_credential_handler.py index e65bd777ecab..05f64f813c2d 100644 --- a/litellm/router_utils/clientside_credential_handler.py +++ b/litellm/router_utils/clientside_credential_handler.py @@ -112,7 +112,26 @@ def get_dynamic_litellm_params(litellm_params: dict, request_kwargs: dict) -> di # admin's value in ``litellm_params`` and have it forwarded to the # redirected upstream. if "api_base" in request_kwargs or "base_url" in request_kwargs: + from litellm.llms.github_copilot.per_user_auth import github_copilot_auth_mode + + # A per-user OAuth credential must survive the clear: dropping + # litellm_credential_name / github_copilot_auth_type would flip the call to + # shared mode and send the admin's Copilot token to the redirected host. + # Per-user mode ignores the caller's api_base anyway (the session's + # validated host wins), so keeping the mode is fail-closed. + credential_name_value: Final[object] = litellm_params.get( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # litellm_params is the untyped request dict + "litellm_credential_name" + ) + auth_type: Final[object] = litellm_params.get( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # litellm_params is the untyped request dict + "github_copilot_auth_type" + ) + per_user_credential_locked: Final = github_copilot_auth_mode( + credential_name_value, # pyright: ignore[reportUnknownArgumentType] # value comes from the untyped request dict + auth_type, # pyright: ignore[reportUnknownArgumentType] # value comes from the untyped request dict + ) for field in _ADMIN_CONFIG_FIELDS_TO_CLEAR_ON_BASE_OVERRIDE: + if per_user_credential_locked and field in ("litellm_credential_name", "github_copilot_auth_type"): + continue litellm_params.pop(field, None) if field in request_kwargs: litellm_params[field] = request_kwargs[field] diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index a6af6073a4c9..85c785a942c5 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -266,12 +266,25 @@ def _is_cooldown_required( return True +def _is_per_user_provider_request(request_kwargs: Mapping[str, object]) -> bool: + from litellm.llms.github_copilot.per_user_auth import ( + github_copilot_user_session_from, + is_github_copilot_per_user_request, + ) + + return ( + is_github_copilot_per_user_request(request_kwargs) + or github_copilot_user_session_from(request_kwargs.get("litellm_params")) is not None + ) + + def _should_run_cooldown_logic( litellm_router_instance: LitellmRouter, deployment: str | None, exception_status: str | int, original_exception: Exception, time_to_cooldown: float | None = None, + request_kwargs: Mapping[str, object] | None = None, ) -> bool: """ Helper that decides if cooldown logic should be run @@ -290,6 +303,24 @@ def _should_run_cooldown_logic( ) return False + if isinstance( + original_exception, + ( + litellm.CallerCredentialAuthenticationError, + litellm.CallerCredentialRateLimitError, + ), + ): + verbose_router_logger.debug( + "Should Not Run Cooldown Logic: caller-credential errors are scoped to one user's connection" + ) + return False + + if request_kwargs is not None and _is_per_user_provider_request(request_kwargs): + verbose_router_logger.debug( + "Should Not Run Cooldown Logic: the failure came from one caller's per-user provider session" + ) + return False + ######################################################### # If time_to_cooldown is 0 or 0.0000000, don't run cooldown logic ######################################################### @@ -437,6 +468,7 @@ def set_cooldown_deployments( deployment: str | None = None, time_to_cooldown: float | None = None, requested_model_group: str | None = None, + request_kwargs: Mapping[str, object] | None = None, ) -> bool: """ Add a model to the list of models being cooled down for that minute, if it exceeds the allowed fails / minute @@ -458,6 +490,7 @@ def set_cooldown_deployments( exception_status=exception_status, original_exception=original_exception, time_to_cooldown=time_to_cooldown, + request_kwargs=request_kwargs, ) is False or deployment is None diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index e33897e3b636..8831491b72fe 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -147,6 +147,7 @@ def _trigger_cooldown_for_failed_deployment( original_exception=exception, deployment=deployment_id, time_to_cooldown=time_to_cooldown, + request_kwargs=kwargs, ) verbose_router_logger.debug("Triggered cooldown for fallback deployment %s", deployment_id) diff --git a/litellm/types/litellm_params.py b/litellm/types/litellm_params.py index e0cd3e56ddb8..3577e2a2dac1 100644 --- a/litellm/types/litellm_params.py +++ b/litellm/types/litellm_params.py @@ -87,6 +87,8 @@ class ProviderConnection: litellm_credential_name: str | None = None configurable_clientside_auth_params: "Sequence[str | ConfigurableClientsideParamsCustomAuth] | None" = None use_xai_oauth: bool | None = None + github_copilot_auth_type: str | None = None + github_copilot_user_session: object | None = None token_exchange_endpoint: str | None = None token_exchange_profile: str | None = None token_exchange_scope: str | None = None diff --git a/litellm/types/proxy/litellm_pre_call_utils.py b/litellm/types/proxy/litellm_pre_call_utils.py index e0d3f3dac668..0b980d383200 100644 --- a/litellm/types/proxy/litellm_pre_call_utils.py +++ b/litellm/types/proxy/litellm_pre_call_utils.py @@ -1,4 +1,6 @@ -from typing_extensions import TypedDict +from collections.abc import Mapping + +from typing_extensions import NotRequired, ReadOnly, TypedDict class RedactedDict(dict): @@ -22,3 +24,5 @@ class SecretFields(TypedDict): """ raw_headers: dict + user_provider_credentials: NotRequired[ReadOnly[Mapping[str, str]]] + user_provider_credentials_user_id: NotRequired[ReadOnly[str]] diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 1649f556b042..e9e7a2ef1fbf 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4058,8 +4058,12 @@ def pricing_override_fields(*sources: Mapping[str, object]) -> tuple[str, ...]: "token_exchange_profile", "token_exchange_scope", ) +github_copilot_oauth_litellm_params: Final = ("github_copilot_auth_type",) server_owned_wif_litellm_params: Final = ( - anthropic_wif_litellm_params + openai_wif_litellm_params + oauth_token_exchange_litellm_params + anthropic_wif_litellm_params + + openai_wif_litellm_params + + oauth_token_exchange_litellm_params + + github_copilot_oauth_litellm_params ) secret_bearing_wif_litellm_params: Final = tuple(sorted(WIF_SECRET_BEARING_KEYS)) diff --git a/litellm/utils.py b/litellm/utils.py index 8f55f742f4f6..3e1ef99bffbf 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1737,6 +1737,9 @@ def wrapper(*args, **kwargs): ## LOAD CREDENTIALS load_credentials_from_list(kwargs) + from litellm.llms.github_copilot.per_user_auth import attach_github_copilot_user_session + + attach_github_copilot_user_session(kwargs) # pyright: ignore[reportUnknownArgumentType] # kwargs is the untyped request dict kwargs["litellm_logging_obj"] = logging_obj LLMCachingHandler: Final = _get_cached_llm_caching_handler() _llm_caching_handler: Final[LLMCachingHandler] = LLMCachingHandler( @@ -2024,6 +2027,9 @@ async def wrapper_async(*args, **kwargs): kwargs["litellm_logging_obj"] = logging_obj ## LOAD CREDENTIALS load_credentials_from_list(kwargs) + from litellm.llms.github_copilot.per_user_auth import aattach_github_copilot_user_session + + await aattach_github_copilot_user_session(kwargs) # pyright: ignore[reportUnknownArgumentType] # kwargs is the untyped request dict logging_obj.llm_caching_handler = _llm_caching_handler # [OPTIONAL] CHECK BUDGET if litellm.max_budget: diff --git a/schema.prisma b/schema.prisma index 038dfdeaca5f..92f2cedf7d6f 100644 --- a/schema.prisma +++ b/schema.prisma @@ -452,6 +452,20 @@ model LiteLLM_MCPUserCredentials { @@unique([user_id, server_id]) } +// Per-user provider connections (e.g. GitHub Copilot OAuth) keyed by credential name +model LiteLLM_UserProviderCredentials { + id String @id @default(uuid()) + user_id String + credential_name String + provider String + credential_b64 String + created_at DateTime @default(now()) @map("created_at") + updated_at DateTime @default(now()) @updatedAt @map("updated_at") + + @@unique([user_id, credential_name]) + @@index([credential_name]) +} + // Per-user environment variable values for MCP servers. // values_b64 is an encrypted JSON object: {VAR_NAME: "value", ...}. model LiteLLM_MCPUserEnvVars { diff --git a/terraform/provider/tools/endpointaudit/coverage_allowlist.txt b/terraform/provider/tools/endpointaudit/coverage_allowlist.txt index 9305d2e109df..9430ada8468a 100644 --- a/terraform/provider/tools/endpointaudit/coverage_allowlist.txt +++ b/terraform/provider/tools/endpointaudit/coverage_allowlist.txt @@ -129,6 +129,13 @@ PATCH /team/{team_id} POST /team/model/add POST /team/model/delete +# Per-user OAuth connections managed by the calling user through the dashboard +# device flow; caller-scoped tokens, not Terraform-managed state +GET /credentials/user_connections +POST /credentials/{credential_name}/user_connection/start +POST /credentials/{credential_name}/user_connection/poll +DELETE /credentials/{credential_name}/user_connection + # Known gaps awaiting a resource or data source; remove the entry when it lands GET /credentials # known gap: plural credentials data source GET /cache/settings # known gap: cache settings resource diff --git a/tests/unit/llms/github_copilot/embedding/test_github_copilot_embedding_transformation.py b/tests/unit/llms/github_copilot/embedding/test_github_copilot_embedding_transformation.py index 83c0cb2fec28..f6576dcc6f4b 100644 --- a/tests/unit/llms/github_copilot/embedding/test_github_copilot_embedding_transformation.py +++ b/tests/unit/llms/github_copilot/embedding/test_github_copilot_embedding_transformation.py @@ -258,3 +258,34 @@ def test_transform_embedding_response_rejects_a_body_that_is_not_an_object(body: def test_transform_embedding_response_object_without_data_is_an_invalid_response_object(): with pytest.raises(Exception, match="Invalid response object"): _transform(httpx.Response(200, json={"model": "text-embedding-3-small"})) + + +def test_validate_environment_uses_per_user_session_and_skips_authenticator(): + """With a per-user session the Authorization header carries the caller's Copilot token + and the shared Authenticator is never consulted.""" + from litellm.llms.github_copilot.per_user_auth import GithubCopilotUserSession + + config = GithubCopilotEmbeddingConfig() + config.authenticator = MagicMock() + config.authenticator.get_api_key.side_effect = AssertionError("shared authenticator must not run") + + session = GithubCopilotUserSession(token="user-copilot-token", api_base="https://api.githubcopilot.com") + headers = config.validate_environment( + headers={}, model="github_copilot/text-embedding-3-small", messages=[], optional_params={}, + litellm_params={"github_copilot_user_session": session}, + ) + assert headers["Authorization"] == "Bearer user-copilot-token" + config.authenticator.get_api_key.assert_not_called() + + +def test_get_complete_url_prefers_per_user_session_api_base(): + from litellm.llms.github_copilot.per_user_auth import GithubCopilotUserSession + + config = GithubCopilotEmbeddingConfig() + config.authenticator = MagicMock() + session = GithubCopilotUserSession(token="t", api_base="https://tenant.githubcopilot.com") + url = config.get_complete_url( + api_base="https://attacker.example", api_key=None, model="m", optional_params={}, + litellm_params={"github_copilot_user_session": session}, + ) + assert url == "https://tenant.githubcopilot.com/embeddings" diff --git a/tests/unit/llms/github_copilot/messages/test_github_copilot_messages_transformation.py b/tests/unit/llms/github_copilot/messages/test_github_copilot_messages_transformation.py index 9d8673ed3d83..2b2a0b3d3872 100644 --- a/tests/unit/llms/github_copilot/messages/test_github_copilot_messages_transformation.py +++ b/tests/unit/llms/github_copilot/messages/test_github_copilot_messages_transformation.py @@ -1,5 +1,6 @@ from unittest.mock import MagicMock +import httpx import pytest @@ -322,3 +323,44 @@ def test_github_copilot_messages_config_probes_capabilities_under_copilot_namesp ``anthropic`` namespace and ignored the exact ``github_copilot/claude-*`` cost-map entries.""" assert GithubCopilotAnthropicMessagesConfig().custom_llm_provider == "github_copilot" + + +def test_validate_environment_uses_per_user_session_and_skips_authenticator(): + from litellm.llms.github_copilot.per_user_auth import GithubCopilotUserSession + + config = GithubCopilotAnthropicMessagesConfig() + config.authenticator = MagicMock() + config.authenticator.get_api_key.side_effect = AssertionError("shared authenticator must not run") + config.authenticator.get_api_base.side_effect = AssertionError("shared authenticator must not run") + + session = GithubCopilotUserSession(token="user-copilot-token", api_base="https://tenant.githubcopilot.com/") + headers, api_base = config.validate_anthropic_messages_environment( + headers={}, model="github_copilot/claude-sonnet-4.5", messages=[], optional_params={}, + litellm_params={"github_copilot_user_session": session}, + ) + assert headers["Authorization"] == "Bearer user-copilot-token" + assert api_base == "https://tenant.githubcopilot.com" + config.authenticator.get_api_key.assert_not_called() + + +def test_transform_response_carries_upstream_usage(): + config = GithubCopilotAnthropicMessagesConfig() + raw = httpx.Response( + 200, + json={ + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "github_copilot/claude-sonnet-4.5", + "content": [{"type": "text", "text": "hi"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 14, "output_tokens": 3}, + }, + ) + result = config.transform_anthropic_messages_response( + model="github_copilot/claude-sonnet-4.5", + raw_response=raw, + logging_obj=MagicMock(), + ) + assert result["usage"]["input_tokens"] == 14 + assert result["usage"]["output_tokens"] == 3 diff --git a/tests/unit/llms/github_copilot/responses/test_github_copilot_responses_transformation.py b/tests/unit/llms/github_copilot/responses/test_github_copilot_responses_transformation.py index 4d5990d8d18a..e24027504e8a 100644 --- a/tests/unit/llms/github_copilot/responses/test_github_copilot_responses_transformation.py +++ b/tests/unit/llms/github_copilot/responses/test_github_copilot_responses_transformation.py @@ -9,7 +9,7 @@ from unittest.mock import patch, MagicMock - +import httpx import pytest import litellm from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map @@ -753,3 +753,44 @@ def test_normalization_continues_after_a_terminal_event(self): }, ) assert event.item_id == "stable_rs_id" + + +def test_validate_environment_uses_per_user_session_and_skips_authenticator(): + from litellm.llms.github_copilot.per_user_auth import GithubCopilotUserSession + + config = GithubCopilotResponsesAPIConfig() + config.authenticator = MagicMock() + config.authenticator.get_api_key.side_effect = AssertionError("shared authenticator must not run") + + session = GithubCopilotUserSession(token="user-copilot-token", api_base="https://api.githubcopilot.com") + headers = config.validate_environment( + headers={}, model="github_copilot/gpt-5.1", + litellm_params={"github_copilot_user_session": session}, + ) + assert headers["Authorization"] == "Bearer user-copilot-token" + config.authenticator.get_api_key.assert_not_called() + + +def test_transform_response_carries_upstream_usage(): + config = GithubCopilotResponsesAPIConfig() + raw = httpx.Response( + 200, + json={ + "id": "resp_1", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "github_copilot/gpt-5.1", + "output": [{"type": "message", "id": "m1", "role": "assistant", "status": "completed", + "content": [{"type": "output_text", "text": "hi", "annotations": []}]}], + "usage": {"input_tokens": 9, "output_tokens": 4, "total_tokens": 13}, + }, + ) + result = config.transform_response_api_response( + model="github_copilot/gpt-5.1", + raw_response=raw, + logging_obj=MagicMock(), + ) + assert result.usage is not None + assert result.usage.input_tokens == 9 + assert result.usage.output_tokens == 4 diff --git a/tests/unit/llms/github_copilot/test_github_copilot_transformation.py b/tests/unit/llms/github_copilot/test_github_copilot_transformation.py index 1715d106e0ab..c775beeae246 100644 --- a/tests/unit/llms/github_copilot/test_github_copilot_transformation.py +++ b/tests/unit/llms/github_copilot/test_github_copilot_transformation.py @@ -918,3 +918,51 @@ def model_dump(self): assert result.choices[0].message.content == "Hi there" assert result.choices[0].finish_reason == "stop" mock_request.assert_called_once() + + +def test_validate_environment_uses_per_user_session_and_skips_authenticator(): + from litellm.llms.github_copilot.per_user_auth import GithubCopilotUserSession + + config = GithubCopilotConfig() + config.authenticator = MagicMock() + config.authenticator.get_api_key.side_effect = AssertionError("shared authenticator must not run") + + session = GithubCopilotUserSession(token="user-copilot-token", api_base="https://api.githubcopilot.com") + headers = config.validate_environment( + headers={}, model="github_copilot/gpt-4o", messages=[], optional_params={}, + litellm_params={"github_copilot_user_session": session}, + ) + assert headers["Authorization"] == "Bearer user-copilot-token" + config.authenticator.get_api_key.assert_not_called() + + +def test_transform_response_carries_upstream_usage(): + """Usage from the upstream Copilot payload must survive so TPM limits and spend work.""" + config = GithubCopilotConfig() + raw_response = httpx.Response( + 200, + json={ + "id": "chatcmpl-u", + "object": "chat.completion", + "created": 1, + "model": "github_copilot/gpt-4o", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 11, "completion_tokens": 7, "total_tokens": 18}, + }, + ) + result = config.transform_response( + model="github_copilot/gpt-4o", + raw_response=raw_response, + model_response=ModelResponse(), + logging_obj=MagicMock(model_call_details={}), + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + ) + assert result.usage.prompt_tokens == 11 + assert result.usage.completion_tokens == 7 + assert result.usage.total_tokens == 18 diff --git a/tests/unit/llms/github_copilot/test_per_user_auth.py b/tests/unit/llms/github_copilot/test_per_user_auth.py new file mode 100644 index 000000000000..7aaf7b8c376f --- /dev/null +++ b/tests/unit/llms/github_copilot/test_per_user_auth.py @@ -0,0 +1,954 @@ +"""Unit tests for per-user GitHub Copilot OAuth connections (per_user_auth).""" + +import asyncio +import datetime +import time +from collections.abc import Callable +from contextlib import contextmanager +from typing import Final +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import pytest + +import litellm +from litellm.exceptions import ( + AuthenticationError, + BadRequestError, + CallerCredentialAuthenticationError, + CallerCredentialRateLimitError, + ServiceUnavailableError, +) +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.github_copilot.per_user_auth import ( + _SESSION_CACHE, + GITHUB_COPILOT_USER_SESSION_KWARG_KEY, + GithubCopilotUserSession, + _session_cache_key, + aattach_github_copilot_user_session, + aexchange_github_token, + apoll_device_flow, + astart_device_flow, + attach_github_copilot_user_session, + evict_copilot_user_session, + exchange_github_token, + github_copilot_auth_mode, + github_copilot_user_session_from, + validated_copilot_api_base, +) +from litellm.types.utils import CredentialItem, ModelResponse + +EXCHANGE_URL = "https://api.github.com/copilot_internal/v2/token" +DEVICE_CODE_URL = "https://github.com/login/device/code" +ACCESS_TOKEN_URL = "https://github.com/login/oauth/access_token" + + +def _credential(auth_type="per_user_oauth"): + values = {} if auth_type is None else {"github_copilot_auth_type": auth_type} + return CredentialItem(credential_name="copilot-cred", credential_values=values, credential_info={}) + + +def _exchange_payload(token="copilot-token", expires_at=None, api_base="https://api.githubcopilot.com"): + return { + "token": token, + "expires_at": expires_at if expires_at is not None else int(time.time()) + 3600, + "endpoints": {"api": api_base}, + } + + +@contextmanager +def github_http(respond: Callable[[httpx.Request], httpx.Response]): + """Swap both module-level HTTP clients for httpx MockTransport versions and + collect every request, so tests assert on the real HTTP edge (URL, headers) + while no socket is ever opened.""" + requests: list[httpx.Request] = [] + + def capture(request: httpx.Request) -> httpx.Response: + requests.append(request) + return respond(request) + + async_client = AsyncHTTPHandler(transport=httpx.MockTransport(capture)) + sync_client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(capture))) + with ( + patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client", return_value=async_client), + patch("litellm.llms.custom_httpx.llm_http_handler.get_async_httpx_client", return_value=async_client), + patch("litellm.llms.custom_httpx.llm_http_handler.get_httpx_client", return_value=sync_client), + patch.object(litellm, "module_level_client", sync_client), + ): + yield requests + + +def _exchange_responder(payload=None, status=200): + def respond(request: httpx.Request) -> httpx.Response: + if str(request.url) == EXCHANGE_URL: + return httpx.Response(status, json=payload or _exchange_payload(), request=request) + pytest.fail(f"unexpected request to {request.url}") + + return respond + + +@pytest.fixture(autouse=True) +def _clear_session_cache(): + _SESSION_CACHE.flush_cache() + yield + _SESSION_CACHE.flush_cache() + + +@pytest.fixture +def per_user_credential(): + with patch.object(litellm, "credential_list", [_credential()]): + yield + + + +def test_named_per_user_credential_selects_per_user_mode(per_user_credential): + assert github_copilot_auth_mode("copilot-cred", None) is True + + +def test_named_shared_credential_stays_shared_even_if_kwargs_ask_for_per_user(): + with patch.object(litellm, "credential_list", [_credential(auth_type="shared")]): + assert github_copilot_auth_mode("copilot-cred", "per_user_oauth") is False + + +def test_named_credential_without_flag_stays_shared(): + with patch.object(litellm, "credential_list", [_credential(auth_type=None)]): + assert github_copilot_auth_mode("copilot-cred", None) is False + + +def test_per_user_kwargs_without_credential_name_raise_400(): + with pytest.raises(BadRequestError): + github_copilot_auth_mode(None, "per_user_oauth") + + +def test_unknown_named_credential_falls_back_to_kwargs(): + with patch.object(litellm, "credential_list", []): + assert github_copilot_auth_mode("missing-cred", "per_user_oauth") is True + assert github_copilot_auth_mode("missing-cred", None) is False + + + +@pytest.mark.parametrize( + "raw,expected", + [ + ("https://api.githubcopilot.com", "https://api.githubcopilot.com"), + ("https://api.githubcopilot.com/", "https://api.githubcopilot.com"), + ("https://tenant.githubcopilot.com:443", "https://tenant.githubcopilot.com:443"), + ("githubcopilot.com", "https://api.githubcopilot.com"), # no scheme -> default + ("http://api.githubcopilot.com", "https://api.githubcopilot.com"), # http rejected + ("https://evil.com", "https://api.githubcopilot.com"), + ("https://githubcopilot.com.evil.com", "https://api.githubcopilot.com"), + ("https://user:pw@api.githubcopilot.com", "https://api.githubcopilot.com"), + ("https://api.githubcopilot.com:8080", "https://api.githubcopilot.com"), + (None, "https://api.githubcopilot.com"), + (1234, "https://api.githubcopilot.com"), + ("https://API.GITHUBCOPILOT.COM", "https://API.GITHUBCOPILOT.COM"), + ], +) +def test_validated_copilot_api_base(raw, expected): + assert validated_copilot_api_base(raw) == expected + + + +def test_session_repr_and_str_hide_the_token(caplog): + session = GithubCopilotUserSession(token="secret-copilot-token", api_base="https://api.githubcopilot.com") + assert "secret-copilot-token" not in repr(session) + assert "secret-copilot-token" not in str(session) + with caplog.at_level("DEBUG"): + import logging + + logging.getLogger("litellm").debug("session: %s", session) + assert "secret-copilot-token" not in caplog.text + + + +def test_exchange_returns_session_and_caches_per_user(): + with github_http(_exchange_responder()) as requests: + s1 = exchange_github_token("user-a", "gh-token", "copilot-cred") + assert len(requests) == 1 + assert requests[0].headers["Authorization"] == "token gh-token" + s2 = exchange_github_token("user-a", "gh-token", "copilot-cred") + assert len(requests) == 1 + assert s2.token == s1.token == "copilot-token" + # different user -> separate exchange + exchange_github_token("user-b", "gh-token", "copilot-cred") + assert len(requests) == 2 + # different github token -> separate exchange + exchange_github_token("user-a", "gh-token-2", "copilot-cred") + assert len(requests) == 3 + + +@pytest.mark.parametrize("status", [401, 403, 404]) +def test_exchange_auth_failures_raise_caller_auth_error_and_evict(status): + with github_http(_exchange_responder(status=status)): + with pytest.raises(CallerCredentialAuthenticationError) as exc: + exchange_github_token("user-a", "gh-token", "copilot-cred") + assert "Reconnect GitHub Copilot for credential 'copilot-cred' in the LiteLLM UI (LLM Credentials)" in str( + exc.value + ) + assert _SESSION_CACHE.get_cache(_session_cache_key("user-a", "gh-token")) is None + + +def test_exchange_429_raises_caller_rate_limit(): + with github_http(_exchange_responder(status=429)): + with pytest.raises(CallerCredentialRateLimitError): + exchange_github_token("user-a", "gh-token", "copilot-cred") + + +def test_exchange_other_failure_raises_service_unavailable_without_body(): + with github_http(_exchange_responder(status=500, payload={"secret": "body"})): + with pytest.raises(ServiceUnavailableError) as exc: + exchange_github_token("user-a", "gh-token", "copilot-cred") + assert "body" not in str(exc.value) + + +def test_exchange_expiry_margin_refetches_near_expiry(): + with github_http(_exchange_responder(_exchange_payload(expires_at=int(time.time()) + 30))) as requests: + exchange_github_token("user-a", "gh-token", "copilot-cred") + # ttl under the 60s safety margin -> not cached, second call refetches + exchange_github_token("user-a", "gh-token", "copilot-cred") + assert len(requests) == 2 + + + +@pytest.mark.asyncio +async def test_aexchange_single_flight_one_http_call(): + def respond(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json=_exchange_payload(), request=request) + + with github_http(respond) as requests: + s1, s2 = await asyncio.gather( + aexchange_github_token("user-a", "gh-token", "copilot-cred"), + aexchange_github_token("user-a", "gh-token", "copilot-cred"), + ) + assert len(requests) == 1 + assert s1.token == s2.token == "copilot-token" + + +@pytest.mark.asyncio +async def test_aexchange_401_raises_and_evicts(): + with github_http(_exchange_responder(status=401)): + with pytest.raises(CallerCredentialAuthenticationError): + await aexchange_github_token("user-a", "gh-token", "copilot-cred") + assert _SESSION_CACHE.get_cache(_session_cache_key("user-a", "gh-token")) is None + + + +def _kwargs_with_connection(token="gho_user_token"): + return { + "litellm_credential_name": "copilot-cred", + "secret_fields": { + "user_provider_credentials_user_id": "user-a", + "user_provider_credentials": {"copilot-cred": token}, + }, + } + + +@pytest.mark.asyncio +async def test_aattach_injects_session_into_kwargs(per_user_credential): + with github_http(_exchange_responder()): + kwargs = _kwargs_with_connection() + await aattach_github_copilot_user_session(kwargs) + session = github_copilot_user_session_from(kwargs) + assert session is not None + assert session.token == "copilot-token" + + +def test_attach_sync_injects_session(per_user_credential): + with github_http(_exchange_responder()): + kwargs = _kwargs_with_connection() + attach_github_copilot_user_session(kwargs) + assert isinstance(kwargs[GITHUB_COPILOT_USER_SESSION_KWARG_KEY], GithubCopilotUserSession) + + +def test_attach_per_user_without_connection_raises_connect_401(per_user_credential): + kwargs = {"litellm_credential_name": "copilot-cred", "secret_fields": {}} + with pytest.raises(CallerCredentialAuthenticationError) as exc: + attach_github_copilot_user_session(kwargs) + assert "Connect GitHub Copilot for credential 'copilot-cred' in the LiteLLM UI (LLM Credentials)" in str(exc.value) + + +def test_attach_shared_mode_does_nothing_without_connection(): + with patch.object(litellm, "credential_list", [_credential(auth_type=None)]): + kwargs = {"litellm_credential_name": "copilot-cred"} + attach_github_copilot_user_session(kwargs) + assert GITHUB_COPILOT_USER_SESSION_KWARG_KEY not in kwargs + + +def test_evict_drops_cached_session(): + with github_http(_exchange_responder()) as requests: + exchange_github_token("user-a", "gh-token", "copilot-cred") + evict_copilot_user_session("user-a", "gh-token") + exchange_github_token("user-a", "gh-token", "copilot-cred") + assert len(requests) == 2 + + + +@pytest.mark.asyncio +async def test_astart_device_flow_shape(): + def respond(request: httpx.Request) -> httpx.Response: + assert str(request.url) == DEVICE_CODE_URL + return httpx.Response( + 200, + json={ + "device_code": "dc", + "user_code": "UC-123", + "verification_uri": "https://github.com/login/device", + "expires_in": 900, + "interval": 5, + }, + request=request, + ) + + with github_http(respond): + start = await astart_device_flow() + assert (start.device_code, start.user_code, start.verification_uri, start.expires_in, start.interval) == ( + "dc", + "UC-123", + "https://github.com/login/device", + 900, + 5, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "github_error,expected_status", + [ + ("authorization_pending", "pending"), + ("slow_down", "slow_down"), + ("expired_token", "expired"), + ("access_denied", "denied"), + ], +) +async def test_apoll_device_flow_error_mapping(github_error, expected_status): + def respond(request: httpx.Request) -> httpx.Response: + assert str(request.url) == ACCESS_TOKEN_URL + return httpx.Response(200, json={"error": github_error}, request=request) + + with github_http(respond): + poll = await apoll_device_flow("dc") + assert poll.status == expected_status + + +@pytest.mark.asyncio +async def test_apoll_device_flow_connected(): + def respond(request: httpx.Request) -> httpx.Response: + assert str(request.url) == ACCESS_TOKEN_URL + return httpx.Response(200, json={"access_token": "gho_x"}, request=request) + + with github_http(respond): + poll = await apoll_device_flow("dc") + assert poll.status == "connected" and poll.access_token == "gho_x" + + + +def test_is_github_copilot_per_user_request(): + from litellm.llms.github_copilot.per_user_auth import is_github_copilot_per_user_request + + session = GithubCopilotUserSession(token="t", api_base="https://api.githubcopilot.com") + assert is_github_copilot_per_user_request({GITHUB_COPILOT_USER_SESSION_KWARG_KEY: session}) is True + assert is_github_copilot_per_user_request({}) is False + assert is_github_copilot_per_user_request({GITHUB_COPILOT_USER_SESSION_KWARG_KEY: "not-a-session"}) is False + assert is_github_copilot_per_user_request({"litellm_credential_name": "copilot-cred"}) is False + + + +def _chat_completion_response(): + from litellm.utils import convert_to_model_response_object + + return convert_to_model_response_object( + response_object={ + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hello"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}, + }, + model_response_object=ModelResponse(), + ) + + +def _secret_fields(user_id: str, github_token: str): + return { + "user_provider_credentials_user_id": user_id, + "user_provider_credentials": {"copilot-cred": github_token}, + } + + +def test_two_per_user_calls_with_identical_bodies_both_reach_upstream( # test-quality-ok: upstream dispatch count is the observable signal of the cache exclusion + per_user_credential, +): + """Response-cache keys ignore caller identity: without the exclusion the second user's call + would be served from the first's cache entry and neither run on their own seat.""" + from litellm.caching import Cache + + litellm.cache = Cache() + try: + with ( + github_http(_exchange_responder()), + patch( + "litellm.main.openai_chat_completions.completion", + return_value=_chat_completion_response(), + ) as mock_completion, + ): + base = { + "model": "github_copilot/gpt-4o", + "messages": [{"role": "user", "content": "hi"}], + "litellm_credential_name": "copilot-cred", + } + litellm.completion(**base, secret_fields=_secret_fields("user-a", "gho_a")) + litellm.completion(**base, secret_fields=_secret_fields("user-b", "gho_b")) + assert mock_completion.call_count == 2 + finally: + litellm.cache = None + + +@pytest.mark.asyncio +async def test_two_per_user_async_calls_with_identical_bodies_both_reach_upstream( # test-quality-ok: upstream dispatch count is the observable signal of the cache exclusion + per_user_credential, +): + """Async path: per-user calls must bypass async_get_cache read and write the same way.""" + from litellm.caching import Cache + + litellm.cache = Cache() + try: + with ( + github_http(_exchange_responder()), + patch( + "litellm.main.openai_chat_completions.acompletion", + new=AsyncMock(return_value=_chat_completion_response()), + ) as mock_completion, + ): + base = { + "model": "github_copilot/gpt-4o", + "messages": [{"role": "user", "content": "hi"}], + "litellm_credential_name": "copilot-cred", + } + await litellm.acompletion(**base, secret_fields=_secret_fields("user-a", "gho_a")) + await litellm.acompletion(**base, secret_fields=_secret_fields("user-b", "gho_b")) + assert mock_completion.call_count == 2 + finally: + litellm.cache = None + + +def test_shared_mode_second_identical_call_hits_response_cache( # test-quality-ok: the cache hit is only observable as the upstream not being dispatched + tmp_path, monkeypatch +): + """Control: shared device-login github_copilot keeps caching exactly as before, so a second + identical call does not touch the upstream.""" + import json + + from litellm.caching import Cache + + # a mounted shared login: api-key.json with a far-future token, no device flow needed + (tmp_path / "api-key.json").write_text( + json.dumps( + { + "token": "shared-copilot-token", + "expires_at": 4102444800, + "endpoints": {"api": "https://api.githubcopilot.com"}, + } + ) + ) + monkeypatch.setenv("GITHUB_COPILOT_TOKEN_DIR", str(tmp_path)) + + litellm.cache = Cache() + try: + with patch( + "litellm.main.openai_chat_completions.completion", + return_value=_chat_completion_response(), + ) as mock_completion: + call = { + "model": "github_copilot/gpt-4o", + "messages": [{"role": "user", "content": "hi"}], + } + litellm.completion(**call) + litellm.completion(**call) + assert mock_completion.call_count == 1, "shared mode must still serve the second call from cache" + finally: + litellm.cache = None + + + +def _embedding_response_payload(): + return { + "object": "list", + "data": [{"object": "embedding", "embedding": [0.1, 0.2], "index": 0}], + "model": "text-embedding-3-small", + "usage": {"prompt_tokens": 4, "total_tokens": 4}, + } + + +def _responses_payload(): + return { + "id": "resp_1", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-5.3-codex", + "output": [ + { + "type": "message", + "id": "m1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "hi", "annotations": []}], + } + ], + "usage": {"input_tokens": 4, "output_tokens": 2, "total_tokens": 6}, + } + + +def _anthropic_messages_payload(): + return { + "id": "msg_1", + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": "hi"}], + "model": "claude-haiku-4.5", + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 4, "output_tokens": 2}, + } + + +def _copilot_surface_responder(model_payload): + def respond(request: httpx.Request) -> httpx.Response: + if str(request.url) == EXCHANGE_URL: + return httpx.Response(200, json=_exchange_payload(), request=request) + if request.url.host.endswith("githubcopilot.com"): + return httpx.Response(200, json=model_payload, request=request) + pytest.fail(f"unexpected request to {request.url}") + + return respond + + +def _no_authenticator(): + from litellm.llms.github_copilot.authenticator import Authenticator + + return patch.object( # test-quality-ok: patching the shared-login edge asserts it is never consulted + Authenticator, "get_api_key", side_effect=AssertionError("shared Authenticator must not run") + ) + + +def _assert_exchange_then_model_call(requests, github_token, model_host="https://api.githubcopilot.com"): + exchange = [r for r in requests if str(r.url) == EXCHANGE_URL] + upstream = [r for r in requests if r.url.host.endswith("githubcopilot.com")] + assert exchange, "expected the GitHub token exchange request" + assert upstream, "expected the model request to the Copilot host" + assert exchange[0].headers["Authorization"] == f"token {github_token}" + assert upstream[0].headers["Authorization"].startswith("Bearer ") + assert upstream[0].url.host == httpx.URL(model_host).host + + +def test_e2e_completion_per_user_credential(per_user_credential): + with ( + github_http(_exchange_responder()) as requests, + _no_authenticator(), + patch( + "litellm.main.openai_chat_completions.completion", + return_value=_chat_completion_response(), + ) as mock_completion, + ): + response = litellm.completion( + model="github_copilot/gpt-4o", + messages=[{"role": "user", "content": "hi"}], + litellm_credential_name="copilot-cred", + secret_fields=_secret_fields("user-a", "gho_a"), + ) + assert len(requests) == 1 and str(requests[0].url) == EXCHANGE_URL + assert requests[0].headers["Authorization"] == "token gho_a" + call_kwargs = mock_completion.call_args.kwargs + assert call_kwargs["api_key"] == "copilot-token" + assert call_kwargs["api_base"] == "https://api.githubcopilot.com" + extra_headers = call_kwargs["optional_params"]["extra_headers"] + assert extra_headers["Authorization"] == "Bearer copilot-token" + assert response.usage.total_tokens > 0 + + +def test_e2e_embedding_per_user_credential(per_user_credential): + with ( + github_http(_copilot_surface_responder(_embedding_response_payload())) as requests, + _no_authenticator(), + ): + response = litellm.embedding( + model="github_copilot/text-embedding-3-small", + input=["hi"], + litellm_credential_name="copilot-cred", + secret_fields=_secret_fields("user-a", "gho_a"), + ) + _assert_exchange_then_model_call(requests, "gho_a") + assert requests[-1].url.path == "/embeddings" + assert response.usage.prompt_tokens == 4 + + +def test_e2e_responses_per_user_credential(per_user_credential): + with ( + github_http(_copilot_surface_responder(_responses_payload())) as requests, + _no_authenticator(), + ): + response = litellm.responses( + model="github_copilot/gpt-5.3-codex", + input=[{"role": "user", "content": "hi"}], + litellm_credential_name="copilot-cred", + secret_fields=_secret_fields("user-a", "gho_a"), + ) + _assert_exchange_then_model_call(requests, "gho_a") + assert requests[-1].url.path == "/responses" + assert response.usage.total_tokens == 6 + + +@pytest.mark.asyncio +async def test_e2e_anthropic_messages_per_user_credential(per_user_credential): + with ( + github_http(_copilot_surface_responder(_anthropic_messages_payload())) as requests, + _no_authenticator(), + ): + response = await litellm.anthropic.messages.acreate( + model="github_copilot/claude-haiku-4.5", + max_tokens=16, + messages=[{"role": "user", "content": "hi"}], + litellm_credential_name="copilot-cred", + secret_fields=_secret_fields("user-a", "gho_a"), + ) + _assert_exchange_then_model_call(requests, "gho_a") + assert requests[-1].url.path == "/v1/messages" + assert response["usage"]["input_tokens"] == 4 + + +_CONNECT_MESSAGE = "Connect GitHub Copilot for credential 'copilot-cred' in the LiteLLM UI (LLM Credentials)" + + +def test_e2e_completion_not_connected_raises_connect_401(per_user_credential): + with pytest.raises(AuthenticationError, match="Connect GitHub Copilot"): + litellm.completion( + model="github_copilot/gpt-4o", + messages=[{"role": "user", "content": "hi"}], + litellm_credential_name="copilot-cred", + secret_fields={"user_provider_credentials_user_id": "user-a", "user_provider_credentials": {}}, + ) + + +def test_e2e_embedding_not_connected_raises_connect_401(per_user_credential): + with pytest.raises(AuthenticationError, match="Connect GitHub Copilot"): + litellm.embedding( + model="github_copilot/text-embedding-3-small", + input=["hi"], + litellm_credential_name="copilot-cred", + secret_fields={"user_provider_credentials_user_id": "user-a", "user_provider_credentials": {}}, + ) + + +def test_e2e_responses_not_connected_raises_connect_401(per_user_credential): + with pytest.raises(AuthenticationError, match="Connect GitHub Copilot"): + litellm.responses( + model="github_copilot/gpt-5.3-codex", + input=[{"role": "user", "content": "hi"}], + litellm_credential_name="copilot-cred", + secret_fields={"user_provider_credentials_user_id": "user-a", "user_provider_credentials": {}}, + ) + + +@pytest.mark.asyncio +async def test_e2e_anthropic_messages_not_connected_raises_connect_401(per_user_credential): + with pytest.raises(AuthenticationError, match="Connect GitHub Copilot"): + await litellm.anthropic.messages.acreate( + model="github_copilot/claude-haiku-4.5", + max_tokens=16, + messages=[{"role": "user", "content": "hi"}], + litellm_credential_name="copilot-cred", + secret_fields={"user_provider_credentials_user_id": "user-a", "user_provider_credentials": {}}, + ) + + +def _shared_login(tmp_path, monkeypatch): + import json + + (tmp_path / "api-key.json").write_text( + json.dumps( + { + "token": "shared-copilot-token", + "expires_at": 4102444800, + "endpoints": {"api": "https://api.githubcopilot.com"}, + } + ) + ) + monkeypatch.setenv("GITHUB_COPILOT_TOKEN_DIR", str(tmp_path)) + + +def test_e2e_completion_shared_mode_still_uses_authenticator(tmp_path, monkeypatch): + _shared_login(tmp_path, monkeypatch) + with patch( + "litellm.main.openai_chat_completions.completion", + return_value=_chat_completion_response(), + ) as mock_completion: + response = litellm.completion( + model="github_copilot/gpt-4o", + messages=[{"role": "user", "content": "hi"}], + ) + assert mock_completion.call_args.kwargs["api_key"] == "shared-copilot-token" + assert response.usage.total_tokens > 0 + + +def test_e2e_embedding_shared_mode_still_uses_authenticator(tmp_path, monkeypatch): + _shared_login(tmp_path, monkeypatch) + with github_http(_copilot_surface_responder(_embedding_response_payload())) as requests: + response = litellm.embedding( + model="github_copilot/text-embedding-3-small", + input=["hi"], + ) + upstream = [r for r in requests if r.url.host.endswith("githubcopilot.com")] + assert upstream and upstream[0].headers["Authorization"] == "Bearer shared-copilot-token" + assert response.usage.prompt_tokens == 4 + + +def test_e2e_responses_shared_mode_still_uses_authenticator(tmp_path, monkeypatch): + _shared_login(tmp_path, monkeypatch) + with github_http(_copilot_surface_responder(_responses_payload())) as requests: + response = litellm.responses( + model="github_copilot/gpt-5.3-codex", + input=[{"role": "user", "content": "hi"}], + ) + upstream = [r for r in requests if r.url.host.endswith("githubcopilot.com")] + assert upstream and upstream[0].headers["Authorization"] == "Bearer shared-copilot-token" + assert response.usage.total_tokens == 6 + + +@pytest.mark.asyncio +async def test_e2e_anthropic_messages_shared_mode_still_uses_authenticator(tmp_path, monkeypatch): + # The shared Authenticator's access-token refresh refuses to run inside an event + # loop, so on this async surface the mounted-login path can't reach it; patch the + # shared-login edge to emulate a mounted api-key.json. + from litellm.llms.github_copilot.authenticator import Authenticator + + with ( + github_http(_copilot_surface_responder(_anthropic_messages_payload())) as requests, + patch.object( # test-quality-ok: patching the shared-login edge is the fixture under test + Authenticator, "get_api_key", return_value="shared-copilot-token" + ) as mock_api_key, + patch.object( # test-quality-ok: same edge; api_key read is mocked so base must be too + Authenticator, "get_api_base", return_value="https://api.githubcopilot.com" + ), + ): + response = await litellm.anthropic.messages.acreate( + model="github_copilot/claude-haiku-4.5", + max_tokens=16, + messages=[{"role": "user", "content": "hi"}], + ) + mock_api_key.assert_called() + upstream = [r for r in requests if r.url.host.endswith("githubcopilot.com")] + assert upstream and upstream[0].headers["Authorization"] == "Bearer shared-copilot-token" + assert response["usage"]["input_tokens"] == 4 + + + +def test_sync_exchange_single_flight_one_http_call(): + import threading + + def slow_respond(request: httpx.Request) -> httpx.Response: + time.sleep(0.05) + return httpx.Response(200, json=_exchange_payload(), request=request) + + sessions: list[GithubCopilotUserSession] = [] + with github_http(slow_respond) as requests: + threads = [ + threading.Thread( + target=lambda: sessions.append(exchange_github_token("user-a", "gh-token", "copilot-cred")) + ) + for _ in range(6) + ] + for t in threads: + t.start() + for t in threads: + t.join() + assert len(requests) == 1 + assert len(sessions) == 6 and all(s.token == "copilot-token" for s in sessions) + + +def test_two_users_same_github_token_do_not_share_session(): + with github_http(_exchange_responder()): + s1 = exchange_github_token("user-a", "gh-same", "copilot-cred") + s2 = exchange_github_token("user-b", "gh-same", "copilot-cred") + assert _session_cache_key("user-a", "gh-same") != _session_cache_key("user-b", "gh-same") + assert s1.token == s2.token == "copilot-token" + + + +def test_router_constructs_with_per_user_credential_deployment(per_user_credential): + from litellm.llms.github_copilot.authenticator import Authenticator + + with patch.object(Authenticator, "get_api_key", side_effect=AssertionError("shared login")) as get_key: + router = litellm.Router( + model_list=[ + { + "model_name": "copilot", + "litellm_params": { + "model": "github_copilot/gpt-4.1", + "litellm_credential_name": "copilot-cred", + }, + } + ] + ) + get_key.assert_not_called() + deployments: Final = router.get_model_list() + assert deployments is not None + assert any(d["model_name"] == "copilot" for d in deployments) + + +def test_provider_info_per_user_without_session_returns_default_base_and_no_key(per_user_credential): + from litellm.llms.github_copilot.authenticator import Authenticator + from litellm.types.router import LiteLLM_Params + + params: Final = LiteLLM_Params( + model="github_copilot/gpt-4.1", + litellm_credential_name="copilot-cred", + ) + with patch.object(Authenticator, "get_api_key", side_effect=AssertionError("shared login")) as get_key: + _, _, dynamic_api_key, api_base = litellm.get_llm_provider( + model="github_copilot/gpt-4.1", litellm_params=params + ) + get_key.assert_not_called() + assert api_base == "https://api.githubcopilot.com" + assert dynamic_api_key is None + + +def test_provider_info_shared_github_copilot_still_uses_authenticator(): + from litellm.llms.github_copilot.authenticator import Authenticator + from litellm.types.router import LiteLLM_Params + + params: Final = LiteLLM_Params(model="github_copilot/gpt-4.1") + with ( + patch.object(Authenticator, "get_api_key", return_value="shared-token") as get_key, + patch.object(Authenticator, "get_api_base", return_value="https://api.githubcopilot.com"), + ): + _, _, dynamic_api_key, _ = litellm.get_llm_provider(model="github_copilot/gpt-4.1", litellm_params=params) + get_key.assert_called_once() + assert dynamic_api_key == "shared-token" + + +@pytest.mark.asyncio +async def test_aexchange_cancelled_waiter_does_not_cancel_the_shared_exchange(per_user_credential): + """Two waiters share one in-flight exchange; cancelling one must not cancel the + underlying future the other is waiting on.""" + gate = asyncio.Event() + + class _GatedClient: + async def get(self, url, headers=None): + await gate.wait() + return httpx.Response( + 200, json=_exchange_payload(), request=httpx.Request("GET", url) + ) + + with patch( # test-quality-ok: the gate holds the HTTP edge open so two waiters overlap + "litellm.llms.custom_httpx.http_handler.get_async_httpx_client", return_value=_GatedClient() + ): + first = asyncio.ensure_future(aexchange_github_token("user-a", "gh-shared", "copilot-cred")) + await asyncio.sleep(0) + second = asyncio.ensure_future(aexchange_github_token("user-a", "gh-shared", "copilot-cred")) + await asyncio.sleep(0) + second.cancel() + gate.set() + session = await first + assert session.token == "copilot-token" and session.api_base == "https://api.githubcopilot.com" + with pytest.raises(asyncio.CancelledError): + await second + + +@pytest.mark.asyncio +async def test_aexchange_failure_with_a_single_caller_emits_no_unretrieved_warning(per_user_credential): + """The leader removes the failed future before anyone awaits it; the stored + exception must be consumed so the loop does not log 'never retrieved'.""" + import litellm.llms.github_copilot.per_user_auth as pua + + captured: list[tuple[object, object]] = [] + loop = asyncio.get_running_loop() + previous_handler = loop.get_exception_handler() + loop.set_exception_handler(lambda _loop, context: captured.append((_loop, context))) + try: + with ( + github_http(_exchange_responder(status=401)), + pytest.raises(CallerCredentialAuthenticationError), + ): + await aexchange_github_token("user-a", "gh-fail", "copilot-cred") + _IN_FLIGHT_KEYS = list(pua._IN_FLIGHT) + import gc + + gc.collect() + await asyncio.sleep(0) + finally: + loop.set_exception_handler(previous_handler) + assert _IN_FLIGHT_KEYS == [] + assert not any( + "never retrieved" in str(context.get("message", "")) for _, context in captured if isinstance(context, dict) + ) + + +@pytest.mark.asyncio +async def test_per_user_streaming_call_writes_nothing_to_the_response_cache(per_user_credential): + """Per-user streamed completions must never be written to the shared response + cache: the handler built before the session attach would otherwise use a stale + request_kwargs copy without the session marker.""" + from litellm.caching import Cache + from litellm.caching.caching_handler import LLMCachingHandler + + litellm.cache = Cache() + try: + handler = LLMCachingHandler( + original_function=litellm.acompletion, + request_kwargs={"model": "github_copilot/gpt-4o"}, + start_time=datetime.datetime.now(), + ) + kwargs = { + "model": "github_copilot/gpt-4o", + "messages": [{"role": "user", "content": "hi"}], + GITHUB_COPILOT_USER_SESSION_KWARG_KEY: GithubCopilotUserSession( + token="copilot-token", api_base="https://api.githubcopilot.com" + ), + } + await handler.async_get_cache( + model="github_copilot/gpt-4o", + original_function=litellm.acompletion, + logging_obj=MagicMock(), + start_time=datetime.datetime.now(), + call_type="acompletion", + kwargs=kwargs, + ) + assert handler.should_store_result_in_cache( + original_function=litellm.acompletion, kwargs=handler.request_kwargs + ) is False + finally: + litellm.cache = None + + +@pytest.mark.asyncio +async def test_shared_streaming_call_still_writes_response_cache(tmp_path, monkeypatch): + """Control: shared device-login streaming keeps caching normally.""" + _shared_login(tmp_path, monkeypatch) + from litellm.caching import Cache + from litellm.caching.caching_handler import LLMCachingHandler + + litellm.cache = Cache() + try: + handler = LLMCachingHandler( + original_function=litellm.acompletion, + request_kwargs={"model": "github_copilot/gpt-4o"}, + start_time=datetime.datetime.now(), + ) + kwargs = { + "model": "github_copilot/gpt-4o", + "messages": [{"role": "user", "content": "hi"}], + } + await handler.async_get_cache( + model="github_copilot/gpt-4o", + original_function=litellm.acompletion, + logging_obj=MagicMock(), + start_time=datetime.datetime.now(), + call_type="acompletion", + kwargs=kwargs, + ) + assert handler.should_store_result_in_cache( + original_function=litellm.acompletion, kwargs=handler.request_kwargs + ) is True + finally: + litellm.cache = None diff --git a/tests/unit/proxy/auth/test_auth_utils.py b/tests/unit/proxy/auth/test_auth_utils.py index 076eb878fa87..a69a97a16a34 100644 --- a/tests/unit/proxy/auth/test_auth_utils.py +++ b/tests/unit/proxy/auth/test_auth_utils.py @@ -2497,6 +2497,67 @@ def test_caller_resupplied_value_overrides_admin_value_on_base_override(self): assert out["organization"] == "org-attacker" assert out["extra_body"] == {"attacker": "value"} + def test_per_user_oauth_credential_survives_base_override(self): + # Fail-closed: a caller-redirected api_base must not drop + # litellm_credential_name / github_copilot_auth_type off a per-user + # deployment, which would flip the call to shared mode and send the + # admin's Copilot token to the attacker's host. Keeping them is safe + # because per-user mode ignores the caller api_base anyway. + from litellm.router_utils.clientside_credential_handler import ( + get_dynamic_litellm_params, + ) + from litellm.types.utils import CredentialItem + + with patch.object( + litellm, + "credential_list", + [ + CredentialItem( + credential_name="copilot-cred", + credential_values={"github_copilot_auth_type": "per_user_oauth"}, + credential_info={}, + ) + ], + ): + out = get_dynamic_litellm_params( + litellm_params={ + "model": "github_copilot/gpt-4o", + "litellm_credential_name": "copilot-cred", + "github_copilot_auth_type": "per_user_oauth", + }, + request_kwargs={"api_base": "https://attacker.example"}, + ) + assert out["litellm_credential_name"] == "copilot-cred" + assert out["github_copilot_auth_type"] == "per_user_oauth" + assert out["api_base"] == "https://attacker.example" + + def test_shared_oauth_credential_name_is_cleared_on_base_override(self): + from litellm.router_utils.clientside_credential_handler import ( + get_dynamic_litellm_params, + ) + from litellm.types.utils import CredentialItem + + with patch.object( + litellm, + "credential_list", + [ + CredentialItem( + credential_name="copilot-cred", + credential_values={"github_copilot_auth_type": "shared"}, + credential_info={}, + ) + ], + ): + out = get_dynamic_litellm_params( + litellm_params={ + "model": "github_copilot/gpt-4o", + "litellm_credential_name": "copilot-cred", + "github_copilot_auth_type": "shared", + }, + request_kwargs={"api_base": "https://attacker.example"}, + ) + assert "github_copilot_auth_type" not in out + def test_field_echo_does_not_preserve_admin_value(self): # Regression: a caller that echoes an admin-config field name with # an *empty* value (or any value) must not be able to keep the @@ -4349,3 +4410,21 @@ def test_get_model_from_request_vertex_ai_passthrough( model = get_model_from_request(request_data, route) assert model == expected_model + + +@pytest.mark.parametrize( + "field", + ["github_copilot_auth_type", "user_provider_credentials", "github_copilot_user_session"], +) +def test_per_user_credential_slots_in_request_body_are_rejected(field: str) -> None: + """The per-user Copilot mode is decided by the stored credential's values and the caller's + connection by proxy-injected secret_fields; a body-supplied value could only spoof either.""" + with pytest.raises(ValueError, match="Rejected Request") as error: + is_request_body_safe( + request_body={"model": "github_copilot/gpt-4o", field: "attacker-chosen"}, + general_settings={}, + llm_router=None, + model="github_copilot/gpt-4o", + ) + + assert field in str(error.value) diff --git a/tests/unit/proxy/auth/test_route_checks.py b/tests/unit/proxy/auth/test_route_checks.py index 540b66ddaba8..f2868db661f9 100644 --- a/tests/unit/proxy/auth/test_route_checks.py +++ b/tests/unit/proxy/auth/test_route_checks.py @@ -4720,3 +4720,91 @@ def test_route_matches_pattern(): is False ) assert RouteChecks._route_matches_pattern("/v1/{thread_id}/messages", "/v1/messages/thread_2345") is False + + +def _scope_request(method: str, path: str) -> Request: + return Request({"type": "http", "method": method, "path": path, "query_string": b""}) + + +def _internal_user_check(route: str, method: str, role: LitellmUserRoles = LitellmUserRoles.INTERNAL_USER) -> None: + user_obj = LiteLLM_UserTable(user_id="u", user_email="u@x", user_role=role.value) + valid_token = UserAPIKeyAuth(user_id="u", user_role=role) + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=user_obj, + _user_role=role.value, + route=route, + request=_scope_request(method, route), + valid_token=valid_token, + request_data={}, + ) + + +@pytest.mark.parametrize( + ("method", "route"), + [ + ("GET", "/credentials/user_connections"), + ("POST", "/credentials/x/user_connection/start"), + ("POST", "/credentials/x/user_connection/poll"), + ("DELETE", "/credentials/x/user_connection"), + ("DELETE", "/credentials/a/b/user_connection"), + ], +) +def test_internal_user_credential_connection_routes_allowed(method: str, route: str) -> None: + _internal_user_check(route, method) + + +@pytest.mark.parametrize( + ("method", "route"), + [ + ("PATCH", "/credentials/x/user_connection"), + ("GET", "/credentials/by_name/x/user_connection"), + ("PATCH", "/credentials/x/user_connection/start"), + ("GET", "/credentials/x/user_connection/poll"), + ("DELETE", "/credentials/user_connections"), + ("PATCH", "/credentials/user_connections"), + ], +) +def test_internal_user_credential_connection_route_collisions_denied(method: str, route: str) -> None: + """A name colliding with the connection paths would fall through to the generic + credential CRUD handlers, so only the exact method/route pairs are allowed.""" + with pytest.raises(Exception, match="Only proxy admin"): + _internal_user_check(route, method) + + +@pytest.mark.parametrize( + ("method", "route"), + [ + ("GET", "/credentials/user_connections"), + ("POST", "/credentials/x/user_connection/start"), + ("POST", "/credentials/x/user_connection/poll"), + ("DELETE", "/credentials/x/user_connection"), + ], +) +def test_view_only_user_denied_credential_connection_routes(method: str, route: str) -> None: + with pytest.raises(Exception, match="Only proxy admin"): + _internal_user_check(route, method, role=LitellmUserRoles.INTERNAL_USER_VIEW_ONLY) + + +def test_user_connection_route_registration_order_wins_over_generic_credential_routes() -> None: + """The :path placeholders in the generic CRUD routes could swallow the + user_connection paths; the concrete routes must register (and match) first.""" + from starlette.routing import Match + + from litellm.proxy.credential_endpoints import endpoints as credential_endpoints + from litellm.proxy.proxy_server import app + + for method, path, expected in ( + ("DELETE", "/credentials/foo/user_connection", credential_endpoints.delete_user_connection), + ("POST", "/credentials/foo/user_connection/start", credential_endpoints.start_user_connection), + ): + scope = {"type": "http", "method": method, "path": path} + match = next( + ( + route + for route in app.router.routes + if hasattr(route, "matches") and route.matches(scope)[0] == Match.FULL + ), + None, + ) + assert match is not None, f"no route matched {method} {path}" + assert getattr(match, "endpoint", None) is expected diff --git a/tests/unit/proxy/credential_endpoints/test_endpoints.py b/tests/unit/proxy/credential_endpoints/test_endpoints.py index c6959ffd0fae..7d4e76489054 100644 --- a/tests/unit/proxy/credential_endpoints/test_endpoints.py +++ b/tests/unit/proxy/credential_endpoints/test_endpoints.py @@ -4,13 +4,15 @@ from contextlib import contextmanager from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import ec +from fastapi import HTTPException, Request from fastapi.testclient import TestClient import litellm -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import LiteLLM_UserTable, LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.credential_endpoints.endpoints import get_llm_router from litellm.proxy.proxy_server import app @@ -75,7 +77,9 @@ def install( llm_router: object | None = None, **repository_calls: AsyncMock, ) -> None: - patch("litellm.proxy.proxy_server.prisma_client", _prisma_without_credential_rows() if connected else None).start() + patch( + "litellm.proxy.proxy_server.prisma_client", _prisma_without_credential_rows() if connected else None + ).start() patch("litellm.proxy.proxy_server.master_key", "sk-test-master").start() patch.object(litellm, "credential_list", list(in_memory)).start() app.dependency_overrides[get_llm_router] = lambda: llm_router @@ -752,7 +756,9 @@ def test_proxy_admin_can_patch_wif_fields_onto_a_credential_through_model_id(sel update_by_name = AsyncMock(return_value=None) router = MagicMock() router.get_deployment.return_value = {"model_name": "claude-opus-5-5"} - router.get_deployment_credentials.return_value = {"anthropic_keycloak_token_url": "https://keycloak.internal/token"} + router.get_deployment_credentials.return_value = { + "anthropic_keycloak_token_url": "https://keycloak.internal/token" + } credential_store(find_by_name=AsyncMock(return_value=stored), update_by_name=update_by_name, llm_router=router) response = _patch_credential( @@ -1210,7 +1216,9 @@ async def test_authoritative_hydrate_falls_back_to_memory_when_the_row_is_absent prisma = MagicMock() prisma.db.litellm_credentialstable.find_unique = AsyncMock(return_value=None) - with patch.object(litellm, "credential_list", [only_in_memory]): # test-quality-ok: the config.yaml fallback under test + with patch.object( + litellm, "credential_list", [only_in_memory] + ): # test-quality-ok: the config.yaml fallback under test resolved = await hydrate_named_credential_authoritative("config-yaml-credential", prisma) assert resolved is not None @@ -1428,3 +1436,524 @@ def test_update_credential_still_accepts_a_body_without_credential_values(creden written = update_by_name.await_args.kwargs["data"] assert json.loads(written["credential_info"]) == {"custom_llm_provider": "openai"} assert set(json.loads(written["credential_values"])) == {"api_key"}, "stored values survive an info-only patch" + + +from contextlib import contextmanager as _ctx + +from litellm.caching.dual_cache import DualCache +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.proxy.credential_endpoints import user_provider_credentials as upc + +_DEVICE_CODE_URL = "https://github.com/login/device/code" +_ACCESS_TOKEN_URL = "https://github.com/login/oauth/access_token" +_COPILOT_TOKEN_URL = "https://api.github.com/copilot_internal/v2/token" +_GITHUB_USER_URL = "https://api.github.com/user" + + +def _as_user(user_id="user-a"): + return lambda: UserAPIKeyAuth(api_key="test-key", user_role="internal_user", user_id=user_id) + + +def _per_user_credential(name="copilot-cred"): + return CredentialItem( + credential_name=name, + credential_values={"github_copilot_auth_type": "per_user_oauth"}, + credential_info={}, + ) + + +def _connection_row(user_id="user-a", credential_name="copilot-cred", login="octo"): + from datetime import datetime, timezone + from types import SimpleNamespace + + return SimpleNamespace( + user_id=user_id, + credential_name=credential_name, + provider="github_copilot", + credential_b64=upc._encode( + upc.GithubCopilotUserConnectionPayload(access_token="gho_secret", github_login=login) + ), + created_at=datetime(2026, 1, 1, tzinfo=timezone.utc), + updated_at=datetime(2026, 1, 1, tzinfo=timezone.utc), + ) + + +@pytest.fixture(autouse=True) +def _connection_master_key(monkeypatch): + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "sk-test-master") + + +@_ctx +def _github_http(mapping): + """Patch the shared async HTTP client to an httpx MockTransport that answers + GitHub endpoints from {url: response-json dict or callable}; anything else fails.""" + + def respond(request): + entry = mapping.get(str(request.url)) + if entry is None: + raise AssertionError(f"unexpected request to {request.url}") + payload = entry(request) if callable(entry) else entry + return httpx.Response(200, json=payload, request=request) + + client = AsyncHTTPHandler(transport=httpx.MockTransport(respond)) + with patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client", return_value=client): + yield + + +_DEVICE_FLOW_START = { + "device_code": "dc-1", + "user_code": "UC-1", + "verification_uri": "https://github.com/login/device", + "expires_in": 900, + "interval": 5, +} + + +def _patch_user_connection_env(monkeypatch, credentials, rows=()): + """Per-user env: credential in memory, prisma table answering find_many with rows, + and a real in-memory cache for device-flow + credential caching.""" + monkeypatch.setattr(litellm, "credential_list", list(credentials)) + prisma_client = MagicMock() + table = MagicMock() + table.find_many = AsyncMock(return_value=list(rows)) + table.find_unique = AsyncMock(return_value=rows[0] if rows else None) + table.upsert = AsyncMock(return_value=None) + table.delete_many = AsyncMock(return_value=None) + prisma_client.db.litellm_userprovidercredentials = table + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma_client) + cache = DualCache() + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", cache) + return table + + +def test_user_connections_lists_per_user_credentials_with_connection_state(monkeypatch): + _patch_user_connection_env( + monkeypatch, + [_per_user_credential(), CredentialItem(credential_name="shared", credential_values={}, credential_info={})], + rows=[_connection_row()], + ) + response = _call_as("GET", "/credentials/user_connections", auth=_as_user()) + assert response.status_code == 200, response.text + assert response.json() == { + "connections": [ + { + "credential_name": "copilot-cred", + "provider": "github_copilot", + "connected": True, + "github_login": "octo", + "connected_at": "2026-01-01T00:00:00+00:00", + } + ] + } + + +def test_user_connections_requires_a_user_id(monkeypatch): + _patch_user_connection_env(monkeypatch, [_per_user_credential()]) + response = _call_as("GET", "/credentials/user_connections", auth=_as_admin) + assert response.status_code == 401 + + +def test_user_connection_start_returns_device_flow_fields(monkeypatch): + _patch_user_connection_env(monkeypatch, [_per_user_credential()]) + with _github_http({_DEVICE_CODE_URL: _DEVICE_FLOW_START}): + response = _call_as("POST", "/credentials/copilot-cred/user_connection/start", auth=_as_user()) + assert response.status_code == 200, response.text + assert response.json() == { + "user_code": "UC-1", + "verification_uri": "https://github.com/login/device", + "expires_in": 900, + "interval": 5, + } + + +def test_user_connection_start_404s_for_shared_credential(monkeypatch): + _patch_user_connection_env( + monkeypatch, [CredentialItem(credential_name="shared", credential_values={}, credential_info={})] + ) + response = _call_as("POST", "/credentials/shared/user_connection/start", auth=_as_user()) + assert response.status_code == 404 + + +def test_user_connection_poll_expired_when_no_device_flow(monkeypatch): + _patch_user_connection_env(monkeypatch, [_per_user_credential()]) + response = _call_as("POST", "/credentials/copilot-cred/user_connection/poll", auth=_as_user()) + assert response.status_code == 200 + assert response.json() == {"status": "expired", "interval": None, "github_login": None} + + +def test_user_connection_poll_connected_persists_and_reports_login(monkeypatch): + table = _patch_user_connection_env(monkeypatch, [_per_user_credential()]) + with _github_http( + { + _DEVICE_CODE_URL: _DEVICE_FLOW_START, + _ACCESS_TOKEN_URL: {"access_token": "gho_1"}, + _COPILOT_TOKEN_URL: {"token": "copilot-tok", "expires_at": 4102444800}, + _GITHUB_USER_URL: {"login": "octo"}, + } + ): + start = _call_as("POST", "/credentials/copilot-cred/user_connection/start", auth=_as_user()) + assert start.status_code == 200, start.text + poll = _call_as("POST", "/credentials/copilot-cred/user_connection/poll", auth=_as_user()) + assert poll.status_code == 200, poll.text + assert poll.json() == {"status": "connected", "interval": None, "github_login": "octo"} + table.upsert.assert_awaited_once() + + +def test_user_connection_poll_pending(monkeypatch): + table = _patch_user_connection_env(monkeypatch, [_per_user_credential()]) + with _github_http( + { + _DEVICE_CODE_URL: _DEVICE_FLOW_START, + _ACCESS_TOKEN_URL: {"error": "authorization_pending"}, + } + ): + _call_as("POST", "/credentials/copilot-cred/user_connection/start", auth=_as_user()) + poll = _call_as("POST", "/credentials/copilot-cred/user_connection/poll", auth=_as_user()) + assert poll.json()["status"] == "pending" + table.upsert.assert_not_awaited() + + +def test_delete_user_connection_disconnects_and_is_idempotent(monkeypatch): + table = _patch_user_connection_env(monkeypatch, [_per_user_credential()], rows=[_connection_row()]) + response = _call_as("DELETE", "/credentials/copilot-cred/user_connection", auth=_as_user()) + assert response.status_code == 200 + assert response.json() == {"status": "disconnected"} + table.delete_many.assert_awaited_once_with(where={"user_id": "user-a", "credential_name": "copilot-cred"}) + + # idempotent: row now gone + table.find_unique = AsyncMock(return_value=None) + again = _call_as("DELETE", "/credentials/copilot-cred/user_connection", auth=_as_user()) + assert again.status_code == 200 + assert again.json() == {"status": "disconnected"} + + +def test_deleting_the_credential_purges_user_connections(monkeypatch, credential_store): + rows = [_connection_row(user_id="u1"), _connection_row(user_id="u2")] + credential_store( + in_memory=[_per_user_credential()], + delete_by_name=AsyncMock(return_value=_per_user_credential()), + ) + prisma_client = MagicMock() + table = MagicMock() + table.find_many = AsyncMock(return_value=rows) + table.delete_many = AsyncMock(return_value=None) + prisma_client.db.litellm_userprovidercredentials = table + prisma_client.db.litellm_credentialstable.find_unique = AsyncMock(return_value=None) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma_client) + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", DualCache()) + + response = _delete_credential("copilot-cred") + assert response.status_code == 200, response.text + table.delete_many.assert_awaited_with(where={"credential_name": "copilot-cred"}) + + +def test_user_connections_isolated_per_user(monkeypatch): + """Two users connected to the same credential: A's listing shows only A's login, + A's delete removes only A's row, and a poll only reads A's device-flow entry.""" + from litellm.proxy.credential_endpoints.endpoints import _device_flow_cache_key + + rows = [_connection_row(user_id="user-a", login="octo"), _connection_row(user_id="user-b", login="hubot")] + table = _patch_user_connection_env(monkeypatch, [_per_user_credential()], rows=rows) + cache = getattr(litellm.proxy.proxy_server, "user_api_key_cache") + + async def _find_many_matching_user(*, where): + return [row for row in rows if row.user_id == where["user_id"]] + + table.find_many = AsyncMock(side_effect=_find_many_matching_user) + + listing = _call_as("GET", "/credentials/user_connections", auth=_as_user("user-a")) + assert listing.status_code == 200 + body = listing.json()["connections"] + assert [c["github_login"] for c in body] == ["octo"] + # find_many must be scoped to the calling user + where = table.find_many.await_args.kwargs["where"] + assert where["user_id"] == "user-a" + + # B has an in-flight device flow; A's poll must not see it + cache_key_b = _device_flow_cache_key("user-b", "copilot-cred") + cache.set_cache(cache_key_b, "enc-dc-b") + poll = _call_as("POST", "/credentials/copilot-cred/user_connection/poll", auth=_as_user("user-a")) + assert poll.json()["status"] == "expired" + assert cache.get_cache(cache_key_b) == "enc-dc-b", "B's device-flow entry must be untouched" + + delete = _call_as("DELETE", "/credentials/copilot-cred/user_connection", auth=_as_user("user-a")) + assert delete.json() == {"status": "disconnected"} + table.delete_many.assert_awaited_once_with(where={"user_id": "user-a", "credential_name": "copilot-cred"}) + + +@pytest.mark.parametrize("seat_status", [403, 404]) +def test_user_connection_poll_reports_no_copilot_seat(monkeypatch, seat_status): + """A GitHub seat check that rejects the user's token maps to no_copilot_seat, stores + nothing, and clears the in-flight device code.""" + table = _patch_user_connection_env(monkeypatch, [_per_user_credential()]) + + def respond(request): + url = str(request.url) + if url == _DEVICE_CODE_URL: + return httpx.Response(200, json=_DEVICE_FLOW_START, request=request) + if url == _ACCESS_TOKEN_URL: + return httpx.Response(200, json={"access_token": "gho_seat"}, request=request) + if url == _COPILOT_TOKEN_URL: + return httpx.Response(seat_status, json={}, request=request) + pytest.fail(f"unexpected request to {url}") + + client_obj = AsyncHTTPHandler(transport=httpx.MockTransport(respond)) + with patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client", return_value=client_obj): + start = _call_as("POST", "/credentials/copilot-cred/user_connection/start", auth=_as_user()) + assert start.status_code == 200 + poll = _call_as("POST", "/credentials/copilot-cred/user_connection/poll", auth=_as_user()) + assert poll.status_code == 200, poll.text + assert poll.json()["status"] == "no_copilot_seat" + table.upsert.assert_not_awaited() + + +def test_user_connection_poll_rate_limit_returns_429(monkeypatch): + table = _patch_user_connection_env(monkeypatch, [_per_user_credential()]) + + def respond(request): + url = str(request.url) + if url == _DEVICE_CODE_URL: + return httpx.Response(200, json=_DEVICE_FLOW_START, request=request) + if url == _ACCESS_TOKEN_URL: + return httpx.Response(200, json={"access_token": "gho_limited"}, request=request) + if url == _COPILOT_TOKEN_URL: + return httpx.Response(429, json={}, request=request) + pytest.fail(f"unexpected request to {url}") + + client_obj = AsyncHTTPHandler(transport=httpx.MockTransport(respond)) + with patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client", return_value=client_obj): + _call_as("POST", "/credentials/copilot-cred/user_connection/start", auth=_as_user()) + poll = _call_as("POST", "/credentials/copilot-cred/user_connection/poll", auth=_as_user()) + assert poll.status_code == 429 + table.upsert.assert_not_awaited() + + +@pytest.mark.parametrize( + "method,path", + [ + ("POST", "/credentials/missing/user_connection/start"), + ("POST", "/credentials/missing/user_connection/poll"), + ("DELETE", "/credentials/missing/user_connection"), + ("POST", "/credentials/shared/user_connection/poll"), + ("DELETE", "/credentials/shared/user_connection"), + ], +) +def test_user_connection_routes_404_for_missing_or_shared_credentials(monkeypatch, method, path): + _patch_user_connection_env( + monkeypatch, [CredentialItem(credential_name="shared", credential_values={}, credential_info={})] + ) + response = _call_as(method, path, auth=_as_user()) + assert response.status_code == 404 + + +@pytest.mark.parametrize( + "method,path", + [ + ("GET", "/credentials/user_connections"), + ("POST", "/credentials/copilot-cred/user_connection/start"), + ("POST", "/credentials/copilot-cred/user_connection/poll"), + ("DELETE", "/credentials/copilot-cred/user_connection"), + ], +) +def test_user_connection_routes_401_without_user_id(monkeypatch, method, path): + _patch_user_connection_env(monkeypatch, [_per_user_credential()]) + response = _call_as(method, path, auth=_as_admin) # admin token has no user_id + assert response.status_code == 401 + + +def test_user_connection_routes_roles(): + """internal_user is allowed (self-scoped), internal_user_view_only is denied, proxy_admin + is allowed; and POST /credentials stays admin-only for an internal user.""" + from litellm.proxy.auth.route_checks import RouteChecks + + routes = [ + ("/credentials/user_connections", "GET"), + ("/credentials/copilot-cred/user_connection/start", "POST"), + ("/credentials/copilot-cred/user_connection/poll", "POST"), + ("/credentials/copilot-cred/user_connection", "DELETE"), + ] + + def outcome(role, route, method="GET"): + if role == LitellmUserRoles.PROXY_ADMIN: + return "allowed" + user_obj = LiteLLM_UserTable(user_id="u", user_email="u@x", user_role=role.value) + valid_token = UserAPIKeyAuth(user_id="u", user_role=role) + request = MagicMock(spec=Request) + request.method = method + request.query_params = {} + try: + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=user_obj, + _user_role=role.value, + route=route, + request=request, + valid_token=valid_token, + request_data={}, + ) + except HTTPException as exc: + return f"denied:{exc.status_code}" + except Exception: + return "denied" + return "allowed" + + for route, method in routes: + assert outcome(LitellmUserRoles.INTERNAL_USER, route, method) == "allowed" + assert outcome(LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, route, method) == "denied" + assert outcome(LitellmUserRoles.INTERNAL_USER, "/credentials", method="POST") != "allowed" + + +class TestNonAdminCannotSetPerUserOauthOnCredential: + """github_copilot_auth_type selects the GitHub OAuth path; it is server-owned WIF, so a + team admin must not set it on a credential and a proxy admin can.""" + + def test_non_admin_cannot_create_a_per_user_oauth_credential(self): + with patch( # test-quality-ok: the proxy wiring under test is what this patches + "litellm.proxy.proxy_server.prisma_client", MagicMock() + ): + response = _post_credential( + { + "credential_name": "attacker-cred", + "credential_values": {"github_copilot_auth_type": "per_user_oauth"}, + "credential_info": {"custom_llm_provider": "github_copilot"}, + }, + auth=_as_non_admin, + ) + assert response.status_code == 403, response.text + assert "github_copilot_auth_type" in response.json()["error"]["message"] + + def test_non_admin_cannot_patch_a_credential_to_per_user_oauth(self, restore_credential_list): + stored = CredentialItem( + credential_name="existing", + credential_values={"api_key": "sk"}, + credential_info={"custom_llm_provider": "github_copilot"}, + ) + with _repository_holding(stored) as repository: + repository.find_unique_by_name = AsyncMock(return_value=stored) + response = _patch_credential( + "existing", + { + "credential_name": "existing", + "credential_values": {"github_copilot_auth_type": "per_user_oauth"}, + "credential_info": {}, + }, + auth=_as_non_admin, + ) + assert response.status_code == 403, response.text + + def test_proxy_admin_can_create_a_per_user_oauth_credential(self, restore_credential_list): + with _repository_holding(None) as repository: + response = _post_credential( + { + "credential_name": "admin-cred", + "credential_values": {"github_copilot_auth_type": "per_user_oauth"}, + "credential_info": {"custom_llm_provider": "github_copilot"}, + }, + ) + assert response.status_code == 200, response.text + repository.create.assert_awaited_once() + + def test_proxy_admin_can_patch_a_credential_to_per_user_oauth(self, restore_credential_list): + stored = CredentialItem( + credential_name="existing", + credential_values={"api_key": "sk"}, + credential_info={"custom_llm_provider": "github_copilot"}, + ) + with _repository_holding(stored) as repository: + repository.find_unique_by_name = AsyncMock(return_value=stored) + repository.update_by_name = AsyncMock(return_value=stored) + response = _patch_credential( + "existing", + { + "credential_name": "existing", + "credential_values": {"github_copilot_auth_type": "per_user_oauth"}, + "credential_info": {}, + }, + ) + assert response.status_code == 200, response.text + + +def test_delete_user_connection_clears_a_pending_device_flow(monkeypatch): + """Disconnect while a device flow is in flight must remove the pending flow too: + otherwise a later poll would resurrect the deleted connection.""" + from litellm.proxy.credential_endpoints.endpoints import _device_flow_cache_key + + table = _patch_user_connection_env(monkeypatch, [_per_user_credential()], rows=[_connection_row()]) + cache = getattr(litellm.proxy.proxy_server, "user_api_key_cache") + + with _github_http({_DEVICE_CODE_URL: _DEVICE_FLOW_START}): + start = _call_as("POST", "/credentials/copilot-cred/user_connection/start", auth=_as_user()) + assert start.status_code == 200 + assert cache.get_cache(_device_flow_cache_key("user-a", "copilot-cred")) is not None, ( + "device flow start stores a pending device code" + ) + + delete = _call_as("DELETE", "/credentials/copilot-cred/user_connection", auth=_as_user()) + assert delete.json() == {"status": "disconnected"} + assert cache.get_cache(_device_flow_cache_key("user-a", "copilot-cred")) is None + + # a poll after disconnect sees no pending flow -> expired, and stores nothing + table.upsert.reset_mock() + poll = _call_as("POST", "/credentials/copilot-cred/user_connection/poll", auth=_as_user()) + assert poll.json() == {"status": "expired", "interval": None, "github_login": None} + table.upsert.assert_not_awaited() + + +def test_user_connection_cache_keys_cannot_collide_on_colons(): + """user_id/credential_name are joined losslessly, so ('a:b','c') and ('a','b:c') + can never share a cache or device-flow entry.""" + from litellm.proxy.credential_endpoints.endpoints import _device_flow_cache_key + + assert upc._cache_key("a:b", "c") != upc._cache_key("a", "b:c") + assert _device_flow_cache_key("a:b", "c") != _device_flow_cache_key("a", "b:c") + + +def test_user_connection_slashed_credential_name_routes(monkeypatch): + """A credential named 'team/copilot' must still reach start/poll/delete: the + routes take :path parameters the same way the admin CRUD routes do.""" + table = _patch_user_connection_env(monkeypatch, [_per_user_credential(name="team/copilot")]) + with _github_http( + { + _DEVICE_CODE_URL: _DEVICE_FLOW_START, + _ACCESS_TOKEN_URL: {"error": "authorization_pending"}, + } + ): + start = _call_as("POST", "/credentials/team/copilot/user_connection/start", auth=_as_user()) + assert start.status_code == 200, start.text + assert start.json()["user_code"] == "UC-1" + + poll = _call_as("POST", "/credentials/team/copilot/user_connection/poll", auth=_as_user()) + assert poll.status_code == 200 + assert poll.json()["status"] == "pending" + + delete = _call_as("DELETE", "/credentials/team/copilot/user_connection", auth=_as_user()) + assert delete.status_code == 200 + assert delete.json() == {"status": "disconnected"} + table.delete_many.assert_awaited_once_with(where={"user_id": "user-a", "credential_name": "team/copilot"}) + + +def test_user_connection_route_check_allows_slashed_names_for_internal_users(): + """The RouteChecks allowlist entries use :path placeholders too, so an internal + user's request to /credentials/team/copilot/user_connection/... still matches.""" + from litellm.proxy.auth.route_checks import RouteChecks + + for route in ( + "/credentials/team/copilot/user_connection/start", + "/credentials/team/copilot/user_connection/poll", + "/credentials/team/copilot/user_connection", + ): + assert RouteChecks._route_matches_pattern( + route=route, pattern=route.replace("team/copilot", "{credential_name:path}") + ) + + user_obj = LiteLLM_UserTable(user_id="u", user_email="u@x", user_role=LitellmUserRoles.INTERNAL_USER.value) + request = MagicMock(spec=Request) + request.method = "POST" + request.query_params = {} + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=user_obj, + _user_role=LitellmUserRoles.INTERNAL_USER.value, + route="/credentials/team/copilot/user_connection/poll", + request=request, + valid_token=UserAPIKeyAuth(user_id="u", user_role=LitellmUserRoles.INTERNAL_USER), + request_data={}, + ) diff --git a/tests/unit/proxy/credential_endpoints/test_user_provider_credentials.py b/tests/unit/proxy/credential_endpoints/test_user_provider_credentials.py new file mode 100644 index 000000000000..b739ccfb3e7b --- /dev/null +++ b/tests/unit/proxy/credential_endpoints/test_user_provider_credentials.py @@ -0,0 +1,149 @@ +"""Tests for the per-user provider credentials DB/cache module.""" + +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from litellm.caching.dual_cache import DualCache +from litellm.proxy.credential_endpoints.user_provider_credentials import ( + GithubCopilotUserConnectionPayload, + aget_user_provider_tokens, + decode_user_provider_credential, + delete_user_provider_credential, + delete_user_provider_credentials_for_credential, + get_user_provider_credential, + invalidate_user_provider_credential_cache, + upsert_user_provider_credential, +) + + +def _prisma(table=None): + prisma_client = MagicMock() + prisma_client.db.litellm_userprovidercredentials = table or MagicMock() + return prisma_client + + +def _row(user_id="user-a", credential_name="copilot-cred", credential_b64="cipher", provider="github_copilot"): + return SimpleNamespace( + user_id=user_id, + credential_name=credential_name, + credential_b64=credential_b64, + provider=provider, + ) + + +@pytest.fixture(autouse=True) +def _master_key(monkeypatch): + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "sk-test-master") + + +@pytest.mark.asyncio +async def test_upsert_then_get_roundtrips_payload(): + stored = {} + + async def fake_upsert(*, where, data): + stored["row"] = _row( + user_id=where["user_id_credential_name"]["user_id"], + credential_name=where["user_id_credential_name"]["credential_name"], + credential_b64=data["create"]["credential_b64"], + ) + + async def fake_find_unique(*, where): + return stored.get("row") + + table = MagicMock() + table.upsert = AsyncMock(side_effect=fake_upsert) + table.find_unique = AsyncMock(side_effect=fake_find_unique) + prisma_client = _prisma(table) + + await upsert_user_provider_credential( + prisma_client, + "user-a", + "copilot-cred", + "github_copilot", + GithubCopilotUserConnectionPayload(access_token="gho_secret", github_login="octo"), + ) + payload = await get_user_provider_credential(prisma_client, "user-a", "copilot-cred") + assert payload is not None + assert payload.access_token == "gho_secret" + assert payload.github_login == "octo" + # ciphertext at rest, not plaintext + assert "gho_secret" not in stored["row"].credential_b64 + + +@pytest.mark.asyncio +async def test_delete_returns_prior_payload_and_is_idempotent(): + table = MagicMock() + table.delete_many = AsyncMock(return_value=None) + table.find_unique = AsyncMock(return_value=None) + prisma_client = _prisma(table) + assert await delete_user_provider_credential(prisma_client, "user-a", "copilot-cred") is None + table.delete_many.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_delete_for_credential_returns_affected_user_ids(): + table = MagicMock() + table.find_many = AsyncMock(return_value=[_row(user_id="u1"), _row(user_id="u2"), _row(user_id="u1")]) + table.delete_many = AsyncMock(return_value=None) + prisma_client = _prisma(table) + user_ids = await delete_user_provider_credentials_for_credential(prisma_client, "copilot-cred") + assert sorted(user_ids) == ["u1", "u2"] + table.delete_many.assert_awaited_once_with(where={"credential_name": "copilot-cred"}) + + +@pytest.mark.asyncio +async def test_aget_tokens_returns_plaintext_and_caches_ciphertext(): + payload = GithubCopilotUserConnectionPayload(access_token="gho_secret", github_login="octo") + from litellm.proxy.credential_endpoints import user_provider_credentials as upc + + ciphertext = upc._encode(payload) + table = MagicMock() + table.find_many = AsyncMock(return_value=[_row(credential_b64=ciphertext)]) + prisma_client = _prisma(table) + cache = DualCache() + + tokens = await aget_user_provider_tokens(prisma_client, cache, "user-a", ["copilot-cred"]) + assert tokens == {"copilot-cred": "gho_secret"} + + # second call served from cache: no new DB read, and cache holds ciphertext not plaintext + tokens2 = await aget_user_provider_tokens(prisma_client, cache, "user-a", ["copilot-cred"]) + assert tokens2 == {"copilot-cred": "gho_secret"} + assert table.find_many.await_count == 1 + cached = await cache.async_get_cache(upc._cache_key("user-a", "copilot-cred")) + assert cached == ciphertext + assert "gho_secret" not in cached + + +@pytest.mark.asyncio +async def test_aget_tokens_negative_caches_missing_connection(): + table = MagicMock() + table.find_many = AsyncMock(return_value=[]) + prisma_client = _prisma(table) + cache = DualCache() + + assert await aget_user_provider_tokens(prisma_client, cache, "user-a", ["copilot-cred"]) == {} + assert await aget_user_provider_tokens(prisma_client, cache, "user-a", ["copilot-cred"]) == {} + assert table.find_many.await_count == 1 + + +@pytest.mark.asyncio +async def test_invalidate_cache_forces_db_refetch(): + payload = GithubCopilotUserConnectionPayload(access_token="gho_new", github_login="octo") + from litellm.proxy.credential_endpoints import user_provider_credentials as upc + + table = MagicMock() + table.find_many = AsyncMock(return_value=[_row(credential_b64=upc._encode(payload))]) + prisma_client = _prisma(table) + cache = DualCache() + + await aget_user_provider_tokens(prisma_client, cache, "user-a", ["copilot-cred"]) + await invalidate_user_provider_credential_cache(cache, "user-a", "copilot-cred") + tokens = await aget_user_provider_tokens(prisma_client, cache, "user-a", ["copilot-cred"]) + assert tokens == {"copilot-cred": "gho_new"} + assert table.find_many.await_count == 2 + + +def test_decode_rejects_garbage(): + assert decode_user_provider_credential("not-a-cipher") is None diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index 889af0875f1c..098833475140 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -9455,3 +9455,188 @@ async def test_team_admin_can_still_delete_a_federated_deployment(self): assert "deleted successfully" in result["message"] mock_prisma.db.litellm_proxymodeltable.delete.assert_awaited_once() + + +class TestNonAdminCannotSetPerUserOauthOnModel: + """github_copilot_auth_type is a server-owned WIF field; a team admin must not write it + onto a deployment's litellm_params nor attach a per-user credential by name.""" + + @pytest.mark.asyncio + async def test_add_new_model_non_admin_cannot_set_github_copilot_auth_type(self): + from litellm.proxy._types import ProxyException + from litellm.proxy.management_endpoints.model_management_endpoints import ( + add_new_model, + ) + + non_admin = UserAPIKeyAuth(user_id="team_admin", user_role=LitellmUserRoles.INTERNAL_USER) + mock_prisma = MagicMock() + mock_prisma.writer_db.litellm_uisettings.find_unique = AsyncMock(return_value=None) + + with ( + patch( # test-quality-ok: the proxy wiring under test is what this patches + "litellm.proxy.proxy_server.prisma_client", mock_prisma + ), + patch( # test-quality-ok: the proxy wiring under test is what this patches + "litellm.proxy.proxy_server.store_model_in_db", True + ), + patch( # test-quality-ok: the proxy wiring under test is what this patches + "litellm.proxy.proxy_server.premium_user", True + ), + ): + with pytest.raises(ProxyException) as exc_info: + await add_new_model( + model_params=Deployment( + model_name="my-model", + litellm_params=LiteLLM_Params( + model="github_copilot/gpt-4o", + github_copilot_auth_type="per_user_oauth", + ), + model_info={"id": "wif-gate-copilot-1"}, + ), + user_api_key_dict=non_admin, + ) + assert "proxy admin" in str(exc_info.value.message).lower() + assert exc_info.value.param == "github_copilot_auth_type" + mock_prisma.db.litellm_proxymodeltable.create.assert_not_called() + + @pytest.mark.asyncio + async def test_add_new_model_admin_can_set_github_copilot_auth_type(self): + from litellm.proxy.management_endpoints.model_management_endpoints import ( + add_new_model, + ) + + admin = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) + mock_prisma = MagicMock() + created_row = MagicMock() + created_row.model_id = "wif-gate-copilot-2" + created_row.model_dump_json.return_value = "{}" + mock_prisma.db.litellm_proxymodeltable.create = AsyncMock(return_value=created_row) + + with ( + patch( # test-quality-ok: the proxy wiring under test is what this patches + "litellm.proxy.proxy_server.prisma_client", mock_prisma + ), + patch( # test-quality-ok: the proxy wiring under test is what this patches + "litellm.proxy.proxy_server.store_model_in_db", True + ), + patch( # test-quality-ok: the proxy wiring under test is what this patches + "litellm.proxy.proxy_server.premium_user", True + ), + patch( # test-quality-ok: the proxy wiring under test is what this patches + "litellm.proxy.proxy_server.master_key", "sk-test-master" + ), + patch( # test-quality-ok: the proxy wiring under test is what this patches + "litellm.proxy.proxy_server.llm_router", + MagicMock(**{"get_model_ids.return_value": ["wif-gate-copilot-2"]}), + ), + patch( # test-quality-ok: the proxy wiring under test is what this patches + "litellm.proxy.proxy_server.proxy_config", + MagicMock(add_deployment=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None))), + ), + ): + result = await add_new_model( + model_params=Deployment( + model_name="my-model", + litellm_params=LiteLLM_Params( + model="github_copilot/gpt-4o", + github_copilot_auth_type="per_user_oauth", + ), + model_info={"id": "wif-gate-copilot-2"}, + ), + user_api_key_dict=admin, + ) + assert result is created_row + + @pytest.mark.asyncio + async def test_add_new_model_non_admin_cannot_attach_a_per_user_oauth_credential(self): + from litellm.proxy._types import ProxyException + from litellm.proxy.management_endpoints.model_management_endpoints import ( + add_new_model, + ) + + non_admin = UserAPIKeyAuth(user_id="team_admin", user_role=LitellmUserRoles.INTERNAL_USER) + mock_prisma = MagicMock() + mock_prisma.writer_db.litellm_uisettings.find_unique = AsyncMock(return_value=None) + per_user_credential_row = MagicMock() + per_user_credential_row.credential_values = {"github_copilot_auth_type": "per_user_oauth"} + mock_prisma.db.litellm_credentialstable.find_unique = AsyncMock( + return_value=per_user_credential_row + ) + + with ( + patch( # test-quality-ok: the proxy wiring under test is what this patches + "litellm.proxy.proxy_server.prisma_client", mock_prisma + ), + patch( # test-quality-ok: the proxy wiring under test is what this patches + "litellm.proxy.proxy_server.store_model_in_db", True + ), + patch( # test-quality-ok: the proxy wiring under test is what this patches + "litellm.proxy.proxy_server.premium_user", True + ), + ): + with pytest.raises(ProxyException) as exc_info: + await add_new_model( + model_params=Deployment( + model_name="my-model", + litellm_params=LiteLLM_Params( + model="github_copilot/gpt-4o", + litellm_credential_name="admin-copilot-cred", + ), + model_info={"id": "wif-gate-copilot-3"}, + ), + user_api_key_dict=non_admin, + ) + mock_prisma.db.litellm_proxymodeltable.create.assert_not_called() + + @pytest.mark.asyncio + async def test_add_new_model_admin_can_attach_a_per_user_oauth_credential(self): + from litellm.proxy.management_endpoints.model_management_endpoints import ( + add_new_model, + ) + + admin = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) + mock_prisma = MagicMock() + created_row = MagicMock() + created_row.model_id = "wif-gate-copilot-4" + created_row.model_dump_json.return_value = "{}" + mock_prisma.db.litellm_proxymodeltable.create = AsyncMock(return_value=created_row) + per_user_credential_row = MagicMock() + per_user_credential_row.credential_values = {"github_copilot_auth_type": "per_user_oauth"} + mock_prisma.db.litellm_credentialstable.find_unique = AsyncMock( + return_value=per_user_credential_row + ) + + with ( + patch( # test-quality-ok: the proxy wiring under test is what this patches + "litellm.proxy.proxy_server.prisma_client", mock_prisma + ), + patch( # test-quality-ok: the proxy wiring under test is what this patches + "litellm.proxy.proxy_server.store_model_in_db", True + ), + patch( # test-quality-ok: the proxy wiring under test is what this patches + "litellm.proxy.proxy_server.premium_user", True + ), + patch( # test-quality-ok: the proxy wiring under test is what this patches + "litellm.proxy.proxy_server.master_key", "sk-test-master" + ), + patch( # test-quality-ok: the proxy wiring under test is what this patches + "litellm.proxy.proxy_server.llm_router", + MagicMock(**{"get_model_ids.return_value": ["wif-gate-copilot-4"]}), + ), + patch( # test-quality-ok: the proxy wiring under test is what this patches + "litellm.proxy.proxy_server.proxy_config", + MagicMock(add_deployment=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None))), + ), + ): + result = await add_new_model( + model_params=Deployment( + model_name="my-model", + litellm_params=LiteLLM_Params( + model="github_copilot/gpt-4o", + litellm_credential_name="admin-copilot-cred", + ), + model_info={"id": "wif-gate-copilot-4"}, + ), + user_api_key_dict=admin, + ) + assert result is created_row diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 86492537fd3d..0d7191b025e9 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -16,15 +16,29 @@ from starlette.datastructures import Headers import litellm +from litellm.constants import ( + ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY, + SESSION_ID_GENERATED_METADATA_KEY, + SESSION_ID_OMITTED_METADATA_KEY, +) +from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs +from litellm.litellm_core_utils.get_provider_specific_headers import ( + ProviderSpecificHeaderUtils, +) +from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + TRUSTED_CALLBACK_VARS_FIELD, +) +from litellm.litellm_core_utils.internal_call_metadata import MODEL_ACCESS_GROUP_METADATA_KEY +from litellm.litellm_core_utils.redact_messages import _get_turn_off_message_logging_from_dynamic_params +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id from litellm.proxy._types import AddTeamCallback, ProxyException, TeamCallbackMetadata, UserAPIKeyAuth from litellm.proxy.litellm_pre_call_utils import ( KeyAndTeamLoggingSettings, LiteLLMProxyRequestSetup, _apply_credential_overrides_from_model_config, _extract_credential_from_entry, - get_dynamic_logging_metadata, _get_enforced_params, - get_metadata_variable_name, _match_and_track_policies, _promoted_trace_control_fields, _resolve_credential_from_model_config, @@ -35,25 +49,11 @@ add_provider_specific_headers_to_request, check_if_token_is_service_account, clean_headers, + get_dynamic_logging_metadata, + get_metadata_variable_name, move_guardrails_to_metadata, ) -from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload -from litellm.litellm_core_utils.internal_call_metadata import MODEL_ACCESS_GROUP_METADATA_KEY -from litellm.litellm_core_utils.redact_messages import _get_turn_off_message_logging_from_dynamic_params -from litellm.litellm_core_utils.get_provider_specific_headers import ( - ProviderSpecificHeaderUtils, -) -from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( - TRUSTED_CALLBACK_VARS_FIELD, -) -from litellm.constants import ( - ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY, - SESSION_ID_GENERATED_METADATA_KEY, - SESSION_ID_OMITTED_METADATA_KEY, -) -from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM -from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id from litellm.types.utils import CredentialItem @@ -850,8 +850,13 @@ def test_initial_snapshot_refresh_clears_a_previous_guardrail_checkpoint() -> No from litellm.proxy.litellm_pre_call_utils import refresh_proxy_server_request_body_snapshot logging_obj: Final = Logging( - model="test-model", messages=[], stream=False, call_type="acompletion", - start_time=datetime.now(), litellm_call_id="new-request", function_id="new-request", + model="test-model", + messages=[], + stream=False, + call_type="acompletion", + start_time=datetime.now(), + litellm_call_id="new-request", + function_id="new-request", ) logging_obj.shadow_eval_request_snapshot = GuardrailRequestSnapshot.capture( {"messages": [{"role": "user", "content": "previous request"}]}, @@ -913,8 +918,14 @@ async def test_post_guardrail_snapshot_preserves_logging_only_masking_in_spend_l } data: Final = {"messages": messages, "metadata": metadata, "proxy_server_request": {}} logging_obj: Final = Logging( - model="test-model", messages=messages, stream=False, call_type="acompletion", - start_time=datetime.now(), litellm_call_id="mask-spend", function_id="mask-spend", kwargs=data, + model="test-model", + messages=messages, + stream=False, + call_type="acompletion", + start_time=datetime.now(), + litellm_call_id="mask-spend", + function_id="mask-spend", + kwargs=data, ) data["litellm_logging_obj"] = logging_obj refresh_proxy_server_request_body_snapshot(data, guardrails_applied=True) @@ -928,9 +939,13 @@ async def test_post_guardrail_snapshot_preserves_logging_only_masking_in_spend_l kwargs, _ = await guardrail.async_logging_hook( kwargs=logging_obj.model_call_details, result=None, call_type="acompletion" ) - stored: Final = json.loads(_get_proxy_server_request_for_spend_logs_payload( - metadata={}, litellm_params=kwargs["litellm_params"], kwargs=kwargs, - )) + stored: Final = json.loads( + _get_proxy_server_request_for_spend_logs_payload( + metadata={}, + litellm_params=kwargs["litellm_params"], + kwargs=kwargs, + ) + ) assert kwargs["messages"] == [{"role": "user", "content": "email [EMAIL]"}] assert stored["messages"] == kwargs["messages"] @@ -5421,9 +5436,9 @@ async def test_team_guardrail_merges_with_global_policy(): shadowed and non-default guardrails silently received an empty requested_guardrails list. """ + from litellm.proxy.litellm_pre_call_utils import move_guardrails_to_metadata from litellm.proxy.policy_engine.attachment_registry import get_attachment_registry from litellm.proxy.policy_engine.policy_registry import get_policy_registry - from litellm.proxy.litellm_pre_call_utils import move_guardrails_to_metadata from litellm.types.proxy.policy_engine import ( Policy, PolicyAttachment, @@ -8539,20 +8554,27 @@ async def test_mcp_credentials_only_removed_from_logging_copies(path: str, custo request.headers = Headers(request.headers) settings: Final = {"mcp_client_side_auth_header_name": custom_auth, "user_header_name": "x-user-id"} server: Final = MCPServer( - server_id="header-test", name="header-test", transport="http", url="https://example.com/mcp", + server_id="header-test", + name="header-test", + transport="http", + url="https://example.com/mcp", extra_headers=["x-service-token", "x-user-id"], ) with ( patch("litellm.proxy.proxy_server.general_settings", settings), patch.dict( "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager.config_mcp_servers", - {"header-test": server}, clear=True, + {"header-test": server}, + clear=True, ), ): updated: Final = await add_litellm_data_to_request( data={"model": "test-model", "messages": [{"role": "user", "content": "hello"}]}, - request=request, user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), - proxy_config=MagicMock(), general_settings=settings, version="test", + request=request, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings=settings, + version="test", ) for header_dict in _all_header_dicts(updated, metadata_name): assert not any(value in json.dumps(header_dict) for value in secrets.values()) @@ -8572,7 +8594,10 @@ def test_signoz_callback_vars_are_scoped_to_the_signoz_callback(): data=AddTeamCallback( callback_name="signoz", callback_type="success", - callback_vars={"signoz_ingestion_key": "team-key", "signoz_ingestion_endpoint": "https://ingest.eu.signoz.cloud:443"}, + callback_vars={ + "signoz_ingestion_key": "team-key", + "signoz_ingestion_endpoint": "https://ingest.eu.signoz.cloud:443", + }, ), team_callback_settings_obj=None, ) @@ -8625,3 +8650,431 @@ def test_arize_otlp_protocol_on_a_key_logging_entry_reaches_the_destination(monk assert destinations[0].endpoint == "https://arize.internal.example/v1/traces" finally: is_otel_v2_enabled.cache_clear() + + +class TestResolveUserProviderCredentials: + """``_resolve_user_provider_credentials_for_request`` injects the calling user's stored + provider tokens into secret_fields for deployments carrying a per_user_oauth credential.""" + + def _router(self): + from litellm import Router + + return Router( + model_list=[ + { + "model_name": "copilot-chat", + "litellm_params": { + "model": "openai/gpt-4o", + "litellm_credential_name": "copilot-cred", + }, + }, + { + "model_name": "shared-chat", + "litellm_params": { + "model": "openai/gpt-4o", + "litellm_credential_name": "shared-cred", + }, + }, + ], + fallbacks=[{"copilot-chat": ["shared-chat"]}, {"*": ["copilot-chat"]}], + ) + + def _env(self, monkeypatch): + from litellm.caching.dual_cache import DualCache + + monkeypatch.setattr( + litellm, + "credential_list", + [ + CredentialItem( + credential_name="copilot-cred", + credential_values={"github_copilot_auth_type": "per_user_oauth"}, + credential_info={}, + ), + CredentialItem(credential_name="shared-cred", credential_values={"api_key": "k"}, credential_info={}), + ], + ) + prisma_client = MagicMock() + table = MagicMock() + table.find_many = AsyncMock(return_value=[]) + prisma_client.db.litellm_userprovidercredentials = table + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma_client) + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", DualCache()) + return table + + @pytest.mark.asyncio + async def test_injects_tokens_for_per_user_deployments_only(self, monkeypatch): + from litellm.proxy.credential_endpoints import user_provider_credentials as upc + from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request + + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "sk-test-master") + table = self._env(monkeypatch) + ciphertext = upc._encode(upc.GithubCopilotUserConnectionPayload(access_token="gho_secret", github_login="octo")) + row = SimpleNamespace(user_id="user-a", credential_name="copilot-cred", credential_b64=ciphertext) + table.find_many = AsyncMock(return_value=[row]) + + data = {"model": "copilot-chat", "secret_fields": {}} + await _resolve_user_provider_credentials_for_request( + data=data, + authenticated_user_id="user-a", + llm_router=self._router(), + ) + + assert data["secret_fields"]["user_provider_credentials"] == {"copilot-cred": "gho_secret"} + assert data["secret_fields"]["user_provider_credentials_user_id"] == "user-a" + # the shared fallback credential is never looked up + call_where = table.find_many.await_args.kwargs["where"] + assert call_where == {"AND": ({"user_id": "user-a"}, {"credential_name": {"in": ["copilot-cred"]}})} + + @pytest.mark.asyncio + async def test_group_with_no_per_user_deployment_leaves_secret_fields_untouched(self, monkeypatch): + from litellm import Router + from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request + + self._env(monkeypatch) + data = {"model": "shared-chat", "secret_fields": {}} + await _resolve_user_provider_credentials_for_request( + data=data, + authenticated_user_id="user-a", + llm_router=Router( + model_list=[ + { + "model_name": "shared-chat", + "litellm_params": { + "model": "openai/gpt-4o", + "litellm_credential_name": "shared-cred", + }, + } + ] + ), + ) + assert "user_provider_credentials" not in data["secret_fields"] + + @pytest.mark.asyncio + async def test_unconnected_user_gets_empty_map_not_an_error(self, monkeypatch): + """A wildcard fallback to a per-user group is legitimate coverage; a user with no + connection gets an empty token map and the provider-side helper fails the call later.""" + from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request + + self._env(monkeypatch) + data = {"model": "shared-chat", "secret_fields": {}} + await _resolve_user_provider_credentials_for_request( + data=data, + authenticated_user_id="user-a", + llm_router=self._router(), + ) + assert data["secret_fields"]["user_provider_credentials"] == {} + + @pytest.mark.asyncio + async def test_missing_user_id_does_nothing(self, monkeypatch): + from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request + + self._env(monkeypatch) + data = {"model": "copilot-chat", "secret_fields": {}} + await _resolve_user_provider_credentials_for_request( + data=data, + authenticated_user_id=None, + llm_router=self._router(), + ) + assert "user_provider_credentials" not in data["secret_fields"] + + @pytest.mark.asyncio + async def test_unrelated_model_makes_zero_db_and_cache_calls(self, monkeypatch): + from litellm import Router + from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request + + table = self._env(monkeypatch) + data = {"model": "unrelated", "secret_fields": {}} + await _resolve_user_provider_credentials_for_request( + data=data, + authenticated_user_id="user-a", + llm_router=Router( + model_list=[ + { + "model_name": "unrelated", + "litellm_params": {"model": "openai/gpt-4o"}, + } + ] + ), + ) + table.find_many.assert_not_awaited() + assert "user_provider_credentials" not in data["secret_fields"] + + @pytest.mark.asyncio + async def test_per_user_deployment_with_pydantic_litellm_params(self, monkeypatch): + """Router deployments may carry litellm_params as a LiteLLM_Params model rather + than a dict; the per-user credential must still be collected.""" + from litellm.proxy.litellm_pre_call_utils import _per_user_credential_names_for_groups + from litellm.types.router import LiteLLM_Params + + self._env(monkeypatch) + router = MagicMock() + router.get_model_list = MagicMock( + return_value=[ + { + "model_name": "copilot-chat", + "litellm_params": LiteLLM_Params( + model="github_copilot/gpt-4o", + litellm_credential_name="copilot-cred", + ), + } + ] + ) + assert _per_user_credential_names_for_groups(router, frozenset({"copilot-chat"})) == ("copilot-cred",) + + +def _per_user_env(monkeypatch, credential_names=("copilot-cred",)): + from litellm.caching.dual_cache import DualCache + + monkeypatch.setattr( + litellm, + "credential_list", + [ + CredentialItem( + credential_name=name, + credential_values={"github_copilot_auth_type": "per_user_oauth"}, + credential_info={}, + ) + for name in credential_names + ], + ) + prisma_client = MagicMock() + table = MagicMock() + table.find_many = AsyncMock(return_value=[]) + prisma_client.db.litellm_userprovidercredentials = table + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma_client) + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", DualCache()) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "sk-test-master") + return table + + +@pytest.mark.asyncio +@pytest.mark.parametrize("body_value", ["", None, "shared-cred"]) +async def test_request_body_litellm_credential_name_is_rejected_for_per_user_models(monkeypatch, body_value): + """A caller-supplied litellm_credential_name (empty, null, or another credential) + would downgrade a per-user deployment to the shared device login; it is a 400.""" + from fastapi import HTTPException + + from litellm import Router + from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request + + _per_user_env(monkeypatch) + router = Router( + model_list=[ + { + "model_name": "copilot-chat", + "litellm_params": { + "model": "github_copilot/gpt-4o", + "litellm_credential_name": "copilot-cred", + }, + } + ] + ) + data = {"model": "copilot-chat", "secret_fields": {}, "litellm_credential_name": body_value} + with pytest.raises(HTTPException) as exc: + await _resolve_user_provider_credentials_for_request( + data=data, + authenticated_user_id="user-a", + llm_router=router, + ) + assert exc.value.status_code == 400 + assert "litellm_credential_name cannot be set in the request body" in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_request_body_litellm_credential_name_untouched_for_shared_models(monkeypatch): + """Regression: models with no per-user credential keep accepting the param.""" + from litellm import Router + from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request + + _per_user_env(monkeypatch) + router = Router( + model_list=[ + { + "model_name": "shared-chat", + "litellm_params": {"model": "openai/gpt-4o", "litellm_credential_name": "shared-cred"}, + } + ] + ) + data = {"model": "shared-chat", "secret_fields": {}, "litellm_credential_name": "shared-cred"} + await _resolve_user_provider_credentials_for_request( + data=data, + authenticated_user_id="user-a", + llm_router=router, + ) + assert "user_provider_credentials" not in data["secret_fields"] + + +@pytest.mark.asyncio +async def test_transitive_fallback_chain_loads_per_user_credential(monkeypatch): + """A -> B -> C where only C is per-user: the request to A must still load the + caller's connection for C's credential.""" + from litellm import Router + from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request + + table = _per_user_env(monkeypatch) + router = Router( + model_list=[ + {"model_name": "a", "litellm_params": {"model": "openai/gpt-4o"}}, + {"model_name": "b", "litellm_params": {"model": "openai/gpt-4o"}}, + { + "model_name": "c", + "litellm_params": { + "model": "github_copilot/gpt-4o", + "litellm_credential_name": "copilot-cred", + }, + }, + ], + fallbacks=[{"a": ["b"]}, {"b": ["c"]}], + ) + data = {"model": "a", "secret_fields": {}} + await _resolve_user_provider_credentials_for_request( + data=data, + authenticated_user_id="user-a", + llm_router=router, + ) + where = table.find_many.await_args.kwargs["where"] + assert where == {"AND": ({"user_id": "user-a"}, {"credential_name": {"in": ["copilot-cred"]}})} + assert data["secret_fields"]["user_provider_credentials_user_id"] == "user-a" + + +@pytest.mark.asyncio +async def test_secret_fields_user_provider_credentials_repr_hides_tokens(monkeypatch): + """Debug logging prints the request body; the credential map must be a + RedactedDict so a GitHub token can never appear in str()/repr() output.""" + from litellm import Router + from litellm.proxy.credential_endpoints import user_provider_credentials as upc + from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request + from litellm.types.proxy.litellm_pre_call_utils import RedactedDict + + table = _per_user_env(monkeypatch) + ciphertext = upc._encode(upc.GithubCopilotUserConnectionPayload(access_token="gho_secret", github_login="octo")) + table.find_many = AsyncMock( + return_value=[SimpleNamespace(user_id="user-a", credential_name="copilot-cred", credential_b64=ciphertext)] + ) + router = Router( + model_list=[ + { + "model_name": "copilot-chat", + "litellm_params": { + "model": "github_copilot/gpt-4o", + "litellm_credential_name": "copilot-cred", + }, + } + ] + ) + data = {"model": "copilot-chat", "secret_fields": {}} + await _resolve_user_provider_credentials_for_request( + data=data, + authenticated_user_id="user-a", + llm_router=router, + ) + credentials = data["secret_fields"]["user_provider_credentials"] + assert isinstance(credentials, RedactedDict) + assert "gho_secret" not in str(data) + assert "gho_secret" not in repr(credentials) + assert credentials["copilot-cred"] == "gho_secret", "provider reads still get the token" + + +@pytest.mark.asyncio +async def test_header_mapped_user_id_cannot_load_another_users_connection(monkeypatch): + """general_settings.user_header_mappings lets a caller rename their user_id through a + request header. Per-user token resolution is bound to the key-authenticated identity, + so a header naming user B still loads only user A's connections.""" + from litellm import Router + from litellm.caching.dual_cache import DualCache + from litellm.proxy.credential_endpoints import user_provider_credentials as upc + from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request + + _per_user_env(monkeypatch) + load = AsyncMock(return_value={"copilot-cred": "gho_user_a_token"}) + monkeypatch.setattr(upc, "aget_user_provider_tokens", load) + router = Router( + model_list=[ + { + "model_name": "copilot-chat", + "litellm_params": { + "model": "github_copilot/gpt-4o", + "litellm_credential_name": "copilot-cred", + }, + } + ] + ) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", router) + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", DualCache()) + + request_mock = MagicMock(spec=Request) + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/v1/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"Content-Type": "application/json", "X-OpenWebUI-User-Id": "user-b"} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + + data = {"model": "copilot-chat"} + updated = await add_litellm_data_to_request( + data=data, + request=request_mock, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key", user_id="user-a"), + proxy_config=MagicMock(), + general_settings={ + "user_header_mappings": [{"header_name": "X-OpenWebUI-User-Id", "litellm_user_role": "internal_user"}] + }, + version="test-version", + ) + + called_user_ids = [call.kwargs["user_id"] for call in load.await_args_list] + assert called_user_ids == ["user-a"], f"header-mapped user-b must never reach the token loader: {called_user_ids}" + credentials = updated["secret_fields"]["user_provider_credentials"] + assert credentials["copilot-cred"] == "gho_user_a_token" + assert updated["secret_fields"]["user_provider_credentials_user_id"] == "user-a" + + +@pytest.mark.asyncio +async def test_per_user_token_resolution_uses_key_identity_without_header_mapping(monkeypatch): + """Control: with no user_header_mappings configured the key's user_id is used as before.""" + from litellm import Router + from litellm.caching.dual_cache import DualCache + from litellm.proxy.credential_endpoints import user_provider_credentials as upc + from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request + + _per_user_env(monkeypatch) + load = AsyncMock(return_value={"copilot-cred": "gho_user_a_token"}) + monkeypatch.setattr(upc, "aget_user_provider_tokens", load) + router = Router( + model_list=[ + { + "model_name": "copilot-chat", + "litellm_params": { + "model": "github_copilot/gpt-4o", + "litellm_credential_name": "copilot-cred", + }, + } + ] + ) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", router) + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", DualCache()) + + request_mock = MagicMock(spec=Request) + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/v1/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"Content-Type": "application/json", "X-OpenWebUI-User-Id": "user-b"} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + + data = {"model": "copilot-chat"} + updated = await add_litellm_data_to_request( + data=data, + request=request_mock, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key", user_id="user-a"), + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + called_user_ids = [call.kwargs["user_id"] for call in load.await_args_list] + assert called_user_ids == ["user-a"] + assert updated["secret_fields"]["user_provider_credentials"]["copilot-cred"] == "gho_user_a_token" diff --git a/tests/unit/router_utils/test_cooldown_handlers.py b/tests/unit/router_utils/test_cooldown_handlers.py index d2e21f172db6..ecb759278b97 100644 --- a/tests/unit/router_utils/test_cooldown_handlers.py +++ b/tests/unit/router_utils/test_cooldown_handlers.py @@ -2,7 +2,6 @@ import importlib import random import time -from typing import Final from unittest.mock import MagicMock, patch import pytest @@ -2463,3 +2462,49 @@ async def test_router_fallbacks_with_cooldowns_and_dynamic_credentials(): await asyncio.sleep(1) cooled_down = await async_get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None) assert len(cooled_down) == 1 and cooled_down[0] in {"123", "456"} + + +@pytest.mark.parametrize( + "exception,status", + [ + (litellm.CallerCredentialAuthenticationError(message="reconnect", llm_provider="github_copilot", model=""), 401), + (litellm.CallerCredentialRateLimitError(message="slow down", llm_provider="github_copilot", model=""), 429), + ], +) +def test_caller_credential_errors_never_cool_down_the_shared_deployment(single_deployment_router, exception, status): + """A per-user credential failure is scoped to one caller's stored token; cooling down the + shared deployment would punish every other user on the group.""" + assert _should_run_cooldown_logic(single_deployment_router, "dep-1", status, exception) is False + + +def test_per_user_session_upstream_errors_never_cool_down_the_shared_deployment(single_deployment_router): + """A 429/401 from the caller's own Copilot seat is scoped to that user; it must + not cool down the shared deployment for every other caller.""" + from litellm.llms.github_copilot.per_user_auth import GithubCopilotUserSession + + session_kwargs = { + "github_copilot_user_session": GithubCopilotUserSession( + token="copilot-token", api_base="https://api.githubcopilot.com" + ) + } + for exc, status in ( + (litellm.RateLimitError("copilot 429", "github_copilot", "gpt-4o"), 429), + (litellm.AuthenticationError("copilot 401", "github_copilot", "gpt-4o"), 401), + (litellm.InternalServerError("copilot 500", "github_copilot", "gpt-4o"), 500), + ): + assert ( + _should_run_cooldown_logic( + single_deployment_router, "dep-1", status, exc, request_kwargs=session_kwargs + ) + is False + ) + + +def test_shared_mode_upstream_429_still_cools_down_the_deployment(single_deployment_router): + """Regression: without the session marker the same 429 is a deployment-health + signal and cools down exactly as before.""" + exc = litellm.RateLimitError("copilot 429", "github_copilot", "gpt-4o") + assert _should_run_cooldown_logic( + single_deployment_router, "dep-1", 429, exc, request_kwargs={"model": "github_copilot/gpt-4o"} + ) is True + assert _should_run_cooldown_logic(single_deployment_router, "dep-1", 429, exc) is True diff --git a/tests/unit/test_internal_context.py b/tests/unit/test_internal_context.py index 1f43f9c63121..ccbe1487fc49 100644 --- a/tests/unit/test_internal_context.py +++ b/tests/unit/test_internal_context.py @@ -54,6 +54,7 @@ "litellm/llms/bedrock/base_aws_llm.py", "litellm/llms/custom_httpx/http_handler.py", "litellm/llms/gigachat/authenticator.py", + "litellm/llms/github_copilot/per_user_auth.py", "litellm/llms/litellm_proxy/skills/handler.py", "litellm/llms/openai/common_utils.py", "litellm/llms/openai_like/model_info.py", diff --git a/tests/unit/types/test_litellm_params.py b/tests/unit/types/test_litellm_params.py index 3dee0118b37b..4210aa6ab075 100644 --- a/tests/unit/types/test_litellm_params.py +++ b/tests/unit/types/test_litellm_params.py @@ -87,6 +87,7 @@ "azure_scope", "azure_ad_token_provider", "litellm_credential_name", + "github_copilot_auth_type", "configurable_clientside_auth_params", "use_xai_oauth", "aws_batch_role_arn", @@ -247,6 +248,7 @@ "attempted_targets", "proxy_server_request", "secret_fields", + "github_copilot_user_session", "litellm_trusted_callback_vars", "_litellm_addressed_response_id", "_litellm_strip_stream_usage", diff --git a/tests/unit/types/test_router.py b/tests/unit/types/test_router.py index dce4225f70a0..89238e04a5a1 100644 --- a/tests/unit/types/test_router.py +++ b/tests/unit/types/test_router.py @@ -19,6 +19,7 @@ CustomPricingLiteLLMParams, MirroredPricingParams, anthropic_wif_litellm_params, + github_copilot_oauth_litellm_params, oauth_token_exchange_litellm_params, openai_wif_litellm_params, server_owned_wif_litellm_params, @@ -242,9 +243,7 @@ def test_model_info_rejects_offset_aware_access_window_times(): with pytest.raises(ValidationError): ModelInfo( id="x", - access_windows=[ - {"start": "22:00+05:00", "end": "06:00", "timezone": "UTC", "team_ids": ["t"]} - ], + access_windows=[{"start": "22:00+05:00", "end": "06:00", "timezone": "UTC", "team_ids": ["t"]}], ) @@ -333,8 +332,12 @@ def test_server_owned_registry_includes_anthropic_openai_and_oauth_token_exchang "token_exchange_profile", "token_exchange_scope", ) + assert github_copilot_oauth_litellm_params == ("github_copilot_auth_type",) assert server_owned_wif_litellm_params == ( - anthropic_wif_litellm_params + openai_wif_litellm_params + oauth_token_exchange_litellm_params + anthropic_wif_litellm_params + + openai_wif_litellm_params + + oauth_token_exchange_litellm_params + + github_copilot_oauth_litellm_params ) assert set(openai_wif_litellm_params) == { "openai_identity_provider_id", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/credentials/useUserConnections.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/credentials/useUserConnections.ts new file mode 100644 index 000000000000..2dac978348c3 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/credentials/useUserConnections.ts @@ -0,0 +1,15 @@ +import { userConnectionsListCall, UserProviderConnectionsResponse } from "@/components/networking"; +import { useQuery } from "@tanstack/react-query"; +import { createQueryKeys } from "../common/queryKeysFactory"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; + +export const userConnectionsKeys = createQueryKeys("userConnections"); + +export const useUserConnections = () => { + const { accessToken } = useAuthorized(); + return useQuery({ + queryKey: userConnectionsKeys.list({}), + queryFn: async () => await userConnectionsListCall(accessToken!), + enabled: Boolean(accessToken), + }); +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.test.tsx index 41f71a3bf125..fb6ed90c002d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.test.tsx @@ -128,13 +128,20 @@ describe("ModelsAndEndpointsPage", () => { expect(teamInfoProps).toHaveBeenLastCalledWith(expect.objectContaining({ is_proxy_admin: false })); }); - it("hides admin-only tabs for a non-admin user", () => { + it("hides admin-only tabs for a non-admin user but keeps LLM Credentials for their connections", () => { mockUseAuthorized.mockReturnValue(NON_ADMIN); renderPage(); - expect(screen.queryByRole("tab", { name: "LLM Credentials" })).not.toBeInTheDocument(); + expect(screen.getByRole("tab", { name: "LLM Credentials" })).toBeInTheDocument(); + expect(screen.queryByRole("tab", { name: "Pass-Through Endpoints" })).not.toBeInTheDocument(); expect(screen.queryByRole("tab", { name: "Health Status" })).not.toBeInTheDocument(); }); + it("hides LLM Credentials from an internal viewer", () => { + mockUseAuthorized.mockReturnValue({ ...NON_ADMIN, userRole: "Internal Viewer", isViewOnly: true }); + renderPage(); + expect(screen.queryByRole("tab", { name: "LLM Credentials" })).not.toBeInTheDocument(); + }); + it("keeps the full admin tab order for a real admin", () => { renderPage(); expect(screen.getAllByRole("tab").map((tab) => tab.textContent)).toEqual([ diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.tsx index a4e5afe0533d..cd2922dff4bd 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.tsx @@ -114,13 +114,14 @@ export default function ModelsAndEndpointsPage() { // effectiveSessionRole reports proxy_admin_viewer as "Admin", so isAdmin alone would show a // viewer these write-only panels; only the raw-role isViewOnly separates them. Health Status // stays: it is the bucket's one read view, and viewers keep read parity with admins. - ...(isAdmin && !isViewOnly ? (["llm-credentials", "pass-through"] as const) : []), + ...(!isViewOnly && (isAdmin || isInternalUser) ? (["llm-credentials"] as const) : []), + ...(isAdmin && !isViewOnly ? (["pass-through"] as const) : []), ...(isAdmin ? (["health"] as const) : []), ...(isAdmin && !isViewOnly ? (["retry-settings", "model-group-alias", "access-group-budgets", "price-data"] as const) : []), ], - [canCreate, canViewAutoRouters, isAdmin, isViewOnly], + [canCreate, canViewAutoRouters, isAdmin, isInternalUser, isViewOnly], ); const allModelsLabel = isAdmin ? "Deployed Models" : "Your Models"; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/LlmCredentialsPanel.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/LlmCredentialsPanel.test.tsx new file mode 100644 index 000000000000..929b14359270 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/LlmCredentialsPanel.test.tsx @@ -0,0 +1,91 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { render, screen, waitFor, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import LlmCredentialsPanel from "./LlmCredentialsPanel"; + +const networking = vi.hoisted(() => ({ + credentialListCall: vi.fn(), + userConnectionsListCall: vi.fn(), + userConnectionDeleteCall: vi.fn(), +})); + +vi.mock("@/components/networking", async () => ({ + ...(await vi.importActual("@/components/networking")), + ...networking, +})); + +const mockUseAuthorized = vi.fn(); +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ default: () => mockUseAuthorized() })); + +const session = (userRole: string) => ({ accessToken: "at", userRole, userId: "u1", isViewOnly: false }); + +const renderPanel = () => + render( + + + , + ); + +describe("LlmCredentialsPanel", () => { + beforeEach(() => { + networking.credentialListCall.mockReset().mockResolvedValue({ + credentials: [ + { credential_name: "shared-openai", credential_values: {}, credential_info: { custom_llm_provider: "openai" } }, + ], + }); + networking.userConnectionsListCall.mockReset().mockResolvedValue({ + connections: [ + { + credential_name: "copilot-per-user", + provider: "github_copilot", + connected: false, + github_login: null, + connected_at: null, + }, + { + credential_name: "copilot-team", + provider: "github_copilot", + connected: true, + github_login: "octocat", + connected_at: "2026-10-07T00:00:00Z", + }, + ], + }); + networking.userConnectionDeleteCall.mockReset().mockResolvedValue(undefined); + }); + + it("shows an internal user only their connections and never fetches the shared credential list", async () => { + mockUseAuthorized.mockReturnValue(session("Internal User")); + renderPanel(); + + expect(await screen.findByText("copilot-per-user")).toBeInTheDocument(); + expect(screen.getByText("Your connections")).toBeInTheDocument(); + expect(screen.getByText("Connected as @octocat")).toBeInTheDocument(); + expect(screen.queryByText("shared-openai")).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: /Add Credential/ })).not.toBeInTheDocument(); + expect(networking.credentialListCall).not.toHaveBeenCalled(); + }); + + it("shows an admin the credential list and their connections", async () => { + mockUseAuthorized.mockReturnValue(session("Admin")); + renderPanel(); + + expect(await screen.findByText("shared-openai")).toBeInTheDocument(); + expect(await screen.findByText("copilot-per-user")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /Add Credential/ })).toBeInTheDocument(); + }); + + it("disconnects the selected connection and refreshes the list", async () => { + const user = userEvent.setup(); + mockUseAuthorized.mockReturnValue(session("Internal User")); + renderPanel(); + + const row = await screen.findByRole("row", { name: /copilot-team/ }); + await user.click(within(row).getByRole("button", { name: "Disconnect" })); + + expect(networking.userConnectionDeleteCall).toHaveBeenCalledWith("at", "copilot-team"); + await waitFor(() => expect(networking.userConnectionsListCall).toHaveBeenCalledTimes(2)); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/LlmCredentialsPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/LlmCredentialsPanel.tsx index c71ed1774165..572d7df250fc 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/LlmCredentialsPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/LlmCredentialsPanel.tsx @@ -1,7 +1,16 @@ "use client"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import CredentialsPanel from "@/components/model_add/CredentialsPanel"; +import UserConnectionsPanel from "@/components/model_add/UserConnectionsPanel"; +import { all_admin_roles } from "@/utils/roles"; export default function LlmCredentialsPanel() { - return ; + const { userRole } = useAuthorized(); + return ( +
+ {all_admin_roles.includes(userRole ?? "") && } + +
+ ); } diff --git a/ui/litellm-dashboard/src/components/add_model/provider_auth_types.test.ts b/ui/litellm-dashboard/src/components/add_model/provider_auth_types.test.ts index 9670c6c6e273..5467ff0085ad 100644 --- a/ui/litellm-dashboard/src/components/add_model/provider_auth_types.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/provider_auth_types.test.ts @@ -78,4 +78,13 @@ describe("provider auth types", () => { expect(hiddenAuthFieldKeys(authTypes, "first")).toEqual(["second_key", "third_key"]); }); + + it("defaults GitHub Copilot to shared device login and recognises a stored per-user credential", () => { + const authTypes = authTypesFor("GITHUB_COPILOT"); + + expect(inferAuthTypeId(authTypes, { api_key: "" })).toBe("shared_device_login"); + expect(inferAuthTypeId(authTypes, { github_copilot_auth_type: "per_user_oauth" })).toBe("per_user_oauth"); + expect(hiddenAuthFieldKeys(authTypes, "per_user_oauth")).toEqual(["api_base", "api_key"]); + expect(hiddenAuthFieldKeys(authTypes, "shared_device_login")).toEqual(["github_copilot_auth_type"]); + }); }); diff --git a/ui/litellm-dashboard/src/components/add_model/provider_auth_types.ts b/ui/litellm-dashboard/src/components/add_model/provider_auth_types.ts index b723e7bba7bb..0555c7eb7e64 100644 --- a/ui/litellm-dashboard/src/components/add_model/provider_auth_types.ts +++ b/ui/litellm-dashboard/src/components/add_model/provider_auth_types.ts @@ -6,11 +6,34 @@ export interface ProviderAuthType { readonly description: string; readonly fieldKeys: readonly string[]; readonly requiredFieldKeys: readonly string[]; + readonly fixedValues?: Readonly>; } const EMPTY_PROVIDER_AUTH_TYPES: readonly ProviderAuthType[] = []; +export const GITHUB_COPILOT_AUTH_TYPE_KEY = "github_copilot_auth_type"; +export const GITHUB_COPILOT_PER_USER_AUTH_TYPE = "per_user_oauth"; + export const PROVIDER_AUTH_TYPES: Partial> = { + GITHUB_COPILOT: [ + { + id: "shared_device_login", + label: "Shared device login", + description: + "One GitHub device login on the proxy host, stored in its token file, is used for every caller of models on this credential.", + fieldKeys: ["api_base", "api_key"], + requiredFieldKeys: [], + }, + { + id: GITHUB_COPILOT_PER_USER_AUTH_TYPE, + label: "Per-user GitHub OAuth", + description: + "Each LiteLLM user connects their own GitHub account from LLM Credentials, and their requests use their own GitHub Copilot access. Users who have not connected get a 401.", + fieldKeys: [GITHUB_COPILOT_AUTH_TYPE_KEY], + requiredFieldKeys: [GITHUB_COPILOT_AUTH_TYPE_KEY], + fixedValues: { [GITHUB_COPILOT_AUTH_TYPE_KEY]: GITHUB_COPILOT_PER_USER_AUTH_TYPE }, + }, + ], MICROSOFT_365_COPILOT: [ { id: "oauth_token_exchange", diff --git a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx index 4291df8ad9e9..88e1250b00a3 100644 --- a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx +++ b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx @@ -331,6 +331,11 @@ const ProviderSpecificFieldsContent: React.FC = ({

{selectedAuthType.description}

)} + {Object.entries(selectedAuthType?.fixedValues ?? {}).map(([fieldKey, fixedValue]) => ( + + {() => null} + + ))} {isLoading && allFields.length === 0 &&

Loading provider fields...

} {loadError && allFields.length === 0 && (

diff --git a/ui/litellm-dashboard/src/components/model_add/CredentialModal.integration.test.tsx b/ui/litellm-dashboard/src/components/model_add/CredentialModal.integration.test.tsx index bec4d04b9043..57fdd29695f0 100644 --- a/ui/litellm-dashboard/src/components/model_add/CredentialModal.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/model_add/CredentialModal.integration.test.tsx @@ -60,6 +60,15 @@ vi.mock("../networking", async () => { { key: "api_key", label: "Delegated Access Token", field_type: "password" }, ], }, + { + provider: "GITHUB_COPILOT", + provider_display_name: Providers.GITHUB_COPILOT, + litellm_provider: "github_copilot", + credential_fields: [ + { key: "api_base", label: "API Base", field_type: "text" }, + { key: "api_key", label: "API Key", field_type: "password" }, + ], + }, ]), }; }); @@ -503,3 +512,60 @@ describe("CredentialModal with Anthropic workload identity federation", () => { ); }); }); + +describe("CredentialModal with GitHub Copilot auth types", () => { + const perUserCopilotCredential: CredentialItem = { + credential_name: "copilot-per-user", + credential_values: { github_copilot_auth_type: "per_user_oauth" }, + credential_info: { custom_llm_provider: "GITHUB_COPILOT" }, + }; + + it("saves a per-user GitHub OAuth credential with only the auth type and no secret", async () => { + const user = userEvent.setup(); + const onSubmit = renderModal({ initialProvider: "GITHUB_COPILOT" }); + + expect(await screen.findByRole("combobox", { name: "Auth Type:" })).toHaveTextContent("Shared device login"); + expect(await screen.findByLabelText("API Key")).toBeInTheDocument(); + await chooseOption(user, /^Auth Type:/, "Per-user GitHub OAuth"); + expect(screen.queryByLabelText("API Key")).not.toBeInTheDocument(); + expect(screen.queryByLabelText("API Base")).not.toBeInTheDocument(); + fill("Credential Name:", "copilot-per-user"); + await user.click(screen.getByRole("button", { name: "Add Credential" })); + + expect(onSubmit).toHaveBeenCalledWith( + { + credential_name: "copilot-per-user", + custom_llm_provider: "GITHUB_COPILOT", + github_copilot_auth_type: "per_user_oauth", + }, + [], + ); + }); + + it("keeps the shared device login credential free of the per-user auth type", async () => { + const user = userEvent.setup(); + const onSubmit = renderModal({ initialProvider: "GITHUB_COPILOT" }); + + await screen.findByLabelText("API Key"); + fill("Credential Name:", "copilot-shared"); + await user.click(screen.getByRole("button", { name: "Add Credential" })); + + expect(onSubmit).toHaveBeenCalledWith( + { credential_name: "copilot-shared", custom_llm_provider: "GITHUB_COPILOT" }, + [], + ); + }); + + it("opens a stored per-user credential on its auth type and deletes it when switched to shared", async () => { + const user = userEvent.setup(); + const onSubmit = renderModal({ mode: "edit", existingCredential: perUserCopilotCredential }); + + expect(await screen.findByRole("combobox", { name: "Auth Type:" })).toHaveTextContent("Per-user GitHub OAuth"); + await chooseOption(user, /^Auth Type:/, "Shared device login"); + await user.click(screen.getByRole("button", { name: "Update Credential" })); + + const [values, valuesToDelete] = onSubmit.mock.calls[0]; + expect(values).toEqual({ credential_name: "copilot-per-user", custom_llm_provider: "GITHUB_COPILOT" }); + expect(valuesToDelete).toEqual(["github_copilot_auth_type"]); + }); +}); diff --git a/ui/litellm-dashboard/src/components/model_add/GithubCopilotConnectModal.test.tsx b/ui/litellm-dashboard/src/components/model_add/GithubCopilotConnectModal.test.tsx new file mode 100644 index 000000000000..72ff22b8e4ce --- /dev/null +++ b/ui/litellm-dashboard/src/components/model_add/GithubCopilotConnectModal.test.tsx @@ -0,0 +1,124 @@ +import { act, fireEvent, render, screen } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import GithubCopilotConnectModal from "./GithubCopilotConnectModal"; + +const networking = vi.hoisted(() => ({ + userConnectionStartCall: vi.fn(), + userConnectionPollCall: vi.fn(), +})); + +vi.mock("@/components/networking", () => networking); +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ default: () => ({ accessToken: "session-token" }) })); + +const START_RESPONSE = { + user_code: "ABCD-1234", + verification_uri: "https://github.com/login/device", + expires_in: 900, + interval: 5, +}; + +const renderModal = () => { + const props = { onClose: vi.fn(), onConnected: vi.fn(), onDisconnect: vi.fn() }; + render(); + return props; +}; + +const advance = async (seconds: number) => { + await act(async () => { + await vi.advanceTimersByTimeAsync(seconds * 1000); + }); +}; + +describe("GithubCopilotConnectModal", () => { + beforeEach(() => { + vi.useFakeTimers({ shouldAdvanceTime: false }); + networking.userConnectionStartCall.mockReset().mockResolvedValue(START_RESPONSE); + networking.userConnectionPollCall.mockReset(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("shows the device code and link, polls at the given interval, honours slow_down and ends connected", async () => { + networking.userConnectionPollCall + .mockResolvedValueOnce({ status: "pending" }) + .mockResolvedValueOnce({ status: "slow_down", interval: 10 }) + .mockResolvedValueOnce({ status: "connected", github_login: "octocat" }); + const props = renderModal(); + await advance(0); + + expect(networking.userConnectionStartCall).toHaveBeenCalledWith("session-token", "copilot-per-user"); + expect(screen.getByTestId("github-device-code")).toHaveTextContent("ABCD-1234"); + expect(screen.getByRole("link", { name: "https://github.com/login/device" })).toHaveAttribute( + "href", + "https://github.com/login/device", + ); + + await advance(4.9); + expect(networking.userConnectionPollCall).not.toHaveBeenCalled(); + await advance(0.1); + expect(networking.userConnectionPollCall).toHaveBeenCalledTimes(1); + await advance(5); + expect(networking.userConnectionPollCall).toHaveBeenCalledTimes(2); + + await advance(9.9); + expect(networking.userConnectionPollCall).toHaveBeenCalledTimes(2); + await advance(0.1); + expect(networking.userConnectionPollCall).toHaveBeenCalledTimes(3); + + expect(screen.getByText("Connected as @octocat")).toBeInTheDocument(); + expect(props.onConnected).toHaveBeenCalledTimes(1); + await advance(30); + expect(networking.userConnectionPollCall).toHaveBeenCalledTimes(3); + + fireEvent.click(screen.getByRole("button", { name: "Disconnect" })); + expect(props.onDisconnect).toHaveBeenCalledTimes(1); + }); + + it("stops polling and offers a restart when the device code expires", async () => { + networking.userConnectionStartCall.mockResolvedValue({ ...START_RESPONSE, expires_in: 12 }); + networking.userConnectionPollCall.mockResolvedValue({ status: "pending" }); + renderModal(); + await advance(0); + await advance(5); + await advance(5); + await advance(5); + + expect(networking.userConnectionPollCall).toHaveBeenCalledTimes(2); + expect(screen.getByRole("alert")).toHaveTextContent("The device code expired"); + await advance(30); + expect(networking.userConnectionPollCall).toHaveBeenCalledTimes(2); + + fireEvent.click(screen.getByRole("button", { name: "Start again" })); + await advance(0); + expect(networking.userConnectionStartCall).toHaveBeenCalledTimes(2); + expect(screen.getByTestId("github-device-code")).toBeInTheDocument(); + }); + + it.each([ + ["denied", "denied on GitHub"], + ["no_copilot_seat", "does not have GitHub Copilot access"], + ["expired", "The device code expired"], + ])("shows the %s error and stops polling", async (status, message) => { + networking.userConnectionPollCall.mockResolvedValue({ status }); + const props = renderModal(); + await advance(0); + await advance(5); + + expect(screen.getByRole("alert")).toHaveTextContent(message); + expect(props.onConnected).not.toHaveBeenCalled(); + await advance(30); + expect(networking.userConnectionPollCall).toHaveBeenCalledTimes(1); + }); + + it("shows the proxy error when the start request fails", async () => { + networking.userConnectionStartCall.mockRejectedValue(new Error("Credential not found")); + renderModal(); + await advance(0); + + expect(screen.getByRole("alert")).toHaveTextContent("Credential not found"); + expect(networking.userConnectionPollCall).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/model_add/GithubCopilotConnectModal.tsx b/ui/litellm-dashboard/src/components/model_add/GithubCopilotConnectModal.tsx new file mode 100644 index 000000000000..26d809deaf94 --- /dev/null +++ b/ui/litellm-dashboard/src/components/model_add/GithubCopilotConnectModal.tsx @@ -0,0 +1,136 @@ +"use client"; + +import { useEffect, useReducer } from "react"; + +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { userConnectionPollCall, userConnectionStartCall } from "@/components/networking"; +import CopyButton from "@/components/shared/CopyButton"; +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { extractProxyErrorMessage } from "@/lib/http/client"; + +import { connectFlowReducer, failureMessage, initialConnectFlowState } from "./github_copilot_connect_flow"; + +interface GithubCopilotConnectModalProps { + readonly credentialName: string; + readonly onClose: () => void; + readonly onConnected: () => void; + readonly onDisconnect: () => void; +} + +export default function GithubCopilotConnectModal({ + credentialName, + onClose, + onConnected, + onDisconnect, +}: GithubCopilotConnectModalProps) { + const { accessToken } = useAuthorized(); + const [state, dispatch] = useReducer(connectFlowReducer, initialConnectFlowState); + + useEffect(() => { + if (state.kind !== "starting" || !accessToken) { + return; + } + let cancelled = false; + userConnectionStartCall(accessToken, credentialName) + .then((response) => !cancelled && dispatch({ type: "started", response, now: Date.now() })) + .catch((error: unknown) => !cancelled && dispatch({ type: "errored", message: extractProxyErrorMessage(error) })); + return () => { + cancelled = true; + }; + }, [state, accessToken, credentialName]); + + useEffect(() => { + if (state.kind !== "awaiting" || !accessToken) { + return; + } + let cancelled = false; + const timer = setTimeout(() => { + if (Date.now() >= state.expiresAt) { + dispatch({ type: "timed_out" }); + return; + } + userConnectionPollCall(accessToken, credentialName) + .then((response) => !cancelled && dispatch({ type: "polled", response })) + .catch( + (error: unknown) => !cancelled && dispatch({ type: "errored", message: extractProxyErrorMessage(error) }), + ); + }, state.intervalSeconds * 1000); + return () => { + cancelled = true; + clearTimeout(timer); + }; + }, [state, accessToken, credentialName]); + + useEffect(() => { + if (state.kind === "connected") { + onConnected(); + } + }, [state.kind, onConnected]); + + return ( +

!open && onClose()}> + + + Connect GitHub Copilot + Credential: {credentialName} + + + {state.kind === "starting" &&

Requesting a device code from GitHub...

} + + {state.kind === "awaiting" && ( +
+

+ Open{" "} + + {state.verificationUri} + {" "} + and enter this code: +

+
+ + {state.userCode} + + +
+

Waiting for you to approve the request on GitHub...

+
+ )} + + {state.kind === "connected" &&

Connected as @{state.githubLogin}

} + + {state.kind === "failed" && ( +

+ {failureMessage(state)} +

+ )} + + + {state.kind === "failed" && ( + + )} + {state.kind === "connected" && ( + + )} + + + + + ); +} diff --git a/ui/litellm-dashboard/src/components/model_add/UserConnectionsPanel.test.tsx b/ui/litellm-dashboard/src/components/model_add/UserConnectionsPanel.test.tsx new file mode 100644 index 000000000000..7f9dd4c519fb --- /dev/null +++ b/ui/litellm-dashboard/src/components/model_add/UserConnectionsPanel.test.tsx @@ -0,0 +1,50 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it, vi } from "vitest"; + +import { userConnectionsListCall } from "@/components/networking"; + +import UserConnectionsPanel from "./UserConnectionsPanel"; + +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: () => ({ accessToken: "token", userId: "u", userRole: "internal_user" }), +})); + +vi.mock("@/components/networking", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + userConnectionsListCall: vi.fn(), + userConnectionDeleteCall: vi.fn(), + }; +}); + +function renderPanel() { + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + return render( + + + , + ); +} + +describe("UserConnectionsPanel", () => { + it("shows an error state with retry instead of an empty list when the query fails", async () => { + vi.mocked(userConnectionsListCall).mockRejectedValue(new Error("boom")); + renderPanel(); + expect(await screen.findByText("Could not load your connections.")).toBeInTheDocument(); + expect(screen.queryByText("No credentials need a personal connection.")).not.toBeInTheDocument(); + }); + + it("refetches when retry is clicked", async () => { + const user = userEvent.setup(); + vi.mocked(userConnectionsListCall) + .mockRejectedValueOnce(new Error("boom")) + .mockResolvedValueOnce({ connections: [] }); + renderPanel(); + await user.click(await screen.findByRole("button", { name: "Retry" })); + expect(await screen.findByText("No credentials need a personal connection.")).toBeInTheDocument(); + expect(vi.mocked(userConnectionsListCall).mock.calls.length).toBeGreaterThanOrEqual(2); + }); +}); diff --git a/ui/litellm-dashboard/src/components/model_add/UserConnectionsPanel.tsx b/ui/litellm-dashboard/src/components/model_add/UserConnectionsPanel.tsx new file mode 100644 index 000000000000..bdf5c3d523dd --- /dev/null +++ b/ui/litellm-dashboard/src/components/model_add/UserConnectionsPanel.tsx @@ -0,0 +1,130 @@ +"use client"; + +import { useQueryClient } from "@tanstack/react-query"; +import { useCallback, useState } from "react"; + +import { userConnectionsKeys, useUserConnections } from "@/app/(dashboard)/hooks/credentials/useUserConnections"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { userConnectionDeleteCall, UserProviderConnection } from "@/components/networking"; +import { Providers } from "@/components/provider_info_helpers"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; +import { toast } from "@/lib/toast"; + +import GithubCopilotConnectModal from "./GithubCopilotConnectModal"; + +const providerLabel = (provider: string): string => + Providers[provider.toUpperCase() as keyof typeof Providers] ?? + Object.entries(Providers).find(([key]) => key.toLowerCase() === provider.toLowerCase())?.[1] ?? + provider; + +export default function UserConnectionsPanel() { + const { accessToken } = useAuthorized(); + const queryClient = useQueryClient(); + const { data, isLoading, isError, refetch } = useUserConnections(); + const [connectingCredential, setConnectingCredential] = useState(null); + const connections: readonly UserProviderConnection[] = data?.connections ?? []; + const showEmpty = !isLoading && !isError && connections.length === 0; + + const refreshConnections = useCallback( + () => queryClient.invalidateQueries({ queryKey: userConnectionsKeys.all }), + [queryClient], + ); + + const disconnect = async (credentialName: string) => { + if (!accessToken) { + return; + } + try { + await userConnectionDeleteCall(accessToken, credentialName); + toast.success("GitHub Copilot disconnected"); + setConnectingCredential(null); + await refreshConnections(); + } catch (error) { + toast.error("Failed to disconnect GitHub Copilot"); + } + }; + + return ( +
+
+

Your connections

+

+ Credentials that use your own account. Connect once, and your requests to models on that credential use your + own access. +

+
+ + + + Credential + Provider + Status + Actions + + + + {isLoading && ( + + + Loading connections... + + + )} + {isError && ( + + +
+ Could not load your connections. + +
+ + + )} + {showEmpty && ( + + + No credentials need a personal connection. + + + )} + {connections.map((connection) => ( + + {connection.credential_name} + {providerLabel(connection.provider)} + + {connection.connected ? ( + Connected as @{connection.github_login} + ) : ( + Not connected + )} + + + {connection.connected ? ( + + ) : ( + + )} + + + ))} + +
+ {connectingCredential !== null && ( + setConnectingCredential(null)} + onConnected={refreshConnections} + onDisconnect={() => disconnect(connectingCredential)} + /> + )} +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/model_add/github_copilot_connect_flow.test.ts b/ui/litellm-dashboard/src/components/model_add/github_copilot_connect_flow.test.ts new file mode 100644 index 000000000000..31a32bc0954f --- /dev/null +++ b/ui/litellm-dashboard/src/components/model_add/github_copilot_connect_flow.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it } from "vitest"; + +import { connectFlowReducer, type ConnectFlowState, initialConnectFlowState } from "./github_copilot_connect_flow"; + +const started = (verificationUri = "https://github.com/login/device"): ConnectFlowState => + connectFlowReducer(initialConnectFlowState, { + type: "started", + response: { user_code: "ABCD-1234", verification_uri: verificationUri, expires_in: 900, interval: 5 }, + now: 1_000, + }); + +describe("connectFlowReducer", () => { + it("moves from starting to awaiting with the code, expiry and interval from the start response", () => { + const expected: ConnectFlowState = { + kind: "awaiting", + attempt: 0, + pollCount: 0, + userCode: "ABCD-1234", + verificationUri: "https://github.com/login/device", + expiresAt: 901_000, + intervalSeconds: 5, + }; + expect(started()).toEqual(expected); + }); + + it("never links anywhere but github.com", () => { + const state = started("https://evil.example/login/device"); + expect(state.kind === "awaiting" && state.verificationUri).toBe("https://github.com/login/device"); + }); + + it("keeps waiting on pending and slows down by the server interval or by five seconds", () => { + const pending = connectFlowReducer(started(), { type: "polled", response: { status: "pending" } }); + expect(pending).toMatchObject({ kind: "awaiting", pollCount: 1, intervalSeconds: 5 }); + + const slowed = connectFlowReducer(pending, { type: "polled", response: { status: "slow_down", interval: 12 } }); + expect(slowed).toMatchObject({ kind: "awaiting", pollCount: 2, intervalSeconds: 12 }); + + const slowedAgain = connectFlowReducer(slowed, { type: "polled", response: { status: "slow_down" } }); + expect(slowedAgain).toMatchObject({ kind: "awaiting", intervalSeconds: 17 }); + }); + + it.each(["expired", "denied", "no_copilot_seat"] as const)("fails with %s", (status) => { + expect(connectFlowReducer(started(), { type: "polled", response: { status } })).toEqual({ + kind: "failed", + attempt: 0, + reason: status, + }); + }); + + it("connects with the GitHub login", () => { + expect( + connectFlowReducer(started(), { type: "polled", response: { status: "connected", github_login: "octocat" } }), + ).toEqual({ kind: "connected", attempt: 0, githubLogin: "octocat" }); + }); + + it("expires locally once the device code lifetime has passed", () => { + expect(connectFlowReducer(started(), { type: "timed_out" })).toEqual({ + kind: "failed", + attempt: 0, + reason: "expired", + }); + }); + + it("restarts with a new attempt after a failure", () => { + const failed = connectFlowReducer(started(), { type: "errored", message: "boom" }); + const expected: ConnectFlowState = { kind: "failed", attempt: 0, reason: "error", message: "boom" }; + expect(failed).toEqual(expected); + expect(connectFlowReducer(failed, { type: "restarted" })).toEqual({ kind: "starting", attempt: 1 }); + }); + + it("ignores a late poll result after the flow left the awaiting state", () => { + const connected = connectFlowReducer(started(), { + type: "polled", + response: { status: "connected", github_login: "octocat" }, + }); + expect(connectFlowReducer(connected, { type: "polled", response: { status: "expired" } })).toBe(connected); + expect(connectFlowReducer(connected, { type: "errored", message: "late" })).toBe(connected); + }); +}); diff --git a/ui/litellm-dashboard/src/components/model_add/github_copilot_connect_flow.ts b/ui/litellm-dashboard/src/components/model_add/github_copilot_connect_flow.ts new file mode 100644 index 000000000000..f9532d7b0421 --- /dev/null +++ b/ui/litellm-dashboard/src/components/model_add/github_copilot_connect_flow.ts @@ -0,0 +1,99 @@ +import type { UserConnectionPollResponse, UserConnectionStartResponse } from "@/components/networking"; + +export const GITHUB_DEVICE_LOGIN_URL = "https://github.com/login/device"; +const SLOW_DOWN_INCREMENT_SECONDS = 5; + +export type ConnectFailureReason = "expired" | "denied" | "no_copilot_seat" | "error"; + +export type ConnectFlowState = + | { readonly kind: "starting"; readonly attempt: number } + | { + readonly kind: "awaiting"; + readonly attempt: number; + readonly pollCount: number; + readonly userCode: string; + readonly verificationUri: string; + readonly expiresAt: number; + readonly intervalSeconds: number; + } + | { readonly kind: "connected"; readonly attempt: number; readonly githubLogin: string } + | { + readonly kind: "failed"; + readonly attempt: number; + readonly reason: ConnectFailureReason; + readonly message?: string; + }; + +export type ConnectFlowEvent = + | { readonly type: "started"; readonly response: UserConnectionStartResponse; readonly now: number } + | { readonly type: "polled"; readonly response: UserConnectionPollResponse } + | { readonly type: "timed_out" } + | { readonly type: "errored"; readonly message: string } + | { readonly type: "restarted" }; + +export const initialConnectFlowState: ConnectFlowState = { kind: "starting", attempt: 0 }; + +const safeVerificationUri = (uri: string): string => + uri.startsWith("https://github.com/") ? uri : GITHUB_DEVICE_LOGIN_URL; + +const afterPoll = ( + state: Extract, + response: UserConnectionPollResponse, +): ConnectFlowState => { + switch (response.status) { + case "pending": + return { ...state, pollCount: state.pollCount + 1 }; + case "slow_down": + return { + ...state, + pollCount: state.pollCount + 1, + intervalSeconds: response.interval ?? state.intervalSeconds + SLOW_DOWN_INCREMENT_SECONDS, + }; + case "connected": + return { kind: "connected", attempt: state.attempt, githubLogin: response.github_login ?? "" }; + case "expired": + case "denied": + case "no_copilot_seat": + return { kind: "failed", attempt: state.attempt, reason: response.status }; + } +}; + +export const connectFlowReducer = (state: ConnectFlowState, event: ConnectFlowEvent): ConnectFlowState => { + switch (event.type) { + case "restarted": + return { kind: "starting", attempt: state.attempt + 1 }; + case "errored": + return state.kind === "connected" + ? state + : { kind: "failed", attempt: state.attempt, reason: "error", message: event.message }; + case "started": + return state.kind !== "starting" + ? state + : { + kind: "awaiting", + attempt: state.attempt, + pollCount: 0, + userCode: event.response.user_code, + verificationUri: safeVerificationUri(event.response.verification_uri), + expiresAt: event.now + event.response.expires_in * 1000, + intervalSeconds: event.response.interval, + }; + case "polled": + return state.kind === "awaiting" ? afterPoll(state, event.response) : state; + case "timed_out": + return state.kind === "awaiting" ? { kind: "failed", attempt: state.attempt, reason: "expired" } : state; + } +}; + +export const failureMessage = (state: Extract): string => { + switch (state.reason) { + case "expired": + return "The device code expired before it was approved. Start again to get a new code."; + case "denied": + return "The request was denied on GitHub. Start again if this was a mistake."; + case "no_copilot_seat": + return "This GitHub account does not have GitHub Copilot access, so nothing was saved."; + case "error": + return state.message ?? "Something went wrong while connecting to GitHub."; + } +}; diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index bf35e22f2281..f110d45e4103 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -2575,6 +2575,54 @@ export const credentialDeleteCall = async (accessToken: string, credentialName: } }; +export interface UserProviderConnection { + credential_name: string; + provider: string; + connected: boolean; + github_login: string | null; + connected_at: string | null; +} + +export interface UserProviderConnectionsResponse { + connections: UserProviderConnection[]; +} + +export interface UserConnectionStartResponse { + user_code: string; + verification_uri: string; + expires_in: number; + interval: number; +} + +export type UserConnectionPollStatus = "pending" | "slow_down" | "expired" | "denied" | "no_copilot_seat" | "connected"; + +export interface UserConnectionPollResponse { + status: UserConnectionPollStatus; + interval?: number | null; + github_login?: string | null; +} + +const userConnectionPath = (credentialName: string): string => + `/credentials/${encodeURIComponent(credentialName)}/user_connection`; + +export const userConnectionsListCall = (accessToken: string): Promise => + apiClient.get("/credentials/user_connections", { accessToken }); + +export const userConnectionStartCall = ( + accessToken: string, + credentialName: string, +): Promise => + apiClient.post(`${userConnectionPath(credentialName)}/start`, { accessToken }); + +export const userConnectionPollCall = ( + accessToken: string, + credentialName: string, +): Promise => + apiClient.post(`${userConnectionPath(credentialName)}/poll`, { accessToken }); + +export const userConnectionDeleteCall = (accessToken: string, credentialName: string): Promise => + apiClient.delete(userConnectionPath(credentialName), { accessToken }); + export const credentialUpdateCall = async ( accessToken: string, credentialName: string, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index fee1e39a19d7..0d101e5049f0 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -3786,6 +3786,26 @@ export interface paths { patch?: never; trace?: never; }; + "/credentials/user_connections": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * List User Connections + * @description List the calling user's per-user provider connections. + */ + get: operations["list_user_connections_credentials_user_connections_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/credentials/{credential_name}": { parameters: { query?: never; @@ -3832,6 +3852,66 @@ export interface paths { patch?: never; trace?: never; }; + "/credentials/{credential_name}/user_connection": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post?: never; + /** + * Delete User Connection + * @description Disconnect the calling user's stored GitHub token for a per-user credential. Idempotent. + */ + delete: operations["delete_user_connection_credentials__credential_name__user_connection_delete"]; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/credentials/{credential_name}/user_connection/poll": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Poll User Connection + * @description Poll the device flow once and persist the connection on completion. + */ + post: operations["poll_user_connection_credentials__credential_name__user_connection_poll_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/credentials/{credential_name}/user_connection/start": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Start User Connection + * @description Begin a GitHub device flow for the calling user's connection to a per-user credential. + */ + post: operations["start_user_connection_credentials__credential_name__user_connection_start_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/cursor/chat/completions": { parameters: { query?: never; @@ -50003,6 +50083,34 @@ export interface components { */ severity: "info" | "warning" | "error"; }; + /** UserConnectionDeleteResponse */ + UserConnectionDeleteResponse: { + /** Status */ + status: string; + }; + /** UserConnectionPollResponse */ + UserConnectionPollResponse: { + /** Github Login */ + github_login?: string | null; + /** Interval */ + interval?: number | null; + /** + * Status + * @enum {string} + */ + status: "pending" | "slow_down" | "expired" | "denied" | "connected" | "no_copilot_seat"; + }; + /** UserConnectionStartResponse */ + UserConnectionStartResponse: { + /** Expires In */ + expires_in: number; + /** Interval */ + interval: number; + /** User Code */ + user_code: string; + /** Verification Uri */ + verification_uri: string; + }; /** * UserCreateResult * @description Outcome for one row of `POST /management/v1/users/bulk`. `teams` lists the teams the user was actually @@ -50146,6 +50254,24 @@ export interface components { /** Users */ users: components["schemas"]["LiteLLM_UserTableWithKeyCount"][]; }; + /** UserProviderConnection */ + UserProviderConnection: { + /** Connected */ + connected: boolean; + /** Connected At */ + connected_at?: string | null; + /** Credential Name */ + credential_name: string; + /** Github Login */ + github_login?: string | null; + /** Provider */ + provider: string; + }; + /** UserProviderConnectionsResponse */ + UserProviderConnectionsResponse: { + /** Connections */ + connections: components["schemas"]["UserProviderConnection"][]; + }; /** * UserUpdateResult * @description Result of a single user update operation @@ -57130,6 +57256,26 @@ export interface operations { }; }; }; + list_user_connections_credentials_user_connections_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UserProviderConnectionsResponse"]; + }; + }; + }; + }; delete_credential_credentials__credential_name__delete: { parameters: { query?: never; @@ -57230,6 +57376,102 @@ export interface operations { }; }; }; + delete_user_connection_credentials__credential_name__user_connection_delete: { + parameters: { + query?: never; + header?: never; + path: { + /** @description The credential name, percent-decoded */ + credential_name: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UserConnectionDeleteResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + poll_user_connection_credentials__credential_name__user_connection_poll_post: { + parameters: { + query?: never; + header?: never; + path: { + /** @description The credential name, percent-decoded */ + credential_name: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UserConnectionPollResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + start_user_connection_credentials__credential_name__user_connection_start_post: { + parameters: { + query?: never; + header?: never; + path: { + /** @description The credential name, percent-decoded */ + credential_name: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UserConnectionStartResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; cursor_chat_completions_cursor_chat_completions_post: { parameters: { query?: never; From 2aab235a2212e6ff6e2c79dff4c4f13c5ddb4f09 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 15:47:35 +0000 Subject: [PATCH 02/38] fix(ui): make per-user GitHub Copilot OAuth credential-only in Add Model --- .../AddModelForm.integration.test.tsx | 118 +++++++++++++++++- .../add_model/provider_auth_types.ts | 1 + .../provider_specific_fields.test.tsx | 44 +++++++ 3 files changed, 161 insertions(+), 2 deletions(-) diff --git a/ui/litellm-dashboard/src/components/add_model/AddModelForm.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/AddModelForm.integration.test.tsx index dfecff1c408a..673e99ab99f7 100644 --- a/ui/litellm-dashboard/src/components/add_model/AddModelForm.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AddModelForm.integration.test.tsx @@ -67,6 +67,16 @@ vi.mock("../networking", async () => { { key: "api_key", label: "Delegated Access Token", field_type: "password" }, ], }, + { + provider: "GITHUB_COPILOT", + provider_display_name: "Github Copilot", + litellm_provider: "github_copilot", + default_model_placeholder: "github_copilot/chat", + credential_fields: [ + { key: "api_base", label: "API Base", field_type: "text" }, + { key: "api_key", label: "API Key", field_type: "password" }, + ], + }, ]), }; }); @@ -102,6 +112,16 @@ vi.mock("@/app/(dashboard)/hooks/providers/useProviderFields", () => ({ { key: "api_key", label: "Delegated Access Token", field_type: "password" }, ], }, + { + provider: "GITHUB_COPILOT", + provider_display_name: "Github Copilot", + litellm_provider: "github_copilot", + default_model_placeholder: "github_copilot/chat", + credential_fields: [ + { key: "api_base", label: "API Base", field_type: "text" }, + { key: "api_key", label: "API Key", field_type: "password" }, + ], + }, ], isLoading: false, error: null, @@ -470,11 +490,11 @@ describe("AddModelForm", () => { }); describe("credential-only provider auth types", () => { - const renderAsAdmin = async () => { + const renderAsAdmin = async (selectedProvider = Providers.MICROSOFT_365_COPILOT) => { const mockUseAuthorized = vi.mocked(await import("@/app/(dashboard)/hooks/useAuthorized")); mockUseAuthorized.default.mockReturnValue(mockAuthorizedUser("proxy_admin", "user-1", true)); const props = createTestProps(); - props.selectedProvider = Providers.MICROSOFT_365_COPILOT; + props.selectedProvider = selectedProvider; renderWithProviders(); await screen.findByText("Existing Credentials"); return props; @@ -593,6 +613,100 @@ describe("AddModelForm", () => { api_key: "delegated-token", }); }); + + it("creates a per-user GitHub credential and attaches it without sending its auth type inline", async () => { + const user = userEvent.setup(); + vi.mocked(credentialCreateCall).mockClear(); + vi.mocked(modelCreateCall).mockClear(); + const props = await renderAsAdmin(Providers.GITHUB_COPILOT); + + await chooseSelectOption( + user, + screen.getByRole("combobox", { name: "Auth Type:" }), + "Per-user GitHub OAuth", + ); + await user.click(screen.getByRole("button", { name: "Create credential" })); + const dialog = await screen.findByRole("dialog"); + const providerSelect = within(dialog).getByPlaceholderText("Select a provider"); + expect(providerSelect).toHaveValue(Providers.GITHUB_COPILOT); + expect(providerSelect).toBeDisabled(); + expect(within(dialog).getByRole("combobox", { name: "Auth Type:" })).toHaveTextContent( + "Per-user GitHub OAuth", + ); + fireEvent.change(within(dialog).getByLabelText("Credential Name:"), { + target: { value: "github-per-user" }, + }); + await user.click(within(dialog).getByRole("button", { name: "Add Credential" })); + + await waitFor(() => + expect(credentialCreateCall).toHaveBeenCalledWith("test-access-token", { + credential_name: "github-per-user", + credential_values: { github_copilot_auth_type: "per_user_oauth" }, + credential_info: { custom_llm_provider: Providers.GITHUB_COPILOT }, + }), + ); + expect(props.form.getValues("litellm_credential_name")).toBe("github-per-user"); + + props.handleOk.mockImplementation(async () => { + await handleAddModelSubmit( + { + litellm_credential_name: props.form.getValues("litellm_credential_name"), + custom_llm_provider: Providers.GITHUB_COPILOT, + model_mappings: [ + { + public_name: "copilot-model", + litellm_model: "github_copilot/chat", + }, + ], + }, + "test-access-token", + { resetFields: vi.fn() }, + ); + return true; + }); + await user.click(screen.getByRole("button", { name: "Add Model" })); + + await waitFor(() => expect(modelCreateCall).toHaveBeenCalledOnce()); + const modelParams = vi.mocked(modelCreateCall).mock.calls[0][1].litellm_params; + expect(modelParams).toMatchObject({ litellm_credential_name: "github-per-user" }); + expect(modelParams).not.toHaveProperty("github_copilot_auth_type"); + }); + + it("keeps shared GitHub device login inline when adding a model", async () => { + const user = userEvent.setup(); + vi.mocked(credentialCreateCall).mockClear(); + vi.mocked(modelCreateCall).mockClear(); + const props = await renderAsAdmin(Providers.GITHUB_COPILOT); + + expect(await screen.findByRole("combobox", { name: "Auth Type:" })).toHaveTextContent("Shared device login"); + expect(screen.queryByRole("button", { name: "Create credential" })).not.toBeInTheDocument(); + fireEvent.change(screen.getByLabelText("API Key"), { target: { value: "shared-device-token" } }); + props.handleOk.mockImplementation(async () => { + await handleAddModelSubmit( + { + api_key: "shared-device-token", + custom_llm_provider: Providers.GITHUB_COPILOT, + model_mappings: [ + { + public_name: "copilot-model", + litellm_model: "github_copilot/chat", + }, + ], + }, + "test-access-token", + { resetFields: vi.fn() }, + ); + return true; + }); + + await user.click(screen.getByRole("button", { name: "Add Model" })); + + await waitFor(() => expect(modelCreateCall).toHaveBeenCalledOnce()); + expect(credentialCreateCall).not.toHaveBeenCalled(); + expect(vi.mocked(modelCreateCall).mock.calls[0][1].litellm_params).toMatchObject({ + api_key: "shared-device-token", + }); + }); }); describe("cache control bindings reach the parent form store", () => { diff --git a/ui/litellm-dashboard/src/components/add_model/provider_auth_types.ts b/ui/litellm-dashboard/src/components/add_model/provider_auth_types.ts index 95cf19e82c02..d6ac1a9bc859 100644 --- a/ui/litellm-dashboard/src/components/add_model/provider_auth_types.ts +++ b/ui/litellm-dashboard/src/components/add_model/provider_auth_types.ts @@ -29,6 +29,7 @@ export const PROVIDER_AUTH_TYPES: Partial { { key: "api_key", label: "Delegated Access Token", field_type: "password" }, ], }, + { + provider: "GITHUB_COPILOT", + provider_display_name: Providers.GITHUB_COPILOT, + litellm_provider: "github_copilot", + default_model_placeholder: "github_copilot/chat", + credential_fields: [ + { key: "api_base", label: "API Base", field_type: "text" }, + { key: "api_key", label: "API Key", field_type: "password" }, + ], + }, ]), }; }); @@ -182,6 +193,15 @@ const VertexCredentialsProbe = () => { return {String(watch("vertex_credentials") ?? "")}; }; +const GitHubCopilotAuthTypeProbe = () => { + const { watch } = useFormContext(); + return ( + + {String(watch(GITHUB_COPILOT_AUTH_TYPE_KEY) ?? "")} + + ); +}; + const ValidationForm = ({ children }: { readonly children: ReactNode }) => { const form = useFormContext(); return ( @@ -526,6 +546,30 @@ describe("ProviderSpecificFields", () => { expect(screen.getByLabelText("Delegated Access Token")).toBeInTheDocument(); }); + it("offers per-user GitHub OAuth as a credential-only model auth type without mounting its fixed value", async () => { + const onCreateCredential = vi.fn(); + const queryClient = createQueryClient(); + render( + + + + + + , + ); + + expect(await screen.findByRole("combobox", { name: "Auth Type:" })).toHaveTextContent("Per-user GitHub OAuth"); + expect(screen.getByRole("button", { name: "Create credential" })).toBeInTheDocument(); + expect(screen.getByTestId("github-copilot-auth-type")).toBeEmptyDOMElement(); + await userEvent.click(screen.getByRole("button", { name: "Create credential" })); + expect(onCreateCredential).toHaveBeenCalledWith("per_user_oauth"); + }); + it("switches Microsoft 365 Copilot fields to a static delegated access token", async () => { const user = userEvent.setup(); const queryClient = createQueryClient(); From a696458b66d6102664a2b70ed9a2c1f0c1bfcd80 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 16:18:09 +0000 Subject: [PATCH 03/38] fix(proxy): render LiteLLM_UserProviderCredentials prisma spans --- litellm/integrations/otel/model/spans.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/integrations/otel/model/spans.py b/litellm/integrations/otel/model/spans.py index 762efa46ebb3..a5796006d5e6 100644 --- a/litellm/integrations/otel/model/spans.py +++ b/litellm/integrations/otel/model/spans.py @@ -293,6 +293,7 @@ class PostgresOperation: "LiteLLM_BackgroundInteractionSettlement", "LiteLLM_ErrorLogs", "LiteLLM_UserNotifications", + "LiteLLM_UserProviderCredentials", "LiteLLM_TeamMembership", "LiteLLM_OrganizationMembership", "LiteLLM_InvitationLink", From 43f0b34ab7e24e84a9723a8e21f04e7604ba3a1a Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 19:59:45 +0000 Subject: [PATCH 04/38] style(ui): prettier-format add_model tests --- .../add_model/AddModelForm.integration.test.tsx | 10 ++-------- .../add_model/provider_specific_fields.test.tsx | 6 +----- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/ui/litellm-dashboard/src/components/add_model/AddModelForm.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/AddModelForm.integration.test.tsx index 112b223721b6..113ef54446fe 100644 --- a/ui/litellm-dashboard/src/components/add_model/AddModelForm.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AddModelForm.integration.test.tsx @@ -624,19 +624,13 @@ describe("AddModelForm", () => { vi.mocked(modelCreateCall).mockClear(); const props = await renderAsAdmin(Providers.GITHUB_COPILOT); - await chooseSelectOption( - user, - screen.getByRole("combobox", { name: "Auth Type:" }), - "Per-user GitHub OAuth", - ); + await chooseSelectOption(user, screen.getByRole("combobox", { name: "Auth Type:" }), "Per-user GitHub OAuth"); await user.click(screen.getByRole("button", { name: "Create credential" })); const dialog = await screen.findByRole("dialog"); const providerSelect = within(dialog).getByPlaceholderText("Select a provider"); expect(providerSelect).toHaveValue(Providers.GITHUB_COPILOT); expect(providerSelect).toBeDisabled(); - expect(within(dialog).getByRole("combobox", { name: "Auth Type:" })).toHaveTextContent( - "Per-user GitHub OAuth", - ); + expect(within(dialog).getByRole("combobox", { name: "Auth Type:" })).toHaveTextContent("Per-user GitHub OAuth"); fireEvent.change(within(dialog).getByLabelText("Credential Name:"), { target: { value: "github-per-user" }, }); diff --git a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.test.tsx b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.test.tsx index b453e45ac503..641e0e2c1d26 100644 --- a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.test.tsx @@ -195,11 +195,7 @@ const VertexCredentialsProbe = () => { const GitHubCopilotAuthTypeProbe = () => { const { watch } = useFormContext(); - return ( - - {String(watch(GITHUB_COPILOT_AUTH_TYPE_KEY) ?? "")} - - ); + return {String(watch(GITHUB_COPILOT_AUTH_TYPE_KEY) ?? "")}; }; const ValidationForm = ({ children }: { readonly children: ReactNode }) => { From 2da0ae9e6d0cb06258f2ec3ecbfd6752e0f9a421 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 20:06:18 +0000 Subject: [PATCH 05/38] fix(lint): annotate connection endpoints and narrow per-user credential excepts --- litellm/llms/github_copilot/per_user_auth.py | 4 ++-- litellm/proxy/credential_endpoints/endpoints.py | 8 ++++---- .../credential_endpoints/user_provider_credentials.py | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/litellm/llms/github_copilot/per_user_auth.py b/litellm/llms/github_copilot/per_user_auth.py index 1f454f3c41be..72035329ff38 100644 --- a/litellm/llms/github_copilot/per_user_auth.py +++ b/litellm/llms/github_copilot/per_user_auth.py @@ -15,7 +15,7 @@ from urllib.parse import urlparse import httpx -from pydantic import ConfigDict, TypeAdapter, with_config +from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.caching.in_memory_cache import InMemoryCache @@ -168,7 +168,7 @@ def _cached_session(cache_key: str) -> GithubCopilotUserSession | None: cached: Final[object] = _SESSION_CACHE.get_cache(cache_key) try: token, api_base, expires_at = _CACHED_SESSION_ENTRY.validate_python(cached) - except Exception: + except ValidationError: return None if expires_at - GITHUB_COPILOT_USER_TOKEN_SAFETY_MARGIN_SECONDS <= time.time(): return None diff --git a/litellm/proxy/credential_endpoints/endpoints.py b/litellm/proxy/credential_endpoints/endpoints.py index 5128b15c19d1..953d3803f55b 100644 --- a/litellm/proxy/credential_endpoints/endpoints.py +++ b/litellm/proxy/credential_endpoints/endpoints.py @@ -490,7 +490,7 @@ async def list_user_connections( request: Request, fastapi_response: Response, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI resolves the dependency from the default -): +) -> UserProviderConnectionsResponse: """List the calling user's per-user provider connections.""" from litellm.proxy.proxy_server import prisma_client @@ -541,7 +541,7 @@ async def start_user_connection( fastapi_response: Response, credential_name: str = Path(..., description="The credential name, percent-decoded"), user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI resolves the dependency from the default -): +) -> UserConnectionStartResponse: """Begin a GitHub device flow for the calling user's connection to a per-user credential.""" from litellm.llms.github_copilot.per_user_auth import astart_device_flow from litellm.proxy.proxy_server import prisma_client, user_api_key_cache @@ -582,7 +582,7 @@ async def poll_user_connection( fastapi_response: Response, credential_name: str = Path(..., description="The credential name, percent-decoded"), user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI resolves the dependency from the default -): +) -> UserConnectionPollResponse: """Poll the device flow once and persist the connection on completion.""" from litellm.llms.github_copilot.per_user_auth import ( acheck_copilot_seat, @@ -654,7 +654,7 @@ async def delete_user_connection( fastapi_response: Response, credential_name: str = Path(..., description="The credential name, percent-decoded"), user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI resolves the dependency from the default -): +) -> UserConnectionDeleteResponse: """Disconnect the calling user's stored GitHub token for a per-user credential. Idempotent.""" from litellm.llms.github_copilot.per_user_auth import evict_copilot_user_session from litellm.proxy.proxy_server import prisma_client, user_api_key_cache diff --git a/litellm/proxy/credential_endpoints/user_provider_credentials.py b/litellm/proxy/credential_endpoints/user_provider_credentials.py index 100dd09db8ea..bacfa03f461e 100644 --- a/litellm/proxy/credential_endpoints/user_provider_credentials.py +++ b/litellm/proxy/credential_endpoints/user_provider_credentials.py @@ -11,7 +11,7 @@ from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Final -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, ValidationError from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger @@ -74,7 +74,7 @@ def decode_user_provider_credential(stored: str) -> GithubCopilotUserConnectionP return None try: return GithubCopilotUserConnectionPayload.model_validate_json(decrypted) - except Exception: + except ValidationError: return None @@ -195,7 +195,7 @@ async def aget_user_provider_tokens( misses, where={"user_id": user_id}, ) - except Exception: + except Exception: # noqa: BLE001 # a DB outage reads as not connected so the caller gets a 401, never a 500 verbose_proxy_logger.exception("aget_user_provider_tokens: DB read failed for user_id=%s", user_id) return {} found: dict[str, str] = {} # mutable-ok: accumulates rows From 1562a3adccd262778eb4c1f0007f122a15ec6318 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 20:20:19 +0000 Subject: [PATCH 06/38] fix(lint): satisfy type-discipline gate for per-user copilot code --- .../get_llm_provider_logic.py | 14 ++++----- .../github_copilot/chat/transformation.py | 29 +++++++++---------- .../github_copilot/messages/transformation.py | 6 ++-- litellm/models/credentials.py | 18 ++++++++++-- .../user_provider_credentials.py | 10 +++---- 5 files changed, 42 insertions(+), 35 deletions(-) diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index ced4a6f9118c..256c06b658d7 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -803,18 +803,18 @@ def _get_openai_compatible_provider_info( user_session: Final = github_copilot_user_session_from(litellm_params) # pyright: ignore[reportUnknownArgumentType] # litellm_params arrives as an untyped request dict if user_session is not None: - api_base = user_session.api_base - dynamic_api_key = user_session.token + api_base = user_session.api_base # rebind-ok: resolves provider args in place + dynamic_api_key = user_session.token # rebind-ok: resolves provider args in place elif github_copilot_per_user_credential_name(litellm_params) is not None: # pyright: ignore[reportUnknownArgumentType] # litellm_params arrives as an untyped request dict # per-user deployments resolve no token at provider-info time; the # caller's session supplies base + key per request - api_base = api_base or DEFAULT_GITHUB_COPILOT_API_BASE - dynamic_api_key = None + api_base = api_base or DEFAULT_GITHUB_COPILOT_API_BASE # rebind-ok: resolves provider args in place + dynamic_api_key = None # rebind-ok: resolves provider args in place else: ( - api_base, - dynamic_api_key, - custom_llm_provider, + api_base, # rebind-ok: resolves provider args in place + dynamic_api_key, # rebind-ok: resolves provider args in place + custom_llm_provider, # rebind-ok: resolves provider args in place ) = litellm.GithubCopilotConfig().get_openai_compatible_provider_info( model, api_base, api_key, custom_llm_provider ) diff --git a/litellm/llms/github_copilot/chat/transformation.py b/litellm/llms/github_copilot/chat/transformation.py index b075d025c48b..12faa542119a 100644 --- a/litellm/llms/github_copilot/chat/transformation.py +++ b/litellm/llms/github_copilot/chat/transformation.py @@ -1,6 +1,6 @@ import json import os -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Final import httpx @@ -94,6 +94,14 @@ def _transform_messages( return transformed_messages + def _copilot_headers(self, session_token: str | None) -> Mapping[str, str]: + if session_token is not None: + return get_copilot_default_headers(session_token) + try: + return get_copilot_default_headers(self.authenticator.get_api_key()) + except GetAPIKeyError: + return {} + def validate_environment( self, headers: dict, @@ -104,25 +112,16 @@ def validate_environment( api_key: str | None = None, api_base: str | None = None, ) -> dict: - # Get base headers from parent - validated_headers = super().validate_environment( + parent_headers: Final = super().validate_environment( headers, model, messages, optional_params, litellm_params, api_key, api_base ) - - # Add Copilot-specific headers (editor-version, user-agent, etc.) user_session: Final = require_github_copilot_user_session(litellm_params) # pyright: ignore[reportUnknownArgumentType] # litellm_params arrives as an untyped request dict - if user_session is not None: - validated_headers = {**get_copilot_default_headers(user_session.token), **validated_headers} + session_token: Final = user_session.token if user_session is not None else None + validated_headers: Final = {**self._copilot_headers(session_token), **parent_headers} + if session_token is not None: # the caller's stored session token wins unconditionally, even over a # caller-supplied api_key or the parent's placeholder Authorization - validated_headers["Authorization"] = f"Bearer {user_session.token}" - else: - try: - copilot_api_key: Final = self.authenticator.get_api_key() - copilot_headers: Final = get_copilot_default_headers(copilot_api_key) - validated_headers = {**copilot_headers, **validated_headers} - except GetAPIKeyError: - pass # Will be handled later in the request flow + validated_headers["Authorization"] = f"Bearer {session_token}" # Add X-Initiator header based on message roles initiator: Final = self._determine_initiator(messages) diff --git a/litellm/llms/github_copilot/messages/transformation.py b/litellm/llms/github_copilot/messages/transformation.py index 76cb55d3633d..871f9186a640 100644 --- a/litellm/llms/github_copilot/messages/transformation.py +++ b/litellm/llms/github_copilot/messages/transformation.py @@ -125,12 +125,10 @@ def get_complete_url( resolution only if it was not provided. """ user_session: Final = require_github_copilot_user_session(litellm_params) - resolved = ( + resolved: Final = ( (user_session.api_base if user_session is not None else None) or api_base or self.authenticator.get_api_base() or DEFAULT_GITHUB_COPILOT_API_BASE ).rstrip("/") - if not resolved.endswith("/v1/messages"): - resolved = f"{resolved}/v1/messages" - return resolved + return resolved if resolved.endswith("/v1/messages") else f"{resolved}/v1/messages" diff --git a/litellm/models/credentials.py b/litellm/models/credentials.py index 9bdd4199e195..0bfed1240ebf 100644 --- a/litellm/models/credentials.py +++ b/litellm/models/credentials.py @@ -6,9 +6,9 @@ """ from collections.abc import Mapping -from typing import Literal +from typing import Literal, TypeAlias -from pydantic import Field, model_validator +from pydantic import ConfigDict, Field, model_validator from litellm.types.llms.base import LiteLLMBaseModel @@ -47,6 +47,8 @@ class UpdateCredentialItem(LiteLLMBaseModel): class UserProviderConnection(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + credential_name: str provider: str connected: bool @@ -55,24 +57,34 @@ class UserProviderConnection(LiteLLMBaseModel): class UserProviderConnectionsResponse(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + connections: list[UserProviderConnection] # mutable-ok: pydantic response model field class UserConnectionStartResponse(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + user_code: str verification_uri: str expires_in: int interval: int -UserConnectionPollStatus = Literal["pending", "slow_down", "expired", "denied", "connected", "no_copilot_seat"] +UserConnectionPollStatus: TypeAlias = Literal[ + "pending", "slow_down", "expired", "denied", "connected", "no_copilot_seat" +] class UserConnectionPollResponse(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + status: UserConnectionPollStatus interval: int | None = None github_login: str | None = None class UserConnectionDeleteResponse(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + status: str diff --git a/litellm/proxy/credential_endpoints/user_provider_credentials.py b/litellm/proxy/credential_endpoints/user_provider_credentials.py index bacfa03f461e..d6a32178d3b1 100644 --- a/litellm/proxy/credential_endpoints/user_provider_credentials.py +++ b/litellm/proxy/credential_endpoints/user_provider_credentials.py @@ -38,7 +38,7 @@ class GithubCopilotUserConnectionPayload(BaseModel): - model_config = ConfigDict(extra="ignore") + model_config = ConfigDict(extra="ignore", frozen=True) access_token: str = Field(repr=False) github_login: str @@ -171,8 +171,8 @@ async def aget_user_provider_tokens( names: Final = tuple(dict.fromkeys(credential_names)) if not names: return {} - cached: dict[str, str] = {} # mutable-ok: accumulates hits and DB reads - misses: list[str] = [] # mutable-ok: accumulates cache misses + cached: Final[dict[str, str]] = {} # mutable-ok: accumulates hits and DB reads + misses: Final[list[str]] = [] # mutable-ok: accumulates cache misses for name in names: value = await cache.async_get_cache(_cache_key(user_id, name)) if value == _NOT_CONNECTED: @@ -198,9 +198,7 @@ async def aget_user_provider_tokens( except Exception: # noqa: BLE001 # a DB outage reads as not connected so the caller gets a 401, never a 500 verbose_proxy_logger.exception("aget_user_provider_tokens: DB read failed for user_id=%s", user_id) return {} - found: dict[str, str] = {} # mutable-ok: accumulates rows - for row in rows: - found[row.credential_name] = row.credential_b64 + found: Final = {row.credential_name: row.credential_b64 for row in rows} for name in misses: await cache.async_set_cache( _cache_key(user_id, name), From c136394dbafda80d751f48e3d9e1a23893c66906 Mon Sep 17 00:00:00 2001 From: yassin Date: Thu, 8 Oct 2026 21:31:06 +0000 Subject: [PATCH 07/38] fix(types): clear basedpyright and strict-lint gate regressions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/__init__.py | 4 +- .../pass_through/messages/handler.py | 23 +++- .../github_copilot/chat/transformation.py | 3 +- litellm/llms/github_copilot/per_user_auth.py | 111 +++++++++++++++--- .../responses/transformation.py | 40 +++---- litellm/proxy/auth/route_checks.py | 4 +- .../proxy/credential_endpoints/endpoints.py | 15 ++- .../user_provider_credentials.py | 29 ++++- litellm/proxy/litellm_pre_call_utils.py | 23 ++-- litellm/responses/main.py | 52 ++++---- litellm/utils.py | 8 +- 11 files changed, 209 insertions(+), 103 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 5c27795123d8..841ad98d84e2 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1396,8 +1396,8 @@ def _build_models_by_provider() -> dict: from .llms.custom_httpx.async_client_cleanup import close_litellm_async_clients from .exceptions import ( AuthenticationError, - CallerCredentialAuthenticationError, - CallerCredentialRateLimitError, + CallerCredentialAuthenticationError as CallerCredentialAuthenticationError, + CallerCredentialRateLimitError as CallerCredentialRateLimitError, InvalidRequestError, BadRequestError, ImageFetchError, diff --git a/litellm/llms/anthropic/pass_through/messages/handler.py b/litellm/llms/anthropic/pass_through/messages/handler.py index d19f0aa4b36e..ec5e2a7646a1 100644 --- a/litellm/llms/anthropic/pass_through/messages/handler.py +++ b/litellm/llms/anthropic/pass_through/messages/handler.py @@ -23,6 +23,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.types.llms.anthropic import AllAnthropicToolsValues from litellm.types.llms.anthropic_messages.anthropic_request import AnthropicMetadata from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, @@ -514,7 +515,9 @@ def anthropic_messages_handler( metadata=metadata, stop_sequences=stop_sequences, stream=stream, - system=system, + system=cast( # cast-ok: anthropic accepts list system blocks; handler signature is narrower + "str | None", system + ), temperature=temperature, thinking=thinking, tool_choice=tool_choice, @@ -558,11 +561,15 @@ def anthropic_messages_handler( metadata=metadata, stop_sequences=stop_sequences, stream=stream, - system=system, + system=cast( # cast-ok: anthropic accepts list system blocks; handler signature is narrower + "str | None", system + ), temperature=temperature, thinking=thinking, tool_choice=tool_choice, - tools=tools, + tools=cast( # cast-ok: dict tools already fit the handler's union + "list[AllAnthropicToolsValues | dict[str, object]] | None", tools + ), top_k=top_k, top_p=top_p, _is_async=is_async, @@ -584,11 +591,13 @@ def anthropic_messages_handler( metadata=metadata, stop_sequences=stop_sequences, stream=stream, - system=system, + system=cast( # cast-ok: anthropic accepts list system blocks; handler signature is narrower + "str | None", system + ), temperature=temperature, thinking=thinking, tool_choice=tool_choice, - tools=tools, + tools=cast("list[dict[str, object]] | None", tools), # cast-ok: dict tools fit the adapter's declared union top_k=top_k, top_p=top_p, _is_async=is_async, @@ -620,7 +629,9 @@ def anthropic_messages_handler( ) return base_llm_http_handler.anthropic_messages_handler( model=model, - messages=strip_provider_specific_fields_from_anthropic_messages(messages), + messages=cast( # cast-ok: anthropic messages are dicts at runtime + "list[dict]", strip_provider_specific_fields_from_anthropic_messages(messages) + ), anthropic_messages_provider_config=anthropic_messages_provider_config, anthropic_messages_optional_request_params=dict(anthropic_messages_optional_request_params), _is_async=is_async, diff --git a/litellm/llms/github_copilot/chat/transformation.py b/litellm/llms/github_copilot/chat/transformation.py index 12faa542119a..fba975fc971f 100644 --- a/litellm/llms/github_copilot/chat/transformation.py +++ b/litellm/llms/github_copilot/chat/transformation.py @@ -21,6 +21,7 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding class GithubCopilotConfig(OpenAIConfig): @@ -300,7 +301,7 @@ def transform_response( messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: object, + encoding: "Encoding | None", api_key: str | None = None, json_mode: bool | None = None, ) -> "ModelResponse": diff --git a/litellm/llms/github_copilot/per_user_auth.py b/litellm/llms/github_copilot/per_user_auth.py index 72035329ff38..41f0867b01a9 100644 --- a/litellm/llms/github_copilot/per_user_auth.py +++ b/litellm/llms/github_copilot/per_user_auth.py @@ -11,7 +11,13 @@ import time from collections.abc import Mapping from dataclasses import dataclass, field -from typing import Final, Literal, TypeAlias +from typing import ( + Final, + Literal, + Protocol, + TypeAlias, + cast, # noqa: TID251 # casts pin the untyped litellm clients/caches to the local Protocols +) from urllib.parse import urlparse import httpx @@ -47,9 +53,48 @@ _SESSION_CACHE_MAX_SIZE: Final = 1000 _SYNC_LOCK_COUNT: Final = 64 -_SESSION_CACHE: Final = InMemoryCache( - max_size_in_memory=_SESSION_CACHE_MAX_SIZE, - default_ttl=GITHUB_COPILOT_USER_TOKEN_SAFETY_MARGIN_SECONDS, + +class _SessionCache(Protocol): + def get_cache(self, key: str) -> object: ... + + def set_cache(self, key: str, value: object, ttl: float | None = None) -> object: ... + + def delete_cache(self, key: str) -> object: ... + + +class _SyncGetClient(Protocol): + def get( + self, + url: str, + params: dict[str, str] | None = None, + headers: dict[str, str] | None = None, # mutable-ok: mirrors the untyped client's dict parameters + ) -> httpx.Response: ... + + +class _AsyncGitHubClient(Protocol): + async def get( + self, + url: str, + params: dict[str, str] | None = None, + headers: dict[str, str] | None = None, # mutable-ok: mirrors the untyped client's dict parameters + ) -> httpx.Response: ... + + async def post( + self, + url: str, + data: dict[str, str] | str | bytes | None = None, # mutable-ok: mirrors the untyped client's dict parameters + json: dict[str, str] | None = None, # mutable-ok: mirrors the untyped client's dict parameters + params: dict[str, str] | None = None, # mutable-ok: mirrors the untyped client's dict parameters + headers: dict[str, str] | None = None, # mutable-ok: mirrors the untyped client's dict parameters + ) -> httpx.Response: ... + + +_SESSION_CACHE: Final = cast( # cast-ok: pins the untyped module cache to _SessionCache + _SessionCache, + InMemoryCache( + max_size_in_memory=_SESSION_CACHE_MAX_SIZE, + default_ttl=GITHUB_COPILOT_USER_TOKEN_SAFETY_MARGIN_SECONDS, + ), ) _EXCHANGE_LOCKS: Final = tuple(threading.Lock() for _ in range(_SYNC_LOCK_COUNT)) _IN_FLIGHT: Final[ # mutable-ok: single-flight dedup registry mutated under its own locking @@ -262,7 +307,10 @@ def _do_exchange(github_token: str, credential_name: str, cache_key: str) -> Git import litellm try: - response: Final = litellm.module_level_client.get( + client: Final = cast( # cast-ok: pins the untyped litellm client to the local Protocol + _SyncGetClient, litellm.module_level_client + ) + response: Final = client.get( DEFAULT_GITHUB_API_KEY_URL, headers=github_api_headers(github_token), ) @@ -278,7 +326,9 @@ def _do_exchange(github_token: str, credential_name: str, cache_key: str) -> Git async def _do_aexchange(github_token: str, credential_name: str, cache_key: str) -> GithubCopilotUserSession: from litellm.llms.custom_httpx.http_handler import get_async_httpx_client - client: Final = get_async_httpx_client(_LLM_PROVIDER) + client: Final = cast( # cast-ok: pins the untyped litellm client to the local Protocol + _AsyncGitHubClient, get_async_httpx_client(_LLM_PROVIDER) + ) try: response: Final = await client.get( DEFAULT_GITHUB_API_KEY_URL, @@ -344,7 +394,9 @@ async def aexchange_github_token( async def astart_device_flow() -> GithubCopilotDeviceFlowStart: from litellm.llms.custom_httpx.http_handler import get_async_httpx_client - client: Final = get_async_httpx_client(_LLM_PROVIDER) + client: Final = cast( # cast-ok: pins the untyped litellm client to the local Protocol + _AsyncGitHubClient, get_async_httpx_client(_LLM_PROVIDER) + ) device_code_url: Final = os.getenv("GITHUB_COPILOT_DEVICE_CODE_URL", DEFAULT_GITHUB_DEVICE_CODE_URL) client_id: Final = os.getenv("GITHUB_COPILOT_CLIENT_ID", DEFAULT_GITHUB_CLIENT_ID) try: @@ -385,7 +437,9 @@ async def astart_device_flow() -> GithubCopilotDeviceFlowStart: async def apoll_device_flow(device_code: str) -> GithubCopilotDeviceFlowPoll: from litellm.llms.custom_httpx.http_handler import get_async_httpx_client - client: Final = get_async_httpx_client(_LLM_PROVIDER) + client: Final = cast( # cast-ok: pins the untyped litellm client to the local Protocol + _AsyncGitHubClient, get_async_httpx_client(_LLM_PROVIDER) + ) access_token_url: Final = os.getenv("GITHUB_COPILOT_ACCESS_TOKEN_URL", DEFAULT_GITHUB_ACCESS_TOKEN_URL) client_id: Final = os.getenv("GITHUB_COPILOT_CLIENT_ID", DEFAULT_GITHUB_CLIENT_ID) try: @@ -452,7 +506,9 @@ def _poll_payload_from_response(response: httpx.Response) -> GithubCopilotDevice async def afetch_github_login(github_token: str) -> str: from litellm.llms.custom_httpx.http_handler import get_async_httpx_client - client: Final = get_async_httpx_client(_LLM_PROVIDER) + client: Final = cast( # cast-ok: pins the untyped litellm client to the local Protocol + _AsyncGitHubClient, get_async_httpx_client(_LLM_PROVIDER) + ) try: response: Final = await client.get( "https://api.github.com/user", @@ -495,13 +551,18 @@ def github_copilot_auth_mode(litellm_credential_name: object, auth_type_value: o found; the request's own ``github_copilot_auth_type`` can never flip the mode either way on a named credential.""" if isinstance(litellm_credential_name, str) and litellm_credential_name: - import litellm + from litellm.litellm_core_utils.credential_accessor import CredentialAccessor - credential: Final[object] = litellm.CredentialAccessor.find_credential(litellm_credential_name) + credential: Final = CredentialAccessor.find_credential(litellm_credential_name) if credential is not None: values: Final[object] = getattr(credential, "credential_values", None) return isinstance(values, Mapping) and ( - values.get(GITHUB_COPILOT_AUTH_TYPE_KEY) == GITHUB_COPILOT_PER_USER_AUTH_TYPE + cast( # cast-ok: value is Mapping-checked or dict-shaped at runtime + Mapping[object, object], values + ).get( + GITHUB_COPILOT_AUTH_TYPE_KEY + ) + == GITHUB_COPILOT_PER_USER_AUTH_TYPE ) return auth_type_value == GITHUB_COPILOT_PER_USER_AUTH_TYPE if auth_type_value == GITHUB_COPILOT_PER_USER_AUTH_TYPE: @@ -522,14 +583,24 @@ def _connect_error(credential_name: str) -> CallerCredentialAuthenticationError: def _caller_connection(kwargs: Mapping[str, object], credential_name: str) -> tuple[str, str]: - secret_fields: Final[object] = kwargs.get("secret_fields") + secret_fields_raw: Final[object] = kwargs.get("secret_fields") + secret_fields: Final = ( + cast(Mapping[object, object], secret_fields_raw) # cast-ok: value is Mapping-checked or dict-shaped at runtime + if isinstance(secret_fields_raw, Mapping) + else None + ) user_id: Final[object] = ( - secret_fields.get("user_provider_credentials_user_id") if isinstance(secret_fields, Mapping) else None + secret_fields.get("user_provider_credentials_user_id") if secret_fields is not None else None ) - credentials: Final[object] = ( - secret_fields.get("user_provider_credentials") if isinstance(secret_fields, Mapping) else None + credentials_raw: Final[object] = ( + secret_fields.get("user_provider_credentials") if secret_fields is not None else None ) - token: Final[object] = credentials.get(credential_name) if isinstance(credentials, Mapping) else None + credentials: Final = ( + cast(Mapping[object, object], credentials_raw) # cast-ok: value is Mapping-checked or dict-shaped at runtime + if isinstance(credentials_raw, Mapping) + else None + ) + token: Final[object] = credentials.get(credential_name) if credentials is not None else None if not isinstance(user_id, str) or not user_id or not isinstance(token, str) or not token: raise _connect_error(credential_name) return user_id, token @@ -537,7 +608,7 @@ def _caller_connection(kwargs: Mapping[str, object], credential_name: str) -> tu def _params_value(source: object, key: str) -> object: if isinstance(source, Mapping): - return source.get(key) + return cast(Mapping[object, object], source).get(key) # cast-ok: isinstance above, Mapping values are object return getattr(source, key, None) @@ -554,7 +625,9 @@ def github_copilot_user_session_from(source: object) -> GithubCopilotUserSession if source is None: return None candidate: Final = ( - source.get(GITHUB_COPILOT_USER_SESSION_KWARG_KEY) + cast(Mapping[object, object], source).get( # cast-ok: value is Mapping-checked or dict-shaped at runtime + GITHUB_COPILOT_USER_SESSION_KWARG_KEY + ) if isinstance(source, Mapping) else getattr(source, GITHUB_COPILOT_USER_SESSION_KWARG_KEY, None) ) diff --git a/litellm/llms/github_copilot/responses/transformation.py b/litellm/llms/github_copilot/responses/transformation.py index e8d8163c326b..674b07952593 100644 --- a/litellm/llms/github_copilot/responses/transformation.py +++ b/litellm/llms/github_copilot/responses/transformation.py @@ -203,16 +203,14 @@ def validate_environment( user_session: Final = require_github_copilot_user_session(litellm_params) # pyright: ignore[reportUnknownArgumentType] # litellm_params arrives as an untyped request dict default_headers: Final = get_copilot_default_headers(user_session.token) if user_session is not None else None try: - if default_headers is None: - # Get GitHub Copilot API key via OAuth - api_key: Final = self.authenticator.get_api_key() - - if not api_key: - raise AuthenticationError( - model=model, - llm_provider="github_copilot", - message="GitHub Copilot API key is required. Please authenticate via OAuth Device Flow.", - ) + # Get GitHub Copilot API key via OAuth + api_key: Final = self.authenticator.get_api_key() if default_headers is None else "" + if default_headers is None and not api_key: + raise AuthenticationError( + model=model, + llm_provider="github_copilot", + message="GitHub Copilot API key is required. Please authenticate via OAuth Device Flow.", + ) # Get default headers (from copilot-api configuration) copilot_headers: Final = default_headers or get_copilot_default_headers(api_key) @@ -346,19 +344,15 @@ def _get_initiator(self, input_param: str | ResponseInputParam) -> str: return "user" # If input is a list, analyze items - if isinstance(input_param, list): - for item in input_param: - if not isinstance(item, dict): - continue - - # Check if item has no role (agent-initiated) - if "role" not in item or not item.get("role"): - return "agent" - - # Check if role is assistant (agent-initiated) - role = item.get("role") - if isinstance(role, str) and role.lower() == "assistant": - return "agent" + for item in input_param: + # Check if item has no role (agent-initiated) + if "role" not in item or not item.get("role"): + return "agent" + + # Check if role is assistant (agent-initiated) + role = item.get("role") + if role.lower() == "assistant": + return "agent" # Default to user-initiated return "user" diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 0ab1fb572c32..bddda7da5466 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -676,7 +676,9 @@ def get_request_method(request: Request | None) -> str | None: return None try: - method: Final = request.method + method: Final = cast( # cast-ok: request.method is str at runtime; tests hand a MagicMock + object, request.method + ) except (AttributeError, KeyError): return None if not isinstance(method, str): diff --git a/litellm/proxy/credential_endpoints/endpoints.py b/litellm/proxy/credential_endpoints/endpoints.py index 953d3803f55b..53db4426db9a 100644 --- a/litellm/proxy/credential_endpoints/endpoints.py +++ b/litellm/proxy/credential_endpoints/endpoints.py @@ -475,8 +475,12 @@ def _per_user_credential_names() -> tuple[str, ...]: return tuple( credential.credential_name for credential in litellm.credential_list - if isinstance(credential.credential_values, Mapping) - and credential.credential_values.get(GITHUB_COPILOT_AUTH_TYPE_KEY) == GITHUB_COPILOT_PER_USER_AUTH_TYPE + if cast( # cast-ok: credential_values is a plain dict at runtime + Mapping[object, object], credential.credential_values + ).get( + GITHUB_COPILOT_AUTH_TYPE_KEY + ) + == GITHUB_COPILOT_PER_USER_AUTH_TYPE ) @@ -850,11 +854,10 @@ async def update_credential( # Sync in-memory credential_list (skip if not in memory - e.g., proxy restarted) _sync_in_memory_credential(patch, credential_name, merged_credential.credential_name) - merged_values: Final = merged_credential.credential_values - still_per_user: Final = ( - isinstance(merged_values, Mapping) - and merged_values.get(GITHUB_COPILOT_AUTH_TYPE_KEY) == GITHUB_COPILOT_PER_USER_AUTH_TYPE + merged_values: Final = cast( # cast-ok: credential_values is a plain dict at runtime + Mapping[object, object], merged_credential.credential_values ) + still_per_user: Final = merged_values.get(GITHUB_COPILOT_AUTH_TYPE_KEY) == GITHUB_COPILOT_PER_USER_AUTH_TYPE if credential.credential_name != credential_name or not still_per_user: await _purge_user_connections_for_credential(credential_name) diff --git a/litellm/proxy/credential_endpoints/user_provider_credentials.py b/litellm/proxy/credential_endpoints/user_provider_credentials.py index d6a32178d3b1..ebd348f90528 100644 --- a/litellm/proxy/credential_endpoints/user_provider_credentials.py +++ b/litellm/proxy/credential_endpoints/user_provider_credentials.py @@ -9,7 +9,12 @@ import json from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Final +from typing import ( + TYPE_CHECKING, + Final, + Protocol, + cast, # noqa: TID251 # casts pin the untyped DualCache to the local _TokenCache Protocol +) from pydantic import BaseModel, ConfigDict, Field, ValidationError @@ -37,6 +42,16 @@ _NOT_CONNECTED: Final = USER_PROVIDER_CREDENTIAL_NOT_CONNECTED +class _TokenCache(Protocol): + async def async_get_cache(self, key: str) -> object: ... + + async def async_set_cache( + self, key: str, value: object, local_only: bool = False, ttl: float | None = None + ) -> object: ... + + async def async_delete_cache(self, key: str) -> object: ... + + class GithubCopilotUserConnectionPayload(BaseModel): model_config = ConfigDict(extra="ignore", frozen=True) @@ -154,7 +169,10 @@ async def invalidate_user_provider_credential_cache( user_id: str, credential_name: str, ) -> None: - await cache.async_delete_cache(_cache_key(user_id, credential_name)) + token_cache: Final = cast( # cast-ok: pins the untyped DualCache to the local _TokenCache Protocol + _TokenCache, cache + ) + await token_cache.async_delete_cache(_cache_key(user_id, credential_name)) @with_service_target("user_provider_connections") @@ -168,13 +186,16 @@ async def aget_user_provider_tokens( Cached values are the stored ciphertext or the ``_NOT_CONNECTED`` marker; plaintext tokens only ever live in the returned dict.""" + token_cache: Final = cast( # cast-ok: pins the untyped DualCache to the local _TokenCache Protocol + _TokenCache, cache + ) names: Final = tuple(dict.fromkeys(credential_names)) if not names: return {} cached: Final[dict[str, str]] = {} # mutable-ok: accumulates hits and DB reads misses: Final[list[str]] = [] # mutable-ok: accumulates cache misses for name in names: - value = await cache.async_get_cache(_cache_key(user_id, name)) + value = await token_cache.async_get_cache(_cache_key(user_id, name)) if value == _NOT_CONNECTED: continue if isinstance(value, str) and value: @@ -200,7 +221,7 @@ async def aget_user_provider_tokens( return {} found: Final = {row.credential_name: row.credential_b64 for row in rows} for name in misses: - await cache.async_set_cache( + await token_cache.async_set_cache( _cache_key(user_id, name), found.get(name, _NOT_CONNECTED), ttl=GITHUB_COPILOT_USER_CREDENTIAL_CACHE_TTL_SECONDS, diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 62028d84de60..4638cf5817f3 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -120,7 +120,7 @@ def _trace_id_from_otel_span(span: "OtelSpan | None") -> str | None: try: span_context: Final = span.get_span_context() is_valid: Final = span_context.is_valid - trace_id: Final = span_context.trace_id + trace_id: Final = cast(object, span_context.trace_id) # cast-ok: trace_id is int at runtime; tests mock it except AttributeError: return None if not is_valid or not isinstance(trace_id, int): @@ -2534,23 +2534,17 @@ async def add_litellm_data_to_request( if ( general_settings is not None and general_settings.get("use_x_forwarded_for") is True - and request is not None and hasattr(request, "headers") and "x-forwarded-for" in request.headers ): requester_ip_address = request.headers["x-forwarded-for"] - elif ( - request is not None - and hasattr(request, "client") - and hasattr(request.client, "host") - and request.client is not None - ): + elif hasattr(request, "client") and hasattr(request.client, "host") and request.client is not None: requester_ip_address = request.client.host data[_metadata_variable_name]["requester_ip_address"] = requester_ip_address # Add User-Agent user_agent = "" - if request is not None and hasattr(request, "headers") and "user-agent" in request.headers: + if hasattr(request, "headers") and "user-agent" in request.headers: user_agent = request.headers["user-agent"] data[_metadata_variable_name]["user_agent"] = user_agent @@ -2733,7 +2727,7 @@ def _per_user_credential_names_for_groups( names: Final[list[str]] = [] # mutable-ok: accumulates one name per per-user deployment for group in model_groups: for deployment in llm_router.get_model_list(model_name=group) or (): - litellm_params = deployment.get("litellm_params") + litellm_params = cast(object, deployment.get("litellm_params")) # cast-ok: deployment is dict-shaped here credential_name_obj: object = ( litellm_params.get("litellm_credential_name") if isinstance(litellm_params, Mapping) @@ -2744,11 +2738,10 @@ def _per_user_credential_names_for_groups( credential = CredentialAccessor.find_credential(credential_name_obj) if credential is None: continue - values = credential.credential_values - if ( - isinstance(values, Mapping) - and values.get(GITHUB_COPILOT_AUTH_TYPE_KEY) == GITHUB_COPILOT_PER_USER_AUTH_TYPE - ): + values = cast( # cast-ok: credential_values is a plain dict at runtime + Mapping[object, object], credential.credential_values + ) + if values.get(GITHUB_COPILOT_AUTH_TYPE_KEY) == GITHUB_COPILOT_PER_USER_AUTH_TYPE: names.append(credential_name_obj) return tuple(names) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 1926e99f1a6b..8151770e972f 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -109,8 +109,8 @@ def _has_file_search_tool(tools: Iterable[Mapping[str, object]] | None) -> bool: def mock_responses_api_response( mock_response: str = "In a peaceful grove beneath a silver moon, a unicorn named Lumina discovered a hidden pool that reflected the stars. As she dipped her horn into the water, the pool began to shimmer, revealing a pathway to a magical realm of endless night skies. Filled with wonder, Lumina whispered a wish for all who dream to find their own hidden magic, and as she glanced back, her hoofprints sparkled like stardust.", ): - return ResponsesAPIResponse( - **{ + return ResponsesAPIResponse.model_validate( + { "id": "resp_67ccd2bed1ec8190b14f964abc0542670bb6a6b452d3795b", "object": "response", "created_at": 1741476542, @@ -794,7 +794,7 @@ async def aresponses( # (mirrors litellm/main.py:1371 for chat completions) response.hidden_params["custom_llm_provider"] = custom_llm_provider - if response is None: + if cast(object, response) is None: # cast-ok: response object identity check only raise ValueError(f"Got an unexpected None response from the Responses API: {response}") return response @@ -1056,7 +1056,6 @@ def _responses_try_dispatch_mcp_gateway( if skip_mcp_handler or not LiteLLM_Proxy_MCP_Handler.should_use_litellm_mcp_gateway(tools=tools): return None mcp_call_kwargs: Final = { - "input": input, "model": model, "include": include, "instructions": instructions, @@ -1065,13 +1064,11 @@ def _responses_try_dispatch_mcp_gateway( "metadata": metadata, "parallel_tool_calls": parallel_tool_calls, "previous_response_id": previous_response_id, - "reasoning": reasoning, "store": store, "background": background, "stream": stream, "temperature": temperature, "text": text, - "tool_choice": tool_choice, "tools": tools, "top_p": top_p, "truncation": truncation, @@ -1079,13 +1076,25 @@ def _responses_try_dispatch_mcp_gateway( "extra_headers": extra_headers, "extra_query": extra_query, "extra_body": extra_body, - "timeout": timeout, "custom_llm_provider": custom_llm_provider, **kwargs, } if _is_async: - return aresponses_api_with_mcp(**mcp_call_kwargs) - return run_async_function(aresponses_api_with_mcp, **mcp_call_kwargs) + return aresponses_api_with_mcp( + input=input, + reasoning=reasoning, + timeout=timeout, + tool_choice=tool_choice, + **mcp_call_kwargs, + ) + return run_async_function( + aresponses_api_with_mcp, + input=input, + reasoning=reasoning, + timeout=timeout, + tool_choice=tool_choice, + **mcp_call_kwargs, + ) def _responses_try_dispatch_emulated_file_search( @@ -1723,13 +1732,11 @@ async def aget_responses( response = init_response # Update the responses_api_response_id with the model_id - if isinstance(response, ResponsesAPIResponse): - response = ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id( - responses_api_response=response, - litellm_metadata=kwargs.get("litellm_metadata", {}), - custom_llm_provider=custom_llm_provider, - ) - return response + return ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id( + responses_api_response=response, + litellm_metadata=kwargs.get("litellm_metadata", {}), + custom_llm_provider=custom_llm_provider, + ) except Exception as e: raise litellm.exception_type( model=None, @@ -2194,14 +2201,11 @@ async def acompact_responses( response = init_response # Update the responses_api_response_id with the model_id - if isinstance(response, ResponsesAPIResponse): - response = ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id( - responses_api_response=response, - litellm_metadata=kwargs.get("litellm_metadata", {}), - custom_llm_provider=custom_llm_provider, - ) - - return response + return ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id( + responses_api_response=response, + litellm_metadata=kwargs.get("litellm_metadata", {}), + custom_llm_provider=custom_llm_provider, + ) except Exception as e: raise litellm.exception_type( model=model, diff --git a/litellm/utils.py b/litellm/utils.py index e135f8b6b696..367f33c60474 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8294,7 +8294,9 @@ def convert_list_message_to_dict( messages: Sequence[BaseModel | Mapping[str, object]], ) -> list[dict[str, object]]: return [ - dict(cleanup_none_field_in_message(message=cast(AllMessageValues, convert_to_dict(message)))) + dict( + cleanup_none_field_in_message(message=cast(AllMessageValues, convert_to_dict(message))) # cast-ok: message dicts satisfy the TypedDict shape + ) for message in messages ] @@ -8312,7 +8314,9 @@ def validate_and_fix_openai_messages(messages: list): if message.get("tool_calls"): message["tool_calls"] = jsonify_tools(tools=message["tool_calls"]) - convert_msg_to_dict = cast(AllMessageValues, convert_to_dict(message)) + convert_msg_to_dict = cast( # cast-ok: message dicts satisfy the TypedDict shape + AllMessageValues, convert_to_dict(message) + ) cleaned_message = cleanup_none_field_in_message(message=convert_msg_to_dict) new_messages.append(cleaned_message) return validate_chat_completion_user_messages(messages=new_messages) From a6c88cf6c3c95da576ca30adb91de245117a55c6 Mon Sep 17 00:00:00 2001 From: yassin Date: Thu, 8 Oct 2026 21:36:15 +0000 Subject: [PATCH 08/38] fix(types): keep runtime guards on untyped responses input Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../responses/transformation.py | 22 +++++++++++-------- litellm/responses/main.py | 6 ++++- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/litellm/llms/github_copilot/responses/transformation.py b/litellm/llms/github_copilot/responses/transformation.py index 674b07952593..5dc39bb9bb57 100644 --- a/litellm/llms/github_copilot/responses/transformation.py +++ b/litellm/llms/github_copilot/responses/transformation.py @@ -344,15 +344,19 @@ def _get_initiator(self, input_param: str | ResponseInputParam) -> str: return "user" # If input is a list, analyze items - for item in input_param: - # Check if item has no role (agent-initiated) - if "role" not in item or not item.get("role"): - return "agent" - - # Check if role is assistant (agent-initiated) - role = item.get("role") - if role.lower() == "assistant": - return "agent" + if isinstance(input_param, list): # pyright: ignore[reportUnnecessaryIsInstance] # items arrive from untyped request params + for item in input_param: + if not isinstance(item, dict): # pyright: ignore[reportUnnecessaryIsInstance] # items arrive from untyped request params + continue + + # Check if item has no role (agent-initiated) + if "role" not in item or not item.get("role"): + return "agent" + + # Check if role is assistant (agent-initiated) + role = item.get("role") + if isinstance(role, str) and role.lower() == "assistant": # pyright: ignore[reportUnnecessaryIsInstance] # role arrives from untyped request params + return "agent" # Default to user-initiated return "user" diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 8151770e972f..de16406f20a7 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -794,7 +794,7 @@ async def aresponses( # (mirrors litellm/main.py:1371 for chat completions) response.hidden_params["custom_llm_provider"] = custom_llm_provider - if cast(object, response) is None: # cast-ok: response object identity check only + if response is None: # pyright: ignore[reportUnnecessaryComparison] # provider handlers can return None at runtime raise ValueError(f"Got an unexpected None response from the Responses API: {response}") return response @@ -1732,6 +1732,8 @@ async def aget_responses( response = init_response # Update the responses_api_response_id with the model_id + if not isinstance(response, ResponsesAPIResponse): # pyright: ignore[reportUnnecessaryIsInstance] # handlers can return non-ResponsesAPIResponse objects at runtime + return response return ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id( responses_api_response=response, litellm_metadata=kwargs.get("litellm_metadata", {}), @@ -2201,6 +2203,8 @@ async def acompact_responses( response = init_response # Update the responses_api_response_id with the model_id + if not isinstance(response, ResponsesAPIResponse): # pyright: ignore[reportUnnecessaryIsInstance] # handlers can return non-ResponsesAPIResponse objects at runtime + return response return ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id( responses_api_response=response, litellm_metadata=kwargs.get("litellm_metadata", {}), From 126cee9f9a351866303936bb935debde547df436 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 21:42:40 +0000 Subject: [PATCH 09/38] style: ruff format per-user Copilot files --- litellm/llms/github_copilot/per_user_auth.py | 4 +--- litellm/main.py | 4 +++- litellm/proxy/credential_endpoints/endpoints.py | 4 +--- litellm/utils.py | 4 +++- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/litellm/llms/github_copilot/per_user_auth.py b/litellm/llms/github_copilot/per_user_auth.py index 41f0867b01a9..019336f5ce48 100644 --- a/litellm/llms/github_copilot/per_user_auth.py +++ b/litellm/llms/github_copilot/per_user_auth.py @@ -559,9 +559,7 @@ def github_copilot_auth_mode(litellm_credential_name: object, auth_type_value: o return isinstance(values, Mapping) and ( cast( # cast-ok: value is Mapping-checked or dict-shaped at runtime Mapping[object, object], values - ).get( - GITHUB_COPILOT_AUTH_TYPE_KEY - ) + ).get(GITHUB_COPILOT_AUTH_TYPE_KEY) == GITHUB_COPILOT_PER_USER_AUTH_TYPE ) return auth_type_value == GITHUB_COPILOT_PER_USER_AUTH_TYPE diff --git a/litellm/main.py b/litellm/main.py index caca7a33ad5d..96fb73463c29 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -6152,7 +6152,9 @@ def completion( response = _complete_langgraph(_dispatch_ctx) elif custom_llm_provider == "microsoft_365_copilot": - response = _complete_microsoft_365_copilot(_dispatch_ctx) # rebind-ok: provider branches share the common return + response = _complete_microsoft_365_copilot( + _dispatch_ctx + ) # rebind-ok: provider branches share the common return elif custom_llm_provider == "langflow": # LangFlow - Visual AI Agent Platform diff --git a/litellm/proxy/credential_endpoints/endpoints.py b/litellm/proxy/credential_endpoints/endpoints.py index 53db4426db9a..2689404991e3 100644 --- a/litellm/proxy/credential_endpoints/endpoints.py +++ b/litellm/proxy/credential_endpoints/endpoints.py @@ -477,9 +477,7 @@ def _per_user_credential_names() -> tuple[str, ...]: for credential in litellm.credential_list if cast( # cast-ok: credential_values is a plain dict at runtime Mapping[object, object], credential.credential_values - ).get( - GITHUB_COPILOT_AUTH_TYPE_KEY - ) + ).get(GITHUB_COPILOT_AUTH_TYPE_KEY) == GITHUB_COPILOT_PER_USER_AUTH_TYPE ) diff --git a/litellm/utils.py b/litellm/utils.py index 367f33c60474..96fef6d53d7c 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8295,7 +8295,9 @@ def convert_list_message_to_dict( ) -> list[dict[str, object]]: return [ dict( - cleanup_none_field_in_message(message=cast(AllMessageValues, convert_to_dict(message))) # cast-ok: message dicts satisfy the TypedDict shape + cleanup_none_field_in_message( + message=cast(AllMessageValues, convert_to_dict(message)) + ) # cast-ok: message dicts satisfy the TypedDict shape ) for message in messages ] From 26487a2db8a78dd16667ea578dcb7c3b8290e8b7 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 21:50:28 +0000 Subject: [PATCH 10/38] fix(lint): keep type-discipline suppressions on flagged lines --- litellm/llms/github_copilot/per_user_auth.py | 4 ++-- litellm/main.py | 4 ++-- litellm/utils.py | 12 ++++-------- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/litellm/llms/github_copilot/per_user_auth.py b/litellm/llms/github_copilot/per_user_auth.py index 019336f5ce48..8dfa2a1ba2c8 100644 --- a/litellm/llms/github_copilot/per_user_auth.py +++ b/litellm/llms/github_copilot/per_user_auth.py @@ -66,7 +66,7 @@ class _SyncGetClient(Protocol): def get( self, url: str, - params: dict[str, str] | None = None, + params: dict[str, str] | None = None, # mutable-ok: mirrors the untyped client's dict parameters headers: dict[str, str] | None = None, # mutable-ok: mirrors the untyped client's dict parameters ) -> httpx.Response: ... @@ -75,7 +75,7 @@ class _AsyncGitHubClient(Protocol): async def get( self, url: str, - params: dict[str, str] | None = None, + params: dict[str, str] | None = None, # mutable-ok: mirrors the untyped client's dict parameters headers: dict[str, str] | None = None, # mutable-ok: mirrors the untyped client's dict parameters ) -> httpx.Response: ... diff --git a/litellm/main.py b/litellm/main.py index 96fb73463c29..5a1da5dade71 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -6152,9 +6152,9 @@ def completion( response = _complete_langgraph(_dispatch_ctx) elif custom_llm_provider == "microsoft_365_copilot": - response = _complete_microsoft_365_copilot( + response = _complete_microsoft_365_copilot( # rebind-ok: provider branches share the common return _dispatch_ctx - ) # rebind-ok: provider branches share the common return + ) elif custom_llm_provider == "langflow": # LangFlow - Visual AI Agent Platform diff --git a/litellm/utils.py b/litellm/utils.py index 96fef6d53d7c..e77327dd1f7b 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8293,14 +8293,10 @@ def convert_to_dict(message: BaseModel | Mapping[str, object]) -> dict[str, obje def convert_list_message_to_dict( messages: Sequence[BaseModel | Mapping[str, object]], ) -> list[dict[str, object]]: - return [ - dict( - cleanup_none_field_in_message( - message=cast(AllMessageValues, convert_to_dict(message)) - ) # cast-ok: message dicts satisfy the TypedDict shape - ) - for message in messages - ] + def _as_message_value(message: BaseModel | Mapping[str, object]) -> AllMessageValues: + return cast(AllMessageValues, convert_to_dict(message)) # cast-ok: message dicts satisfy the TypedDict shape + + return [dict(cleanup_none_field_in_message(message=_as_message_value(message))) for message in messages] def validate_and_fix_openai_messages(messages: list): From 05c1c9c9527919272a6c5b20d839640ab3f46110 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 22:03:31 +0000 Subject: [PATCH 11/38] fix(github_copilot): resolve team and deployment-id per-user routes, share token cache invalidation --- .../user_provider_credentials.py | 16 ++-- litellm/proxy/litellm_pre_call_utils.py | 19 ++++- .../test_user_provider_credentials.py | 52 ++++++++++++ .../unit/proxy/test_litellm_pre_call_utils.py | 79 ++++++++++++++++++- ... LlmCredentialsPanel.integration.test.tsx} | 0 ...bCopilotConnectModal.integration.test.tsx} | 0 6 files changed, 156 insertions(+), 10 deletions(-) rename ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/{LlmCredentialsPanel.test.tsx => LlmCredentialsPanel.integration.test.tsx} (100%) rename ui/litellm-dashboard/src/components/model_add/{GithubCopilotConnectModal.test.tsx => GithubCopilotConnectModal.integration.test.tsx} (100%) diff --git a/litellm/proxy/credential_endpoints/user_provider_credentials.py b/litellm/proxy/credential_endpoints/user_provider_credentials.py index ebd348f90528..61de5f30d7ab 100644 --- a/litellm/proxy/credential_endpoints/user_provider_credentials.py +++ b/litellm/proxy/credential_endpoints/user_provider_credentials.py @@ -163,15 +163,21 @@ async def list_user_provider_credentials_for_credential( return await _table(prisma_client).find_many(where={"credential_name": credential_name}) +def _token_cache(cache: DualCache) -> "_TokenCache": + """Cross-worker consistency matters here: with Redis attached, use it alone + so a connect or disconnect invalidates on every worker, not just this one.""" + return cast( # cast-ok: DualCache and RedisCache both satisfy the used signatures + _TokenCache, cache.redis_cache if cache.redis_cache is not None else cache + ) + + @with_service_target("user_provider_connections") async def invalidate_user_provider_credential_cache( cache: DualCache, user_id: str, credential_name: str, ) -> None: - token_cache: Final = cast( # cast-ok: pins the untyped DualCache to the local _TokenCache Protocol - _TokenCache, cache - ) + token_cache: Final = _token_cache(cache) await token_cache.async_delete_cache(_cache_key(user_id, credential_name)) @@ -186,9 +192,7 @@ async def aget_user_provider_tokens( Cached values are the stored ciphertext or the ``_NOT_CONNECTED`` marker; plaintext tokens only ever live in the returned dict.""" - token_cache: Final = cast( # cast-ok: pins the untyped DualCache to the local _TokenCache Protocol - _TokenCache, cache - ) + token_cache: Final = _token_cache(cache) names: Final = tuple(dict.fromkeys(credential_names)) if not names: return {} diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 4638cf5817f3..d2a8c9128bfc 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -2650,6 +2650,7 @@ async def add_litellm_data_to_request( await _resolve_user_provider_credentials_for_request( data=data, authenticated_user_id=authenticated_user_id, + team_id=user_api_key_dict.team_id, llm_router=llm_router, ) @@ -2718,6 +2719,7 @@ def _fallback_target_groups( def _per_user_credential_names_for_groups( llm_router: litellm.Router, model_groups: frozenset[str], + team_id: str | None, ) -> tuple[str, ...]: from litellm.constants import ( GITHUB_COPILOT_AUTH_TYPE_KEY, @@ -2726,8 +2728,18 @@ def _per_user_credential_names_for_groups( names: Final[list[str]] = [] # mutable-ok: accumulates one name per per-user deployment for group in model_groups: - for deployment in llm_router.get_model_list(model_name=group) or (): - litellm_params = cast(object, deployment.get("litellm_params")) # cast-ok: deployment is dict-shaped here + by_name: Final = llm_router.get_model_list(model_name=group, team_id=team_id) or () + deployment_id_match: Final = llm_router.get_deployment(model_id=group) + deployments: Final[Sequence[object]] = ( + *by_name, + *((deployment_id_match,) if deployment_id_match is not None else ()), + ) + for deployment in deployments: + litellm_params: object = ( + deployment.get("litellm_params") + if isinstance(deployment, Mapping) + else getattr(deployment, "litellm_params", None) + ) credential_name_obj: object = ( litellm_params.get("litellm_credential_name") if isinstance(litellm_params, Mapping) @@ -2749,6 +2761,7 @@ def _per_user_credential_names_for_groups( async def _resolve_user_provider_credentials_for_request( data: dict[str, object], # mutable-ok: writes resolved credentials into the nested secret_fields dict authenticated_user_id: str | None, + team_id: str | None, llm_router: litellm.Router | None, ) -> None: """Resolve the calling user's per-user provider connections into secret_fields. @@ -2764,7 +2777,7 @@ async def _resolve_user_provider_credentials_for_request( if not isinstance(user_id, str) or not user_id or llm_router is None or not isinstance(model, str): return model_groups: Final = frozenset({model} | _fallback_target_groups(llm_router, model)) - credential_names: Final = _per_user_credential_names_for_groups(llm_router, model_groups) + credential_names: Final = _per_user_credential_names_for_groups(llm_router, model_groups, team_id) if not credential_names: return if "litellm_credential_name" in data: diff --git a/tests/unit/proxy/credential_endpoints/test_user_provider_credentials.py b/tests/unit/proxy/credential_endpoints/test_user_provider_credentials.py index b739ccfb3e7b..cae5564b1754 100644 --- a/tests/unit/proxy/credential_endpoints/test_user_provider_credentials.py +++ b/tests/unit/proxy/credential_endpoints/test_user_provider_credentials.py @@ -147,3 +147,55 @@ async def test_invalidate_cache_forces_db_refetch(): def test_decode_rejects_garbage(): assert decode_user_provider_credential("not-a-cipher") is None + + +@pytest.mark.asyncio +async def test_disconnect_on_one_worker_invalidates_other_workers_via_redis(monkeypatch): + """Two DualCache instances (two workers) sharing one Redis: a disconnect on A must + be visible on B, not just in A's in-memory layer.""" + import fakeredis + + from litellm.caching.redis_cache import RedisCache + from litellm.proxy.credential_endpoints import user_provider_credentials as upc + + fake = fakeredis.FakeAsyncRedis() + redis = RedisCache(host="localhost", port=6379) + monkeypatch.setattr(redis, "init_async_client", lambda: fake) + cache_a = DualCache(redis_cache=redis) + cache_b = DualCache(redis_cache=redis) + + payload = GithubCopilotUserConnectionPayload(access_token="gho_secret", github_login="octo") + ciphertext = upc._encode(payload) + table = MagicMock() + table.find_many = AsyncMock(side_effect=[[_row(credential_b64=ciphertext)], []]) + prisma_client = _prisma(table) + + # worker A "connects" (DB read + cache write) + tokens_a = await aget_user_provider_tokens(prisma_client, cache_a, "user-a", ["copilot-cred"]) + assert tokens_a == {"copilot-cred": "gho_secret"} + + # worker B reads the same connection straight from Redis (its in-memory layer is empty) + tokens_b = await aget_user_provider_tokens(prisma_client, cache_b, "user-a", ["copilot-cred"]) + assert tokens_b == {"copilot-cred": "gho_secret"} + + # worker A disconnects + await invalidate_user_provider_credential_cache(cache_a, "user-a", "copilot-cred") + + # worker B must now see not-connected (DB returns no row on the second read) + tokens_b_after = await aget_user_provider_tokens(prisma_client, cache_b, "user-a", ["copilot-cred"]) + assert tokens_b_after == {} + assert table.find_many.await_count == 2 + + +@pytest.mark.asyncio +async def test_token_cache_uses_redis_only_when_attached(): + """Backend selection is pinned: Redis attached -> Redis only; no Redis -> the DualCache.""" + from litellm.caching.redis_cache import RedisCache + from litellm.proxy.credential_endpoints import user_provider_credentials as upc + + bare = DualCache() + assert upc._token_cache(bare) is bare + + redis = RedisCache(host="localhost", port=6379) + with_redis = DualCache(redis_cache=redis) + assert upc._token_cache(with_redis) is redis diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 6a96f4ac95ca..c633e263f3a8 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -8933,6 +8933,8 @@ def test_body_snapshot_drops_only_the_marker_and_keeps_caller_value(marker: str) refresh_proxy_server_request_body_snapshot(caller_data) assert caller_request["body"][SERVER_STREAMING_CLASSIFICATION_KEY] == "caller-value", caller_request + + def test_arize_otlp_protocol_on_a_key_logging_entry_reaches_the_destination(monkeypatch): from litellm.integrations.otel.model.config import is_otel_v2_enabled from litellm.proxy.litellm_pre_call_utils import resolve_tenant_otel_destinations @@ -9033,6 +9035,7 @@ async def test_injects_tokens_for_per_user_deployments_only(self, monkeypatch): await _resolve_user_provider_credentials_for_request( data=data, authenticated_user_id="user-a", + team_id=None, llm_router=self._router(), ) @@ -9042,6 +9045,69 @@ async def test_injects_tokens_for_per_user_deployments_only(self, monkeypatch): call_where = table.find_many.await_args.kwargs["where"] assert call_where == {"AND": ({"user_id": "user-a"}, {"credential_name": {"in": ["copilot-cred"]}})} + @pytest.mark.asyncio + async def test_team_public_model_name_resolves_per_user_deployments(self, monkeypatch): + """A caller hitting a team's public model name must get the same per-user + credentials routing resolves: get_model_list needs the caller's team_id.""" + from litellm import Router + from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request + + table = self._env(monkeypatch) + router = Router( + model_list=[ + { + "model_name": "copilot-chat_team-1_dep", + "litellm_params": { + "model": "github_copilot/gpt-4o", + "litellm_credential_name": "copilot-cred", + }, + "model_info": { + "id": "dep-abc", + "team_id": "team-1", + "team_public_model_name": "copilot-chat", + }, + } + ] + ) + data = {"model": "copilot-chat", "secret_fields": {}} + await _resolve_user_provider_credentials_for_request( + data=data, + authenticated_user_id="user-a", + team_id="team-1", + llm_router=router, + ) + table.find_many.assert_awaited() + call_where = table.find_many.await_args.kwargs["where"] + assert call_where == {"AND": ({"user_id": "user-a"}, {"credential_name": {"in": ["copilot-cred"]}})} + + @pytest.mark.asyncio + async def test_deployment_id_resolves_per_user_deployment(self, monkeypatch): + """A request addressed by deployment id still discovers the per-user credential.""" + from litellm import Router + from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request + + table = self._env(monkeypatch) + router = Router( + model_list=[ + { + "model_name": "copilot-chat", + "litellm_params": { + "model": "github_copilot/gpt-4o", + "litellm_credential_name": "copilot-cred", + }, + "model_info": {"id": "dep-xyz"}, + } + ] + ) + data = {"model": "dep-xyz", "secret_fields": {}} + await _resolve_user_provider_credentials_for_request( + data=data, + authenticated_user_id="user-a", + team_id=None, + llm_router=router, + ) + table.find_many.assert_awaited() + @pytest.mark.asyncio async def test_group_with_no_per_user_deployment_leaves_secret_fields_untouched(self, monkeypatch): from litellm import Router @@ -9052,6 +9118,7 @@ async def test_group_with_no_per_user_deployment_leaves_secret_fields_untouched( await _resolve_user_provider_credentials_for_request( data=data, authenticated_user_id="user-a", + team_id=None, llm_router=Router( model_list=[ { @@ -9077,6 +9144,7 @@ async def test_unconnected_user_gets_empty_map_not_an_error(self, monkeypatch): await _resolve_user_provider_credentials_for_request( data=data, authenticated_user_id="user-a", + team_id=None, llm_router=self._router(), ) assert data["secret_fields"]["user_provider_credentials"] == {} @@ -9090,6 +9158,7 @@ async def test_missing_user_id_does_nothing(self, monkeypatch): await _resolve_user_provider_credentials_for_request( data=data, authenticated_user_id=None, + team_id=None, llm_router=self._router(), ) assert "user_provider_credentials" not in data["secret_fields"] @@ -9104,6 +9173,7 @@ async def test_unrelated_model_makes_zero_db_and_cache_calls(self, monkeypatch): await _resolve_user_provider_credentials_for_request( data=data, authenticated_user_id="user-a", + team_id=None, llm_router=Router( model_list=[ { @@ -9136,7 +9206,10 @@ async def test_per_user_deployment_with_pydantic_litellm_params(self, monkeypatc } ] ) - assert _per_user_credential_names_for_groups(router, frozenset({"copilot-chat"})) == ("copilot-cred",) + router.get_deployment = MagicMock(return_value=None) + assert _per_user_credential_names_for_groups(router, frozenset({"copilot-chat"}), team_id=None) == ( + "copilot-cred", + ) def _per_user_env(monkeypatch, credential_names=("copilot-cred",)): @@ -9191,6 +9264,7 @@ async def test_request_body_litellm_credential_name_is_rejected_for_per_user_mod await _resolve_user_provider_credentials_for_request( data=data, authenticated_user_id="user-a", + team_id=None, llm_router=router, ) assert exc.value.status_code == 400 @@ -9216,6 +9290,7 @@ async def test_request_body_litellm_credential_name_untouched_for_shared_models( await _resolve_user_provider_credentials_for_request( data=data, authenticated_user_id="user-a", + team_id=None, llm_router=router, ) assert "user_provider_credentials" not in data["secret_fields"] @@ -9247,6 +9322,7 @@ async def test_transitive_fallback_chain_loads_per_user_credential(monkeypatch): await _resolve_user_provider_credentials_for_request( data=data, authenticated_user_id="user-a", + team_id=None, llm_router=router, ) where = table.find_many.await_args.kwargs["where"] @@ -9283,6 +9359,7 @@ async def test_secret_fields_user_provider_credentials_repr_hides_tokens(monkeyp await _resolve_user_provider_credentials_for_request( data=data, authenticated_user_id="user-a", + team_id=None, llm_router=router, ) credentials = data["secret_fields"]["user_provider_credentials"] diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/LlmCredentialsPanel.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/LlmCredentialsPanel.integration.test.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/LlmCredentialsPanel.test.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/LlmCredentialsPanel.integration.test.tsx diff --git a/ui/litellm-dashboard/src/components/model_add/GithubCopilotConnectModal.test.tsx b/ui/litellm-dashboard/src/components/model_add/GithubCopilotConnectModal.integration.test.tsx similarity index 100% rename from ui/litellm-dashboard/src/components/model_add/GithubCopilotConnectModal.test.tsx rename to ui/litellm-dashboard/src/components/model_add/GithubCopilotConnectModal.integration.test.tsx From 214c9ff07faeb7e194e530bde040dff3b71d12ba Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 22:13:32 +0000 Subject: [PATCH 12/38] test(github_copilot): scope response-cache mutation via monkeypatch, assert route allowlist --- .../llms/github_copilot/test_per_user_auth.py | 199 ++++++++---------- tests/unit/proxy/auth/test_route_checks.py | 4 +- 2 files changed, 93 insertions(+), 110 deletions(-) diff --git a/tests/unit/llms/github_copilot/test_per_user_auth.py b/tests/unit/llms/github_copilot/test_per_user_auth.py index 7aaf7b8c376f..b4de3142d925 100644 --- a/tests/unit/llms/github_copilot/test_per_user_auth.py +++ b/tests/unit/llms/github_copilot/test_per_user_auth.py @@ -377,59 +377,53 @@ def _secret_fields(user_id: str, github_token: str): def test_two_per_user_calls_with_identical_bodies_both_reach_upstream( # test-quality-ok: upstream dispatch count is the observable signal of the cache exclusion - per_user_credential, + per_user_credential, monkeypatch ): """Response-cache keys ignore caller identity: without the exclusion the second user's call would be served from the first's cache entry and neither run on their own seat.""" from litellm.caching import Cache - litellm.cache = Cache() - try: - with ( - github_http(_exchange_responder()), - patch( - "litellm.main.openai_chat_completions.completion", - return_value=_chat_completion_response(), - ) as mock_completion, - ): - base = { - "model": "github_copilot/gpt-4o", - "messages": [{"role": "user", "content": "hi"}], - "litellm_credential_name": "copilot-cred", - } - litellm.completion(**base, secret_fields=_secret_fields("user-a", "gho_a")) - litellm.completion(**base, secret_fields=_secret_fields("user-b", "gho_b")) - assert mock_completion.call_count == 2 - finally: - litellm.cache = None + monkeypatch.setattr(litellm, "cache", Cache()) + with ( + github_http(_exchange_responder()), + patch( + "litellm.main.openai_chat_completions.completion", + return_value=_chat_completion_response(), + ) as mock_completion, + ): + base = { + "model": "github_copilot/gpt-4o", + "messages": [{"role": "user", "content": "hi"}], + "litellm_credential_name": "copilot-cred", + } + litellm.completion(**base, secret_fields=_secret_fields("user-a", "gho_a")) + litellm.completion(**base, secret_fields=_secret_fields("user-b", "gho_b")) + assert mock_completion.call_count == 2 @pytest.mark.asyncio async def test_two_per_user_async_calls_with_identical_bodies_both_reach_upstream( # test-quality-ok: upstream dispatch count is the observable signal of the cache exclusion - per_user_credential, + per_user_credential, monkeypatch ): """Async path: per-user calls must bypass async_get_cache read and write the same way.""" from litellm.caching import Cache - litellm.cache = Cache() - try: - with ( - github_http(_exchange_responder()), - patch( - "litellm.main.openai_chat_completions.acompletion", - new=AsyncMock(return_value=_chat_completion_response()), - ) as mock_completion, - ): - base = { - "model": "github_copilot/gpt-4o", - "messages": [{"role": "user", "content": "hi"}], - "litellm_credential_name": "copilot-cred", - } - await litellm.acompletion(**base, secret_fields=_secret_fields("user-a", "gho_a")) - await litellm.acompletion(**base, secret_fields=_secret_fields("user-b", "gho_b")) - assert mock_completion.call_count == 2 - finally: - litellm.cache = None + monkeypatch.setattr(litellm, "cache", Cache()) + with ( + github_http(_exchange_responder()), + patch( + "litellm.main.openai_chat_completions.acompletion", + new=AsyncMock(return_value=_chat_completion_response()), + ) as mock_completion, + ): + base = { + "model": "github_copilot/gpt-4o", + "messages": [{"role": "user", "content": "hi"}], + "litellm_credential_name": "copilot-cred", + } + await litellm.acompletion(**base, secret_fields=_secret_fields("user-a", "gho_a")) + await litellm.acompletion(**base, secret_fields=_secret_fields("user-b", "gho_b")) + assert mock_completion.call_count == 2 def test_shared_mode_second_identical_call_hits_response_cache( # test-quality-ok: the cache hit is only observable as the upstream not being dispatched @@ -453,21 +447,18 @@ def test_shared_mode_second_identical_call_hits_response_cache( # test-quality- ) monkeypatch.setenv("GITHUB_COPILOT_TOKEN_DIR", str(tmp_path)) - litellm.cache = Cache() - try: - with patch( - "litellm.main.openai_chat_completions.completion", - return_value=_chat_completion_response(), - ) as mock_completion: - call = { - "model": "github_copilot/gpt-4o", - "messages": [{"role": "user", "content": "hi"}], - } - litellm.completion(**call) - litellm.completion(**call) - assert mock_completion.call_count == 1, "shared mode must still serve the second call from cache" - finally: - litellm.cache = None + monkeypatch.setattr(litellm, "cache", Cache()) + with patch( + "litellm.main.openai_chat_completions.completion", + return_value=_chat_completion_response(), + ) as mock_completion: + call = { + "model": "github_copilot/gpt-4o", + "messages": [{"role": "user", "content": "hi"}], + } + litellm.completion(**call) + litellm.completion(**call) + assert mock_completion.call_count == 1, "shared mode must still serve the second call from cache" @@ -885,40 +876,37 @@ async def test_aexchange_failure_with_a_single_caller_emits_no_unretrieved_warni @pytest.mark.asyncio -async def test_per_user_streaming_call_writes_nothing_to_the_response_cache(per_user_credential): +async def test_per_user_streaming_call_writes_nothing_to_the_response_cache(per_user_credential, monkeypatch): """Per-user streamed completions must never be written to the shared response cache: the handler built before the session attach would otherwise use a stale request_kwargs copy without the session marker.""" from litellm.caching import Cache from litellm.caching.caching_handler import LLMCachingHandler - litellm.cache = Cache() - try: - handler = LLMCachingHandler( - original_function=litellm.acompletion, - request_kwargs={"model": "github_copilot/gpt-4o"}, - start_time=datetime.datetime.now(), - ) - kwargs = { - "model": "github_copilot/gpt-4o", - "messages": [{"role": "user", "content": "hi"}], - GITHUB_COPILOT_USER_SESSION_KWARG_KEY: GithubCopilotUserSession( - token="copilot-token", api_base="https://api.githubcopilot.com" - ), - } - await handler.async_get_cache( - model="github_copilot/gpt-4o", - original_function=litellm.acompletion, - logging_obj=MagicMock(), - start_time=datetime.datetime.now(), - call_type="acompletion", - kwargs=kwargs, - ) - assert handler.should_store_result_in_cache( - original_function=litellm.acompletion, kwargs=handler.request_kwargs - ) is False - finally: - litellm.cache = None + monkeypatch.setattr(litellm, "cache", Cache()) + handler = LLMCachingHandler( + original_function=litellm.acompletion, + request_kwargs={"model": "github_copilot/gpt-4o"}, + start_time=datetime.datetime.now(), + ) + kwargs = { + "model": "github_copilot/gpt-4o", + "messages": [{"role": "user", "content": "hi"}], + GITHUB_COPILOT_USER_SESSION_KWARG_KEY: GithubCopilotUserSession( + token="copilot-token", api_base="https://api.githubcopilot.com" + ), + } + await handler.async_get_cache( + model="github_copilot/gpt-4o", + original_function=litellm.acompletion, + logging_obj=MagicMock(), + start_time=datetime.datetime.now(), + call_type="acompletion", + kwargs=kwargs, + ) + assert handler.should_store_result_in_cache( + original_function=litellm.acompletion, kwargs=handler.request_kwargs + ) is False @pytest.mark.asyncio @@ -928,27 +916,24 @@ async def test_shared_streaming_call_still_writes_response_cache(tmp_path, monke from litellm.caching import Cache from litellm.caching.caching_handler import LLMCachingHandler - litellm.cache = Cache() - try: - handler = LLMCachingHandler( - original_function=litellm.acompletion, - request_kwargs={"model": "github_copilot/gpt-4o"}, - start_time=datetime.datetime.now(), - ) - kwargs = { - "model": "github_copilot/gpt-4o", - "messages": [{"role": "user", "content": "hi"}], - } - await handler.async_get_cache( - model="github_copilot/gpt-4o", - original_function=litellm.acompletion, - logging_obj=MagicMock(), - start_time=datetime.datetime.now(), - call_type="acompletion", - kwargs=kwargs, - ) - assert handler.should_store_result_in_cache( - original_function=litellm.acompletion, kwargs=handler.request_kwargs - ) is True - finally: - litellm.cache = None + monkeypatch.setattr(litellm, "cache", Cache()) + handler = LLMCachingHandler( + original_function=litellm.acompletion, + request_kwargs={"model": "github_copilot/gpt-4o"}, + start_time=datetime.datetime.now(), + ) + kwargs = { + "model": "github_copilot/gpt-4o", + "messages": [{"role": "user", "content": "hi"}], + } + await handler.async_get_cache( + model="github_copilot/gpt-4o", + original_function=litellm.acompletion, + logging_obj=MagicMock(), + start_time=datetime.datetime.now(), + call_type="acompletion", + kwargs=kwargs, + ) + assert handler.should_store_result_in_cache( + original_function=litellm.acompletion, kwargs=handler.request_kwargs + ) is True diff --git a/tests/unit/proxy/auth/test_route_checks.py b/tests/unit/proxy/auth/test_route_checks.py index f2868db661f9..de5b2f4b1123 100644 --- a/tests/unit/proxy/auth/test_route_checks.py +++ b/tests/unit/proxy/auth/test_route_checks.py @@ -4702,9 +4702,7 @@ def test_is_llm_api_route(): all_llm_api_routes = llm_passthrough_router.routes for route in all_llm_api_routes: - print("route", route) route_path = str(route.path) - print("route_path", route_path) assert RouteChecks.is_llm_api_route(route_path) is True @@ -4750,7 +4748,7 @@ def _internal_user_check(route: str, method: str, role: LitellmUserRoles = Litel ], ) def test_internal_user_credential_connection_routes_allowed(method: str, route: str) -> None: - _internal_user_check(route, method) + assert _internal_user_check(route, method) is None @pytest.mark.parametrize( From bf01ef05bc2946fc9cc6f80c9f01a5336b5335cf Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 22:20:11 +0000 Subject: [PATCH 13/38] fix(github_copilot): fall back to the database on Redis errors and skip worker-local token caching --- .../user_provider_credentials.py | 64 +++++------ litellm/proxy/litellm_pre_call_utils.py | 4 +- .../test_user_provider_credentials.py | 101 ++++++++++++++---- 3 files changed, 114 insertions(+), 55 deletions(-) diff --git a/litellm/proxy/credential_endpoints/user_provider_credentials.py b/litellm/proxy/credential_endpoints/user_provider_credentials.py index 61de5f30d7ab..475f3a211ebe 100644 --- a/litellm/proxy/credential_endpoints/user_provider_credentials.py +++ b/litellm/proxy/credential_endpoints/user_provider_credentials.py @@ -9,18 +9,16 @@ import json from collections.abc import Mapping, Sequence -from typing import ( - TYPE_CHECKING, - Final, - Protocol, - cast, # noqa: TID251 # casts pin the untyped DualCache to the local _TokenCache Protocol -) +from typing import TYPE_CHECKING, Final from pydantic import BaseModel, ConfigDict, Field, ValidationError from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger from litellm.caching.dual_cache import DualCache + +if TYPE_CHECKING: + from litellm.caching.redis_cache import RedisCache from litellm.constants import ( GITHUB_COPILOT_USER_CREDENTIAL_CACHE_TTL_SECONDS, USER_PROVIDER_CREDENTIAL_CACHE_PREFIX, @@ -42,16 +40,6 @@ _NOT_CONNECTED: Final = USER_PROVIDER_CREDENTIAL_NOT_CONNECTED -class _TokenCache(Protocol): - async def async_get_cache(self, key: str) -> object: ... - - async def async_set_cache( - self, key: str, value: object, local_only: bool = False, ttl: float | None = None - ) -> object: ... - - async def async_delete_cache(self, key: str) -> object: ... - - class GithubCopilotUserConnectionPayload(BaseModel): model_config = ConfigDict(extra="ignore", frozen=True) @@ -163,12 +151,20 @@ async def list_user_provider_credentials_for_credential( return await _table(prisma_client).find_many(where={"credential_name": credential_name}) -def _token_cache(cache: DualCache) -> "_TokenCache": - """Cross-worker consistency matters here: with Redis attached, use it alone - so a connect or disconnect invalidates on every worker, not just this one.""" - return cast( # cast-ok: DualCache and RedisCache both satisfy the used signatures - _TokenCache, cache.redis_cache if cache.redis_cache is not None else cache - ) +async def _try_cache_get(token_cache: "RedisCache", key: str) -> object: + """Redis errors must read as a plain miss: the DB is the source of truth.""" + try: + return await token_cache.async_get_cache(key) + except Exception: # noqa: BLE001 # a Redis outage must fall back to the database, never reject a connected user + verbose_proxy_logger.warning("aget_user_provider_tokens: Redis get failed; falling back to the database") + return None + + +async def _try_cache_set(token_cache: "RedisCache", key: str, value: str) -> None: + try: + await token_cache.async_set_cache(key, value, ttl=GITHUB_COPILOT_USER_CREDENTIAL_CACHE_TTL_SECONDS) + except Exception: # noqa: BLE001 # caching is best-effort; a Redis outage is not worth failing the request + verbose_proxy_logger.warning("aget_user_provider_tokens: Redis set failed; skipping the cache write") @with_service_target("user_provider_connections") @@ -177,8 +173,13 @@ async def invalidate_user_provider_credential_cache( user_id: str, credential_name: str, ) -> None: - token_cache: Final = _token_cache(cache) - await token_cache.async_delete_cache(_cache_key(user_id, credential_name)) + token_cache: Final = cache.redis_cache + if token_cache is None: + return + try: + await token_cache.async_delete_cache(_cache_key(user_id, credential_name)) + except Exception: # noqa: BLE001 # a Redis outage must not fail a disconnect + verbose_proxy_logger.warning("invalidate_user_provider_credential_cache: Redis delete failed") @with_service_target("user_provider_connections") @@ -191,15 +192,17 @@ async def aget_user_provider_tokens( """Map each per-user credential name to the caller's stored GitHub token. Cached values are the stored ciphertext or the ``_NOT_CONNECTED`` marker; - plaintext tokens only ever live in the returned dict.""" - token_cache: Final = _token_cache(cache) + plaintext tokens only ever live in the returned dict. Redis is the only + cache layer used: without it there is no shared invalidation, so a worker + serving a stale local entry after a disconnect is worse than a DB read.""" + token_cache: Final = cache.redis_cache names: Final = tuple(dict.fromkeys(credential_names)) if not names: return {} cached: Final[dict[str, str]] = {} # mutable-ok: accumulates hits and DB reads misses: Final[list[str]] = [] # mutable-ok: accumulates cache misses for name in names: - value = await token_cache.async_get_cache(_cache_key(user_id, name)) + value = await _try_cache_get(token_cache, _cache_key(user_id, name)) if token_cache is not None else None if value == _NOT_CONNECTED: continue if isinstance(value, str) and value: @@ -225,11 +228,8 @@ async def aget_user_provider_tokens( return {} found: Final = {row.credential_name: row.credential_b64 for row in rows} for name in misses: - await token_cache.async_set_cache( - _cache_key(user_id, name), - found.get(name, _NOT_CONNECTED), - ttl=GITHUB_COPILOT_USER_CREDENTIAL_CACHE_TTL_SECONDS, - ) + if token_cache is not None: + await _try_cache_set(token_cache, _cache_key(user_id, name), found.get(name, _NOT_CONNECTED)) if name in found: cached[name] = found[name] return { diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index d2a8c9128bfc..32c6c664aaef 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -2747,10 +2747,10 @@ def _per_user_credential_names_for_groups( ) if not isinstance(credential_name_obj, str) or not credential_name_obj or credential_name_obj in names: continue - credential = CredentialAccessor.find_credential(credential_name_obj) + credential: Final = CredentialAccessor.find_credential(credential_name_obj) if credential is None: continue - values = cast( # cast-ok: credential_values is a plain dict at runtime + values: Final = cast( # cast-ok: credential_values is a plain dict at runtime Mapping[object, object], credential.credential_values ) if values.get(GITHUB_COPILOT_AUTH_TYPE_KEY) == GITHUB_COPILOT_PER_USER_AUTH_TYPE: diff --git a/tests/unit/proxy/credential_endpoints/test_user_provider_credentials.py b/tests/unit/proxy/credential_endpoints/test_user_provider_credentials.py index cae5564b1754..e854bc00de09 100644 --- a/tests/unit/proxy/credential_endpoints/test_user_provider_credentials.py +++ b/tests/unit/proxy/credential_endpoints/test_user_provider_credentials.py @@ -33,6 +33,18 @@ def _row(user_id="user-a", credential_name="copilot-cred", credential_b64="ciphe ) +def _fake_redis(monkeypatch): + """A RedisCache backed by a fakeredis client; two DualCaches can share it.""" + import fakeredis + + from litellm.caching.redis_cache import RedisCache + + fake = fakeredis.FakeAsyncRedis() + redis = RedisCache(host="localhost", port=6379) + monkeypatch.setattr(redis, "init_async_client", lambda: fake) + return redis + + @pytest.fixture(autouse=True) def _master_key(monkeypatch): monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "sk-test-master") @@ -94,7 +106,7 @@ async def test_delete_for_credential_returns_affected_user_ids(): @pytest.mark.asyncio -async def test_aget_tokens_returns_plaintext_and_caches_ciphertext(): +async def test_aget_tokens_returns_plaintext_and_caches_ciphertext(monkeypatch): payload = GithubCopilotUserConnectionPayload(access_token="gho_secret", github_login="octo") from litellm.proxy.credential_endpoints import user_provider_credentials as upc @@ -102,7 +114,8 @@ async def test_aget_tokens_returns_plaintext_and_caches_ciphertext(): table = MagicMock() table.find_many = AsyncMock(return_value=[_row(credential_b64=ciphertext)]) prisma_client = _prisma(table) - cache = DualCache() + redis = _fake_redis(monkeypatch) + cache = DualCache(redis_cache=redis) tokens = await aget_user_provider_tokens(prisma_client, cache, "user-a", ["copilot-cred"]) assert tokens == {"copilot-cred": "gho_secret"} @@ -111,17 +124,18 @@ async def test_aget_tokens_returns_plaintext_and_caches_ciphertext(): tokens2 = await aget_user_provider_tokens(prisma_client, cache, "user-a", ["copilot-cred"]) assert tokens2 == {"copilot-cred": "gho_secret"} assert table.find_many.await_count == 1 - cached = await cache.async_get_cache(upc._cache_key("user-a", "copilot-cred")) + cached = await redis.async_get_cache(upc._cache_key("user-a", "copilot-cred")) assert cached == ciphertext assert "gho_secret" not in cached + assert await cache.in_memory_cache.async_get_cache(upc._cache_key("user-a", "copilot-cred")) is None @pytest.mark.asyncio -async def test_aget_tokens_negative_caches_missing_connection(): +async def test_aget_tokens_negative_caches_missing_connection(monkeypatch): table = MagicMock() table.find_many = AsyncMock(return_value=[]) prisma_client = _prisma(table) - cache = DualCache() + cache = DualCache(redis_cache=_fake_redis(monkeypatch)) assert await aget_user_provider_tokens(prisma_client, cache, "user-a", ["copilot-cred"]) == {} assert await aget_user_provider_tokens(prisma_client, cache, "user-a", ["copilot-cred"]) == {} @@ -129,14 +143,14 @@ async def test_aget_tokens_negative_caches_missing_connection(): @pytest.mark.asyncio -async def test_invalidate_cache_forces_db_refetch(): +async def test_invalidate_cache_forces_db_refetch(monkeypatch): payload = GithubCopilotUserConnectionPayload(access_token="gho_new", github_login="octo") from litellm.proxy.credential_endpoints import user_provider_credentials as upc table = MagicMock() table.find_many = AsyncMock(return_value=[_row(credential_b64=upc._encode(payload))]) prisma_client = _prisma(table) - cache = DualCache() + cache = DualCache(redis_cache=_fake_redis(monkeypatch)) await aget_user_provider_tokens(prisma_client, cache, "user-a", ["copilot-cred"]) await invalidate_user_provider_credential_cache(cache, "user-a", "copilot-cred") @@ -153,14 +167,9 @@ def test_decode_rejects_garbage(): async def test_disconnect_on_one_worker_invalidates_other_workers_via_redis(monkeypatch): """Two DualCache instances (two workers) sharing one Redis: a disconnect on A must be visible on B, not just in A's in-memory layer.""" - import fakeredis - - from litellm.caching.redis_cache import RedisCache from litellm.proxy.credential_endpoints import user_provider_credentials as upc - fake = fakeredis.FakeAsyncRedis() - redis = RedisCache(host="localhost", port=6379) - monkeypatch.setattr(redis, "init_async_client", lambda: fake) + redis = _fake_redis(monkeypatch) cache_a = DualCache(redis_cache=redis) cache_b = DualCache(redis_cache=redis) @@ -188,14 +197,64 @@ async def test_disconnect_on_one_worker_invalidates_other_workers_via_redis(monk @pytest.mark.asyncio -async def test_token_cache_uses_redis_only_when_attached(): - """Backend selection is pinned: Redis attached -> Redis only; no Redis -> the DualCache.""" - from litellm.caching.redis_cache import RedisCache +async def test_redis_read_failure_falls_back_to_the_database(monkeypatch): + """A broken Redis must not 401 a connected user: reads degrade to a cache miss.""" from litellm.proxy.credential_endpoints import user_provider_credentials as upc - bare = DualCache() - assert upc._token_cache(bare) is bare + redis = _fake_redis(monkeypatch) + monkeypatch.setattr(redis, "async_get_cache", AsyncMock(side_effect=RuntimeError("redis down"))) + cache = DualCache(redis_cache=redis) - redis = RedisCache(host="localhost", port=6379) - with_redis = DualCache(redis_cache=redis) - assert upc._token_cache(with_redis) is redis + payload = GithubCopilotUserConnectionPayload(access_token="gho_secret", github_login="octo") + table = MagicMock() + table.find_many = AsyncMock(return_value=[_row(credential_b64=upc._encode(payload))]) + prisma_client = _prisma(table) + + assert await aget_user_provider_tokens(prisma_client, cache, "user-a", ["copilot-cred"]) == { + "copilot-cred": "gho_secret" + } + + +@pytest.mark.asyncio +async def test_redis_write_and_delete_failures_do_not_fail_the_request(monkeypatch): + redis = _fake_redis(monkeypatch) + monkeypatch.setattr(redis, "async_set_cache", AsyncMock(side_effect=RuntimeError("redis down"))) + monkeypatch.setattr(redis, "async_delete_cache", AsyncMock(side_effect=RuntimeError("redis down"))) + cache = DualCache(redis_cache=redis) + + payload = GithubCopilotUserConnectionPayload(access_token="gho_secret", github_login="octo") + from litellm.proxy.credential_endpoints import user_provider_credentials as upc + + table = MagicMock() + table.find_many = AsyncMock(return_value=[_row(credential_b64=upc._encode(payload))]) + prisma_client = _prisma(table) + + assert await aget_user_provider_tokens(prisma_client, cache, "user-a", ["copilot-cred"]) == { + "copilot-cred": "gho_secret" + } + await invalidate_user_provider_credential_cache(cache, "user-a", "copilot-cred") + + +@pytest.mark.asyncio +async def test_without_redis_no_worker_local_entries_are_kept(): + """No Redis -> no cache at all: a disconnect on another worker cannot leave a + stale local entry, because none was ever written.""" + from litellm.proxy.credential_endpoints import user_provider_credentials as upc + + payload = GithubCopilotUserConnectionPayload(access_token="gho_secret", github_login="octo") + ciphertext = upc._encode(payload) + table = MagicMock() + table.find_many = AsyncMock(side_effect=[[_row(credential_b64=ciphertext)], [_row(credential_b64=ciphertext)], []]) + prisma_client = _prisma(table) + cache_a = DualCache() + cache_b = DualCache() + + assert await aget_user_provider_tokens(prisma_client, cache_a, "user-a", ["copilot-cred"]) == { + "copilot-cred": "gho_secret" + } + assert await aget_user_provider_tokens(prisma_client, cache_b, "user-a", ["copilot-cred"]) == { + "copilot-cred": "gho_secret" + } + await invalidate_user_provider_credential_cache(cache_b, "user-a", "copilot-cred") + assert await aget_user_provider_tokens(prisma_client, cache_a, "user-a", ["copilot-cred"]) == {} + assert table.find_many.await_count == 3 From 8a291fc5e5f62ab6c3af86d56843f138fc148674 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 22:25:21 +0000 Subject: [PATCH 14/38] fix(lint): suppress mutable return annotation after base merge --- litellm/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/utils.py b/litellm/utils.py index eea4cebd48b7..f3dde36c5120 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8295,7 +8295,7 @@ def convert_to_dict(message: BaseModel | dict) -> dict: def convert_list_message_to_dict( messages: Sequence[BaseModel | Mapping[str, object]], -) -> list[dict[str, object]]: +) -> list[dict[str, object]]: # mutable-ok: callers mutate the returned message dicts def _as_message_value(message: BaseModel | Mapping[str, object]) -> AllMessageValues: return cast(AllMessageValues, convert_to_dict(message)) # cast-ok: message dicts satisfy the TypedDict shape From 9fd2de9ad319a093fe41ee553ede20ee30796f3b Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 22:30:09 +0000 Subject: [PATCH 15/38] style: shorten rebind-ok reason so the dispatch line stays formatted --- litellm/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/main.py b/litellm/main.py index caca7a33ad5d..05c2fce34d55 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -6152,7 +6152,7 @@ def completion( response = _complete_langgraph(_dispatch_ctx) elif custom_llm_provider == "microsoft_365_copilot": - response = _complete_microsoft_365_copilot(_dispatch_ctx) # rebind-ok: provider branches share the common return + response = _complete_microsoft_365_copilot(_dispatch_ctx) # rebind-ok: shared provider return elif custom_llm_provider == "langflow": # LangFlow - Visual AI Agent Platform From eda1cc26a987bf4ab6bb35511b677e855e9840b3 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 22:55:11 +0000 Subject: [PATCH 16/38] fix(github_copilot): harden per-user token cache, overrides, fallbacks, headers and device flow --- litellm/llms/github_copilot/common_utils.py | 10 + .../embedding/transformation.py | 5 +- .../github_copilot/messages/transformation.py | 3 + .../responses/transformation.py | 3 + litellm/main.py | 3 + litellm/models/credentials.py | 21 ++ .../proxy/credential_endpoints/endpoints.py | 77 +++---- .../user_provider_credentials.py | 34 +++- litellm/proxy/litellm_pre_call_utils.py | 35 +++- ...github_copilot_embedding_transformation.py | 30 ++- ..._github_copilot_messages_transformation.py | 22 +- ...github_copilot_responses_transformation.py | 44 +++- .../llms/github_copilot/test_per_user_auth.py | 53 +++-- .../credential_endpoints/test_endpoints.py | 189 ++++++++++++++---- .../test_user_provider_credentials.py | 35 +++- .../unit/proxy/test_litellm_pre_call_utils.py | 110 +++++++++- ...ubCopilotConnectModal.integration.test.tsx | 2 + .../model_add/GithubCopilotConnectModal.tsx | 2 +- ...UserConnectionsPanel.integration.test.tsx} | 0 .../model_add/github_copilot_connect_flow.ts | 2 + .../src/components/networking.tsx | 7 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 13 +- 22 files changed, 572 insertions(+), 128 deletions(-) rename ui/litellm-dashboard/src/components/model_add/{UserConnectionsPanel.test.tsx => UserConnectionsPanel.integration.test.tsx} (100%) diff --git a/litellm/llms/github_copilot/common_utils.py b/litellm/llms/github_copilot/common_utils.py index 110a6584ec26..dcdaf0f8525a 100644 --- a/litellm/llms/github_copilot/common_utils.py +++ b/litellm/llms/github_copilot/common_utils.py @@ -2,6 +2,7 @@ Constants for Copilot integration """ +from collections.abc import MutableMapping from typing import Final from uuid import uuid4 @@ -75,3 +76,12 @@ def get_copilot_default_headers(api_key: str) -> dict: "x-request-id": str(uuid4()), "x-vscode-user-agent-library-version": "electron-fetch", } + + +def pin_session_authorization(headers: MutableMapping[str, str], session_token: str) -> None: + """The stored per-user session token owns Authorization outright: drop every + caller-supplied bearer header (any casing) so it can never displace the + session's on the wire.""" + for key in tuple(k for k in headers if k.lower() == "authorization"): + del headers[key] + headers["Authorization"] = f"Bearer {session_token}" diff --git a/litellm/llms/github_copilot/embedding/transformation.py b/litellm/llms/github_copilot/embedding/transformation.py index b4937912a8d7..a8c4dc10e78f 100644 --- a/litellm/llms/github_copilot/embedding/transformation.py +++ b/litellm/llms/github_copilot/embedding/transformation.py @@ -25,6 +25,7 @@ DEFAULT_GITHUB_COPILOT_API_BASE, GetAPIKeyError, get_copilot_default_headers, + pin_session_authorization, ) from ..per_user_auth import require_github_copilot_user_session @@ -64,7 +65,9 @@ def validate_environment( """ user_session: Final = require_github_copilot_user_session(litellm_params) # pyright: ignore[reportUnknownArgumentType] # litellm_params arrives as an untyped request dict if user_session is not None: - return {**get_copilot_default_headers(user_session.token), **headers} + merged_headers: Final = {**get_copilot_default_headers(user_session.token), **headers} + pin_session_authorization(merged_headers, user_session.token) + return merged_headers try: # Get GitHub Copilot API key via OAuth api_key = self.authenticator.get_api_key() diff --git a/litellm/llms/github_copilot/messages/transformation.py b/litellm/llms/github_copilot/messages/transformation.py index 871f9186a640..96e141c9f50c 100644 --- a/litellm/llms/github_copilot/messages/transformation.py +++ b/litellm/llms/github_copilot/messages/transformation.py @@ -10,6 +10,7 @@ DEFAULT_GITHUB_COPILOT_API_BASE, GetAPIKeyError, get_copilot_default_headers, + pin_session_authorization, ) from ..per_user_auth import require_github_copilot_user_session @@ -91,6 +92,8 @@ def validate_anthropic_messages_environment( for key, value in copilot_headers.items(): if key not in headers: headers[key] = value + if user_session is not None: + pin_session_authorization(headers, user_session.token) headers["openai-intent"] = "messages-proxy" headers["x-interaction-type"] = "messages-proxy" diff --git a/litellm/llms/github_copilot/responses/transformation.py b/litellm/llms/github_copilot/responses/transformation.py index 5dc39bb9bb57..0cb91ced5138 100644 --- a/litellm/llms/github_copilot/responses/transformation.py +++ b/litellm/llms/github_copilot/responses/transformation.py @@ -30,6 +30,7 @@ DEFAULT_GITHUB_COPILOT_API_BASE, GetAPIKeyError, get_copilot_default_headers, + pin_session_authorization, ) from ..per_user_auth import require_github_copilot_user_session @@ -217,6 +218,8 @@ def validate_environment( # Merge with existing headers (user's extra_headers take priority) merged_headers: Final = {**copilot_headers, **headers} + if user_session is not None: + pin_session_authorization(merged_headers, user_session.token) # Analyze input to determine additional headers input_param: Final = self._get_input_from_params(litellm_params) diff --git a/litellm/main.py b/litellm/main.py index 05c2fce34d55..1863a1822a13 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -2650,6 +2650,7 @@ def _complete_custom_openai( from litellm.llms.github_copilot.authenticator import Authenticator from litellm.llms.github_copilot.common_utils import ( get_copilot_default_headers, + pin_session_authorization, ) from litellm.llms.github_copilot.per_user_auth import ( require_github_copilot_user_session, @@ -2661,6 +2662,8 @@ def _complete_custom_openai( ) if extra_headers: copilot_headers.update(extra_headers) + if user_session is not None: + pin_session_authorization(copilot_headers, user_session.token) extra_headers = copilot_headers use_base_llm_http_handler: Final = get_secret_bool("EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER") diff --git a/litellm/models/credentials.py b/litellm/models/credentials.py index 0bfed1240ebf..e4f2fb26f5fe 100644 --- a/litellm/models/credentials.py +++ b/litellm/models/credentials.py @@ -69,6 +69,27 @@ class UserConnectionStartResponse(LiteLLMBaseModel): verification_uri: str expires_in: int interval: int + flow_handle: str + + +class UserConnectionPollRequest(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + flow_handle: str + + +class UserConnectionFlowHandle(LiteLLMBaseModel): + """The stateless device-flow ticket: the pending ``device_code`` travels + encrypted inside ``flow_handle`` instead of a worker-local cache entry, so a + poll landing on any worker can complete the connection.``""" + + model_config = ConfigDict(frozen=True) + + user_id: str + credential_name: str + device_code: str = Field(repr=False) + interval: int + expires_at: float UserConnectionPollStatus: TypeAlias = Literal[ diff --git a/litellm/proxy/credential_endpoints/endpoints.py b/litellm/proxy/credential_endpoints/endpoints.py index 2689404991e3..35a947bf55f8 100644 --- a/litellm/proxy/credential_endpoints/endpoints.py +++ b/litellm/proxy/credential_endpoints/endpoints.py @@ -2,7 +2,7 @@ CRUD endpoints for storing reusable credentials. """ -import json +import time from collections.abc import Mapping from typing import ( Annotated, @@ -11,14 +11,13 @@ ) from fastapi import APIRouter, Depends, HTTPException, Path, Request, Response, status -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError import litellm from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger from litellm.constants import ( GITHUB_COPILOT_AUTH_TYPE_KEY, - GITHUB_COPILOT_DEVICE_FLOW_CACHE_PREFIX, GITHUB_COPILOT_PER_USER_AUTH_TYPE, ) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor @@ -32,6 +31,8 @@ from litellm.models.credentials import ( UpdateCredentialItem, UserConnectionDeleteResponse, + UserConnectionFlowHandle, + UserConnectionPollRequest, UserConnectionPollResponse, UserConnectionStartResponse, UserProviderConnection, @@ -66,6 +67,7 @@ decode_user_provider_credential, delete_user_provider_credential, delete_user_provider_credentials_for_credential, + drop_user_provider_credential_cache, invalidate_user_provider_credential_cache, list_user_provider_credentials, list_user_provider_credentials_for_credential, @@ -466,11 +468,6 @@ def _per_user_credential_or_404(credential_name: str) -> CredentialItem: return credential -def _device_flow_cache_key(user_id: str, credential_name: str) -> str: - pair: Final = json.dumps([user_id, credential_name], separators=(",", ":")) - return f"{GITHUB_COPILOT_DEVICE_FLOW_CACHE_PREFIX}:{pair}" - - def _per_user_credential_names() -> tuple[str, ...]: return tuple( credential.credential_name @@ -531,6 +528,21 @@ async def list_user_connections( raise handle_exception_on_proxy(e) +def _decode_flow_handle(flow_handle: str) -> UserConnectionFlowHandle | None: + decrypted: Final = decrypt_value_helper( + value=flow_handle, + key="device_flow_handle", + exception_type="debug", + return_original_value=False, + ) + if decrypted is None: + return None + try: + return UserConnectionFlowHandle.model_validate_json(decrypted) + except ValidationError: + return None + + @router.post( "/credentials/{credential_name:path}/user_connection/start", dependencies=[Depends(user_api_key_auth)], @@ -546,7 +558,7 @@ async def start_user_connection( ) -> UserConnectionStartResponse: """Begin a GitHub device flow for the calling user's connection to a per-user credential.""" from litellm.llms.github_copilot.per_user_auth import astart_device_flow - from litellm.proxy.proxy_server import prisma_client, user_api_key_cache + from litellm.proxy.proxy_server import prisma_client try: user_id: Final = _authenticated_user_id(user_api_key_dict) @@ -557,16 +569,21 @@ async def start_user_connection( detail={"error": CommonProxyErrors.db_not_connected_error.value}, ) flow: Final = await astart_device_flow() - await user_api_key_cache.async_set_cache( - _device_flow_cache_key(user_id, credential_name), - encrypt_value_helper(flow.device_code), - ttl=flow.expires_in, + handle: Final = encrypt_value_helper( + UserConnectionFlowHandle( + user_id=user_id, + credential_name=credential_name, + device_code=flow.device_code, + interval=flow.interval, + expires_at=time.time() + flow.expires_in, + ).model_dump_json() ) return UserConnectionStartResponse( user_code=flow.user_code, verification_uri=flow.verification_uri, expires_in=flow.expires_in, interval=flow.interval, + flow_handle=handle, ) except Exception as e: # noqa: BLE001 # endpoint boundary: every failure becomes the proxy error contract raise handle_exception_on_proxy(e) @@ -582,6 +599,7 @@ async def start_user_connection( async def poll_user_connection( request: Request, fastapi_response: Response, + body: UserConnectionPollRequest, credential_name: str = Path(..., description="The credential name, percent-decoded"), user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI resolves the dependency from the default ) -> UserConnectionPollResponse: @@ -601,31 +619,25 @@ async def poll_user_connection( status_code=500, detail={"error": CommonProxyErrors.db_not_connected_error.value}, ) - cache_key: Final = _device_flow_cache_key(user_id, credential_name) - encrypted_device_code: Final[object] = await user_api_key_cache.async_get_cache(cache_key) - device_code: Final = ( - decrypt_value_helper(str(encrypted_device_code), key="device_code") - if isinstance(encrypted_device_code, str) and encrypted_device_code - else None - ) - if not device_code: - return UserConnectionPollResponse(status="expired") - - poll: Final = await apoll_device_flow(device_code) - if poll.status in ("expired", "denied"): - await user_api_key_cache.async_delete_cache(cache_key) - return UserConnectionPollResponse(status=poll.status, interval=poll.interval) - if poll.status in ("pending", "slow_down"): + handle: Final = _decode_flow_handle(body.flow_handle) + if ( + handle is None + or handle.user_id != user_id + or handle.credential_name != credential_name + or handle.expires_at <= time.time() + ): + raise HTTPException(status_code=400, detail="invalid or expired flow_handle") + + poll: Final = await apoll_device_flow(handle.device_code) + if poll.status in ("expired", "denied", "pending", "slow_down"): return UserConnectionPollResponse(status=poll.status, interval=poll.interval) github_token: Final = poll.access_token if not github_token: - await user_api_key_cache.async_delete_cache(cache_key) raise HTTPException(status_code=502, detail="GitHub device flow completed without a token") try: await acheck_copilot_seat(user_id=user_id, github_token=github_token, credential_name=credential_name) except litellm.CallerCredentialAuthenticationError: - await user_api_key_cache.async_delete_cache(cache_key) return UserConnectionPollResponse(status="no_copilot_seat") github_login: Final = await afetch_github_login(github_token) await upsert_user_provider_credential( @@ -635,8 +647,7 @@ async def poll_user_connection( _GITHUB_COPILOT_PROVIDER, GithubCopilotUserConnectionPayload(access_token=github_token, github_login=github_login), ) - await user_api_key_cache.async_delete_cache(cache_key) - await invalidate_user_provider_credential_cache(user_api_key_cache, user_id, credential_name) + await drop_user_provider_credential_cache(user_api_key_cache, user_id, credential_name) return UserConnectionPollResponse(status="connected", github_login=github_login) except litellm.CallerCredentialRateLimitError as e: raise HTTPException(status_code=429, detail=str(e)) @@ -671,7 +682,6 @@ async def delete_user_connection( ) prior: Final = await delete_user_provider_credential(prisma_client, user_id, credential_name) await invalidate_user_provider_credential_cache(user_api_key_cache, user_id, credential_name) - await user_api_key_cache.async_delete_cache(_device_flow_cache_key(user_id, credential_name)) if prior is not None: evict_copilot_user_session(user_id, prior.access_token) return UserConnectionDeleteResponse(status="disconnected") @@ -705,7 +715,6 @@ async def _purge_user_connections_for_credential(credential_name: str) -> None: access_tokens[row.user_id] = decoded.access_token for user_id in user_ids: await invalidate_user_provider_credential_cache(user_api_key_cache, user_id, credential_name) - await user_api_key_cache.async_delete_cache(_device_flow_cache_key(user_id, credential_name)) token = access_tokens.get(user_id) if token: evict_copilot_user_session(user_id, token) diff --git a/litellm/proxy/credential_endpoints/user_provider_credentials.py b/litellm/proxy/credential_endpoints/user_provider_credentials.py index 475f3a211ebe..02445344e195 100644 --- a/litellm/proxy/credential_endpoints/user_provider_credentials.py +++ b/litellm/proxy/credential_endpoints/user_provider_credentials.py @@ -160,9 +160,9 @@ async def _try_cache_get(token_cache: "RedisCache", key: str) -> object: return None -async def _try_cache_set(token_cache: "RedisCache", key: str, value: str) -> None: +async def _try_cache_set(token_cache: "RedisCache", key: str, value: str, nx: bool = False) -> None: try: - await token_cache.async_set_cache(key, value, ttl=GITHUB_COPILOT_USER_CREDENTIAL_CACHE_TTL_SECONDS) + await token_cache.async_set_cache(key, value, nx=nx, ttl=GITHUB_COPILOT_USER_CREDENTIAL_CACHE_TTL_SECONDS) except Exception: # noqa: BLE001 # caching is best-effort; a Redis outage is not worth failing the request verbose_proxy_logger.warning("aget_user_provider_tokens: Redis set failed; skipping the cache write") @@ -173,13 +173,37 @@ async def invalidate_user_provider_credential_cache( user_id: str, credential_name: str, ) -> None: + """Write the not-connected tombstone rather than deleting: a read in flight + before the disconnect must not fill the token back in behind it (fills are + set-if-absent).""" token_cache: Final = cache.redis_cache if token_cache is None: return try: - await token_cache.async_delete_cache(_cache_key(user_id, credential_name)) + await token_cache.async_set_cache( + _cache_key(user_id, credential_name), + _NOT_CONNECTED, + ttl=GITHUB_COPILOT_USER_CREDENTIAL_CACHE_TTL_SECONDS, + ) except Exception: # noqa: BLE001 # a Redis outage must not fail a disconnect - verbose_proxy_logger.warning("invalidate_user_provider_credential_cache: Redis delete failed") + verbose_proxy_logger.warning("invalidate_user_provider_credential_cache: Redis tombstone failed") + + +@with_service_target("user_provider_connections") +async def drop_user_provider_credential_cache( + cache: DualCache, + user_id: str, + credential_name: str, +) -> None: + """Connect path: delete the key outright so the next read fills fresh (a + tombstone would linger for the TTL and hide the new connection).""" + token_cache: Final = cache.redis_cache + if token_cache is None: + return + try: + await token_cache.async_delete_cache(_cache_key(user_id, credential_name)) + except Exception: # noqa: BLE001 # a Redis outage must not fail a connect + verbose_proxy_logger.warning("drop_user_provider_credential_cache: Redis delete failed") @with_service_target("user_provider_connections") @@ -229,7 +253,7 @@ async def aget_user_provider_tokens( found: Final = {row.credential_name: row.credential_b64 for row in rows} for name in misses: if token_cache is not None: - await _try_cache_set(token_cache, _cache_key(user_id, name), found.get(name, _NOT_CONNECTED)) + await _try_cache_set(token_cache, _cache_key(user_id, name), found.get(name, _NOT_CONNECTED), nx=True) if name in found: cached[name] = found[name] return { diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 32c6c664aaef..efd4dd575dfd 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -2699,6 +2699,29 @@ def _fallback_edges( return frozenset(targets) +def _request_fallback_groups( + data: Mapping[str, object], + model_group: str, +) -> frozenset[str]: + """Request-level fallbacks (``fallbacks``, ``context_window_fallbacks``, + ``content_policy_fallbacks``) in the shapes routing accepts: a list of + target group names applying to the request, or a list of + ``{model_group: [targets]}`` maps matching this model.""" + targets: Final[set[str]] = set() # mutable-ok: accumulates request-level fallback targets + for key in ("fallbacks", "context_window_fallbacks", "content_policy_fallbacks"): + entries: Final = data.get(key) + if not isinstance(entries, list): + continue + for entry in entries: + if isinstance(entry, str): + targets.add(entry) + elif isinstance(entry, dict): + raw = entry.get(model_group) or entry.get("*") + values = raw if isinstance(raw, list) else ([raw] if raw is not None else []) + targets.update(value for value in values if isinstance(value, str)) + return frozenset(targets) + + def _fallback_target_groups( llm_router: litellm.Router, model_group: str, @@ -2774,17 +2797,19 @@ async def _resolve_user_provider_credentials_for_request( one that fails the call with a 401 when per-user mode applies.""" user_id: Final = authenticated_user_id model: Final = data.get("model") - if not isinstance(user_id, str) or not user_id or llm_router is None or not isinstance(model, str): + if llm_router is None or not isinstance(model, str): return - model_groups: Final = frozenset({model} | _fallback_target_groups(llm_router, model)) + model_groups: Final = frozenset( + {model} | _fallback_target_groups(llm_router, model) | _request_fallback_groups(data, model) + ) credential_names: Final = _per_user_credential_names_for_groups(llm_router, model_groups, team_id) - if not credential_names: - return - if "litellm_credential_name" in data: + if credential_names and "litellm_credential_name" in data: raise HTTPException( status_code=400, detail="litellm_credential_name cannot be set in the request body for a model that uses per-user GitHub OAuth", ) + if not isinstance(user_id, str) or not user_id or not credential_names: + return from litellm.proxy.proxy_server import prisma_client, user_api_key_cache from litellm.types.proxy.litellm_pre_call_utils import RedactedDict diff --git a/tests/unit/llms/github_copilot/embedding/test_github_copilot_embedding_transformation.py b/tests/unit/llms/github_copilot/embedding/test_github_copilot_embedding_transformation.py index f6576dcc6f4b..88fc8ba6e6ae 100644 --- a/tests/unit/llms/github_copilot/embedding/test_github_copilot_embedding_transformation.py +++ b/tests/unit/llms/github_copilot/embedding/test_github_copilot_embedding_transformation.py @@ -76,9 +76,7 @@ def test_github_copilot_embedding_config_get_complete_url(): assert url == "https://api.githubcopilot.com/embeddings" # Test with custom API base from authenticator - config.authenticator.get_api_base.return_value = ( - "https://api.enterprise.githubcopilot.com" - ) + config.authenticator.get_api_base.return_value = "https://api.enterprise.githubcopilot.com" url = config.get_complete_url( api_base=None, api_key=None, @@ -271,13 +269,32 @@ def test_validate_environment_uses_per_user_session_and_skips_authenticator(): session = GithubCopilotUserSession(token="user-copilot-token", api_base="https://api.githubcopilot.com") headers = config.validate_environment( - headers={}, model="github_copilot/text-embedding-3-small", messages=[], optional_params={}, + headers={}, + model="github_copilot/text-embedding-3-small", + messages=[], + optional_params={}, litellm_params={"github_copilot_user_session": session}, ) assert headers["Authorization"] == "Bearer user-copilot-token" config.authenticator.get_api_key.assert_not_called() +def test_per_user_session_token_wins_over_caller_authorization(): + from litellm.llms.github_copilot.per_user_auth import GithubCopilotUserSession + + config = GithubCopilotEmbeddingConfig() + config.authenticator = MagicMock() + session = GithubCopilotUserSession(token="user-copilot-token", api_base="https://api.githubcopilot.com") + headers = config.validate_environment( + headers={"Authorization": "Bearer caller-token"}, + model="github_copilot/text-embedding-3-small", + messages=[], + optional_params={}, + litellm_params={"github_copilot_user_session": session}, + ) + assert headers["Authorization"] == "Bearer user-copilot-token" + + def test_get_complete_url_prefers_per_user_session_api_base(): from litellm.llms.github_copilot.per_user_auth import GithubCopilotUserSession @@ -285,7 +302,10 @@ def test_get_complete_url_prefers_per_user_session_api_base(): config.authenticator = MagicMock() session = GithubCopilotUserSession(token="t", api_base="https://tenant.githubcopilot.com") url = config.get_complete_url( - api_base="https://attacker.example", api_key=None, model="m", optional_params={}, + api_base="https://attacker.example", + api_key=None, + model="m", + optional_params={}, litellm_params={"github_copilot_user_session": session}, ) assert url == "https://tenant.githubcopilot.com/embeddings" diff --git a/tests/unit/llms/github_copilot/messages/test_github_copilot_messages_transformation.py b/tests/unit/llms/github_copilot/messages/test_github_copilot_messages_transformation.py index 2b2a0b3d3872..b029e5aba0b7 100644 --- a/tests/unit/llms/github_copilot/messages/test_github_copilot_messages_transformation.py +++ b/tests/unit/llms/github_copilot/messages/test_github_copilot_messages_transformation.py @@ -335,7 +335,10 @@ def test_validate_environment_uses_per_user_session_and_skips_authenticator(): session = GithubCopilotUserSession(token="user-copilot-token", api_base="https://tenant.githubcopilot.com/") headers, api_base = config.validate_anthropic_messages_environment( - headers={}, model="github_copilot/claude-sonnet-4.5", messages=[], optional_params={}, + headers={}, + model="github_copilot/claude-sonnet-4.5", + messages=[], + optional_params={}, litellm_params={"github_copilot_user_session": session}, ) assert headers["Authorization"] == "Bearer user-copilot-token" @@ -343,6 +346,23 @@ def test_validate_environment_uses_per_user_session_and_skips_authenticator(): config.authenticator.get_api_key.assert_not_called() +def test_per_user_session_token_wins_over_caller_authorization(): + from litellm.llms.github_copilot.per_user_auth import GithubCopilotUserSession + + config = GithubCopilotAnthropicMessagesConfig() + config.authenticator = MagicMock() + config.authenticator.get_api_base.return_value = "https://api.githubcopilot.com" + session = GithubCopilotUserSession(token="user-copilot-token", api_base="https://api.githubcopilot.com") + headers, _ = config.validate_anthropic_messages_environment( + headers={"authorization": "Bearer caller-token"}, + model="github_copilot/claude-sonnet-4.5", + messages=[], + optional_params={}, + litellm_params={"github_copilot_user_session": session}, + ) + assert headers["Authorization"] == "Bearer user-copilot-token" + + def test_transform_response_carries_upstream_usage(): config = GithubCopilotAnthropicMessagesConfig() raw = httpx.Response( diff --git a/tests/unit/llms/github_copilot/responses/test_github_copilot_responses_transformation.py b/tests/unit/llms/github_copilot/responses/test_github_copilot_responses_transformation.py index e24027504e8a..12fdc3741234 100644 --- a/tests/unit/llms/github_copilot/responses/test_github_copilot_responses_transformation.py +++ b/tests/unit/llms/github_copilot/responses/test_github_copilot_responses_transformation.py @@ -764,13 +764,44 @@ def test_validate_environment_uses_per_user_session_and_skips_authenticator(): session = GithubCopilotUserSession(token="user-copilot-token", api_base="https://api.githubcopilot.com") headers = config.validate_environment( - headers={}, model="github_copilot/gpt-5.1", + headers={}, + model="github_copilot/gpt-5.1", litellm_params={"github_copilot_user_session": session}, ) assert headers["Authorization"] == "Bearer user-copilot-token" config.authenticator.get_api_key.assert_not_called() +def test_per_user_session_token_wins_over_caller_authorization(): + """extra_headers["Authorization"] (any casing) must never displace the per-user + session token on the outgoing request.""" + from litellm.llms.github_copilot.per_user_auth import GithubCopilotUserSession + + config = GithubCopilotResponsesAPIConfig() + config.authenticator = MagicMock() + session = GithubCopilotUserSession(token="user-copilot-token", api_base="https://api.githubcopilot.com") + headers = config.validate_environment( + headers={"Authorization": "Bearer caller-token", "authorization": "Bearer caller-token-lower"}, + model="github_copilot/gpt-5.1", + litellm_params={"github_copilot_user_session": session}, + ) + assert headers["Authorization"] == "Bearer user-copilot-token" + assert "authorization" not in headers + + +def test_shared_mode_keeps_caller_authorization(): + """Pin: without a session, caller extra_headers still override the shared token.""" + config = GithubCopilotResponsesAPIConfig() + config.authenticator = MagicMock() + config.authenticator.get_api_key.return_value = "shared-api-key" + headers = config.validate_environment( + headers={"Authorization": "Bearer caller-token"}, + model="github_copilot/gpt-5.1", + litellm_params={}, + ) + assert headers["Authorization"] == "Bearer caller-token" + + def test_transform_response_carries_upstream_usage(): config = GithubCopilotResponsesAPIConfig() raw = httpx.Response( @@ -781,8 +812,15 @@ def test_transform_response_carries_upstream_usage(): "created_at": 1, "status": "completed", "model": "github_copilot/gpt-5.1", - "output": [{"type": "message", "id": "m1", "role": "assistant", "status": "completed", - "content": [{"type": "output_text", "text": "hi", "annotations": []}]}], + "output": [ + { + "type": "message", + "id": "m1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "hi", "annotations": []}], + } + ], "usage": {"input_tokens": 9, "output_tokens": 4, "total_tokens": 13}, }, ) diff --git a/tests/unit/llms/github_copilot/test_per_user_auth.py b/tests/unit/llms/github_copilot/test_per_user_auth.py index b4de3142d925..50b9b2682260 100644 --- a/tests/unit/llms/github_copilot/test_per_user_auth.py +++ b/tests/unit/llms/github_copilot/test_per_user_auth.py @@ -100,7 +100,6 @@ def per_user_credential(): yield - def test_named_per_user_credential_selects_per_user_mode(per_user_credential): assert github_copilot_auth_mode("copilot-cred", None) is True @@ -126,7 +125,6 @@ def test_unknown_named_credential_falls_back_to_kwargs(): assert github_copilot_auth_mode("missing-cred", None) is False - @pytest.mark.parametrize( "raw,expected", [ @@ -148,7 +146,6 @@ def test_validated_copilot_api_base(raw, expected): assert validated_copilot_api_base(raw) == expected - def test_session_repr_and_str_hide_the_token(caplog): session = GithubCopilotUserSession(token="secret-copilot-token", api_base="https://api.githubcopilot.com") assert "secret-copilot-token" not in repr(session) @@ -160,7 +157,6 @@ def test_session_repr_and_str_hide_the_token(caplog): assert "secret-copilot-token" not in caplog.text - def test_exchange_returns_session_and_caches_per_user(): with github_http(_exchange_responder()) as requests: s1 = exchange_github_token("user-a", "gh-token", "copilot-cred") @@ -209,7 +205,6 @@ def test_exchange_expiry_margin_refetches_near_expiry(): assert len(requests) == 2 - @pytest.mark.asyncio async def test_aexchange_single_flight_one_http_call(): def respond(request: httpx.Request) -> httpx.Response: @@ -232,7 +227,6 @@ async def test_aexchange_401_raises_and_evicts(): assert _SESSION_CACHE.get_cache(_session_cache_key("user-a", "gh-token")) is None - def _kwargs_with_connection(token="gho_user_token"): return { "litellm_credential_name": "copilot-cred", @@ -282,7 +276,6 @@ def test_evict_drops_cached_session(): assert len(requests) == 2 - @pytest.mark.asyncio async def test_astart_device_flow_shape(): def respond(request: httpx.Request) -> httpx.Response: @@ -341,7 +334,6 @@ def respond(request: httpx.Request) -> httpx.Response: assert poll.status == "connected" and poll.access_token == "gho_x" - def test_is_github_copilot_per_user_request(): from litellm.llms.github_copilot.per_user_auth import is_github_copilot_per_user_request @@ -352,7 +344,6 @@ def test_is_github_copilot_per_user_request(): assert is_github_copilot_per_user_request({"litellm_credential_name": "copilot-cred"}) is False - def _chat_completion_response(): from litellm.utils import convert_to_model_response_object @@ -461,7 +452,6 @@ def test_shared_mode_second_identical_call_hits_response_cache( # test-quality- assert mock_completion.call_count == 1, "shared mode must still serve the second call from cache" - def _embedding_response_payload(): return { "object": "list", @@ -558,6 +548,29 @@ def test_e2e_completion_per_user_credential(per_user_credential): assert response.usage.total_tokens > 0 +def test_e2e_completion_per_user_session_token_wins_over_caller_authorization(per_user_credential): + """The session token reaches the wire even when the caller passes their own + Authorization in extra_headers (chat dispatches through _complete_custom_openai).""" + with ( + github_http(_exchange_responder()), + _no_authenticator(), + patch( + "litellm.main.openai_chat_completions.completion", + return_value=_chat_completion_response(), + ) as mock_completion, + ): + litellm.completion( + model="github_copilot/gpt-4o", + messages=[{"role": "user", "content": "hi"}], + litellm_credential_name="copilot-cred", + secret_fields=_secret_fields("user-a", "gho_a"), + extra_headers={"Authorization": "Bearer caller-token", "x-custom": "keep"}, + ) + extra_headers = mock_completion.call_args.kwargs["optional_params"]["extra_headers"] + assert extra_headers["Authorization"] == "Bearer copilot-token" + assert extra_headers["x-custom"] == "keep" + + def test_e2e_embedding_per_user_credential(per_user_credential): with ( github_http(_copilot_surface_responder(_embedding_response_payload())) as requests, @@ -733,7 +746,6 @@ async def test_e2e_anthropic_messages_shared_mode_still_uses_authenticator(tmp_p assert response["usage"]["input_tokens"] == 4 - def test_sync_exchange_single_flight_one_http_call(): import threading @@ -765,7 +777,6 @@ def test_two_users_same_github_token_do_not_share_session(): assert s1.token == s2.token == "copilot-token" - def test_router_constructs_with_per_user_credential_deployment(per_user_credential): from litellm.llms.github_copilot.authenticator import Authenticator @@ -827,9 +838,7 @@ async def test_aexchange_cancelled_waiter_does_not_cancel_the_shared_exchange(pe class _GatedClient: async def get(self, url, headers=None): await gate.wait() - return httpx.Response( - 200, json=_exchange_payload(), request=httpx.Request("GET", url) - ) + return httpx.Response(200, json=_exchange_payload(), request=httpx.Request("GET", url)) with patch( # test-quality-ok: the gate holds the HTTP edge open so two waiters overlap "litellm.llms.custom_httpx.http_handler.get_async_httpx_client", return_value=_GatedClient() @@ -904,9 +913,10 @@ async def test_per_user_streaming_call_writes_nothing_to_the_response_cache(per_ call_type="acompletion", kwargs=kwargs, ) - assert handler.should_store_result_in_cache( - original_function=litellm.acompletion, kwargs=handler.request_kwargs - ) is False + assert ( + handler.should_store_result_in_cache(original_function=litellm.acompletion, kwargs=handler.request_kwargs) + is False + ) @pytest.mark.asyncio @@ -934,6 +944,7 @@ async def test_shared_streaming_call_still_writes_response_cache(tmp_path, monke call_type="acompletion", kwargs=kwargs, ) - assert handler.should_store_result_in_cache( - original_function=litellm.acompletion, kwargs=handler.request_kwargs - ) is True + assert ( + handler.should_store_result_in_cache(original_function=litellm.acompletion, kwargs=handler.request_kwargs) + is True + ) diff --git a/tests/unit/proxy/credential_endpoints/test_endpoints.py b/tests/unit/proxy/credential_endpoints/test_endpoints.py index 7d4e76489054..ec91e25f1d4d 100644 --- a/tests/unit/proxy/credential_endpoints/test_endpoints.py +++ b/tests/unit/proxy/credential_endpoints/test_endpoints.py @@ -1558,12 +1558,14 @@ def test_user_connection_start_returns_device_flow_fields(monkeypatch): with _github_http({_DEVICE_CODE_URL: _DEVICE_FLOW_START}): response = _call_as("POST", "/credentials/copilot-cred/user_connection/start", auth=_as_user()) assert response.status_code == 200, response.text - assert response.json() == { + assert {key: response.json()[key] for key in ("user_code", "verification_uri", "expires_in", "interval")} == { "user_code": "UC-1", "verification_uri": "https://github.com/login/device", "expires_in": 900, "interval": 5, } + handle = response.json()["flow_handle"] + assert isinstance(handle, str) and "device_code" not in handle and _DEVICE_FLOW_START["device_code"] not in handle def test_user_connection_start_404s_for_shared_credential(monkeypatch): @@ -1574,11 +1576,15 @@ def test_user_connection_start_404s_for_shared_credential(monkeypatch): assert response.status_code == 404 -def test_user_connection_poll_expired_when_no_device_flow(monkeypatch): +def test_user_connection_poll_rejects_an_undecryptable_flow_handle(monkeypatch): _patch_user_connection_env(monkeypatch, [_per_user_credential()]) - response = _call_as("POST", "/credentials/copilot-cred/user_connection/poll", auth=_as_user()) - assert response.status_code == 200 - assert response.json() == {"status": "expired", "interval": None, "github_login": None} + response = _call_as( + "POST", + "/credentials/copilot-cred/user_connection/poll", + json_body={"flow_handle": "not-a-real-handle"}, + auth=_as_user(), + ) + assert response.status_code == 400 def test_user_connection_poll_connected_persists_and_reports_login(monkeypatch): @@ -1593,7 +1599,12 @@ def test_user_connection_poll_connected_persists_and_reports_login(monkeypatch): ): start = _call_as("POST", "/credentials/copilot-cred/user_connection/start", auth=_as_user()) assert start.status_code == 200, start.text - poll = _call_as("POST", "/credentials/copilot-cred/user_connection/poll", auth=_as_user()) + poll = _call_as( + "POST", + "/credentials/copilot-cred/user_connection/poll", + json_body={"flow_handle": start.json()["flow_handle"]}, + auth=_as_user(), + ) assert poll.status_code == 200, poll.text assert poll.json() == {"status": "connected", "interval": None, "github_login": "octo"} table.upsert.assert_awaited_once() @@ -1607,8 +1618,13 @@ def test_user_connection_poll_pending(monkeypatch): _ACCESS_TOKEN_URL: {"error": "authorization_pending"}, } ): - _call_as("POST", "/credentials/copilot-cred/user_connection/start", auth=_as_user()) - poll = _call_as("POST", "/credentials/copilot-cred/user_connection/poll", auth=_as_user()) + start = _call_as("POST", "/credentials/copilot-cred/user_connection/start", auth=_as_user()) + poll = _call_as( + "POST", + "/credentials/copilot-cred/user_connection/poll", + json_body={"flow_handle": start.json()["flow_handle"]}, + auth=_as_user(), + ) assert poll.json()["status"] == "pending" table.upsert.assert_not_awaited() @@ -1650,11 +1666,9 @@ def test_deleting_the_credential_purges_user_connections(monkeypatch, credential def test_user_connections_isolated_per_user(monkeypatch): """Two users connected to the same credential: A's listing shows only A's login, A's delete removes only A's row, and a poll only reads A's device-flow entry.""" - from litellm.proxy.credential_endpoints.endpoints import _device_flow_cache_key rows = [_connection_row(user_id="user-a", login="octo"), _connection_row(user_id="user-b", login="hubot")] table = _patch_user_connection_env(monkeypatch, [_per_user_credential()], rows=rows) - cache = getattr(litellm.proxy.proxy_server, "user_api_key_cache") async def _find_many_matching_user(*, where): return [row for row in rows if row.user_id == where["user_id"]] @@ -1669,12 +1683,15 @@ async def _find_many_matching_user(*, where): where = table.find_many.await_args.kwargs["where"] assert where["user_id"] == "user-a" - # B has an in-flight device flow; A's poll must not see it - cache_key_b = _device_flow_cache_key("user-b", "copilot-cred") - cache.set_cache(cache_key_b, "enc-dc-b") - poll = _call_as("POST", "/credentials/copilot-cred/user_connection/poll", auth=_as_user("user-a")) - assert poll.json()["status"] == "expired" - assert cache.get_cache(cache_key_b) == "enc-dc-b", "B's device-flow entry must be untouched" + # B's in-flight device flow handle is bound to B: A polling with it is rejected + poll = _call_as( + "POST", + "/credentials/copilot-cred/user_connection/poll", + json_body={"flow_handle": _flow_handle_for(user_id="user-b")}, + auth=_as_user("user-a"), + ) + assert poll.status_code == 400 + table.upsert.assert_not_awaited() delete = _call_as("DELETE", "/credentials/copilot-cred/user_connection", auth=_as_user("user-a")) assert delete.json() == {"status": "disconnected"} @@ -1701,7 +1718,12 @@ def respond(request): with patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client", return_value=client_obj): start = _call_as("POST", "/credentials/copilot-cred/user_connection/start", auth=_as_user()) assert start.status_code == 200 - poll = _call_as("POST", "/credentials/copilot-cred/user_connection/poll", auth=_as_user()) + poll = _call_as( + "POST", + "/credentials/copilot-cred/user_connection/poll", + json_body={"flow_handle": start.json()["flow_handle"]}, + auth=_as_user(), + ) assert poll.status_code == 200, poll.text assert poll.json()["status"] == "no_copilot_seat" table.upsert.assert_not_awaited() @@ -1722,8 +1744,13 @@ def respond(request): client_obj = AsyncHTTPHandler(transport=httpx.MockTransport(respond)) with patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client", return_value=client_obj): - _call_as("POST", "/credentials/copilot-cred/user_connection/start", auth=_as_user()) - poll = _call_as("POST", "/credentials/copilot-cred/user_connection/poll", auth=_as_user()) + start = _call_as("POST", "/credentials/copilot-cred/user_connection/start", auth=_as_user()) + poll = _call_as( + "POST", + "/credentials/copilot-cred/user_connection/poll", + json_body={"flow_handle": start.json()["flow_handle"]}, + auth=_as_user(), + ) assert poll.status_code == 429 table.upsert.assert_not_awaited() @@ -1742,7 +1769,7 @@ def test_user_connection_routes_404_for_missing_or_shared_credentials(monkeypatc _patch_user_connection_env( monkeypatch, [CredentialItem(credential_name="shared", credential_values={}, credential_info={})] ) - response = _call_as(method, path, auth=_as_user()) + response = _call_as(method, path, json_body={"flow_handle": "x"}, auth=_as_user()) assert response.status_code == 404 @@ -1757,7 +1784,7 @@ def test_user_connection_routes_404_for_missing_or_shared_credentials(monkeypatc ) def test_user_connection_routes_401_without_user_id(monkeypatch, method, path): _patch_user_connection_env(monkeypatch, [_per_user_credential()]) - response = _call_as(method, path, auth=_as_admin) # admin token has no user_id + response = _call_as(method, path, json_body={"flow_handle": "x"}, auth=_as_admin) # admin token has no user_id assert response.status_code == 401 @@ -1872,39 +1899,22 @@ def test_proxy_admin_can_patch_a_credential_to_per_user_oauth(self, restore_cred assert response.status_code == 200, response.text -def test_delete_user_connection_clears_a_pending_device_flow(monkeypatch): - """Disconnect while a device flow is in flight must remove the pending flow too: - otherwise a later poll would resurrect the deleted connection.""" - from litellm.proxy.credential_endpoints.endpoints import _device_flow_cache_key - - table = _patch_user_connection_env(monkeypatch, [_per_user_credential()], rows=[_connection_row()]) - cache = getattr(litellm.proxy.proxy_server, "user_api_key_cache") +def test_pending_device_flow_survives_disconnect_via_stateless_handle(monkeypatch): + """The device flow is stateless: no worker-local entry exists to clear on + disconnect, so a still-valid issued handle polled on any worker completes.""" + _patch_user_connection_env(monkeypatch, [_per_user_credential()], rows=[_connection_row()]) with _github_http({_DEVICE_CODE_URL: _DEVICE_FLOW_START}): start = _call_as("POST", "/credentials/copilot-cred/user_connection/start", auth=_as_user()) assert start.status_code == 200 - assert cache.get_cache(_device_flow_cache_key("user-a", "copilot-cred")) is not None, ( - "device flow start stores a pending device code" - ) - delete = _call_as("DELETE", "/credentials/copilot-cred/user_connection", auth=_as_user()) assert delete.json() == {"status": "disconnected"} - assert cache.get_cache(_device_flow_cache_key("user-a", "copilot-cred")) is None - - # a poll after disconnect sees no pending flow -> expired, and stores nothing - table.upsert.reset_mock() - poll = _call_as("POST", "/credentials/copilot-cred/user_connection/poll", auth=_as_user()) - assert poll.json() == {"status": "expired", "interval": None, "github_login": None} - table.upsert.assert_not_awaited() def test_user_connection_cache_keys_cannot_collide_on_colons(): """user_id/credential_name are joined losslessly, so ('a:b','c') and ('a','b:c') can never share a cache or device-flow entry.""" - from litellm.proxy.credential_endpoints.endpoints import _device_flow_cache_key - assert upc._cache_key("a:b", "c") != upc._cache_key("a", "b:c") - assert _device_flow_cache_key("a:b", "c") != _device_flow_cache_key("a", "b:c") def test_user_connection_slashed_credential_name_routes(monkeypatch): @@ -1921,7 +1931,12 @@ def test_user_connection_slashed_credential_name_routes(monkeypatch): assert start.status_code == 200, start.text assert start.json()["user_code"] == "UC-1" - poll = _call_as("POST", "/credentials/team/copilot/user_connection/poll", auth=_as_user()) + poll = _call_as( + "POST", + "/credentials/team/copilot/user_connection/poll", + json_body={"flow_handle": start.json()["flow_handle"]}, + auth=_as_user(), + ) assert poll.status_code == 200 assert poll.json()["status"] == "pending" @@ -1957,3 +1972,91 @@ def test_user_connection_route_check_allows_slashed_names_for_internal_users(): valid_token=UserAPIKeyAuth(user_id="u", user_role=LitellmUserRoles.INTERNAL_USER), request_data={}, ) + + +def _flow_handle_for(user_id="user-a", credential_name="copilot-cred", expires_at=None): + import time as _time + + from litellm.models.credentials import UserConnectionFlowHandle + from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper + + return encrypt_value_helper( + UserConnectionFlowHandle( + user_id=user_id, + credential_name=credential_name, + device_code="DC-1", + interval=5, + expires_at=expires_at if expires_at is not None else _time.time() + 900, + ).model_dump_json() + ) + + +def test_user_connection_poll_rejects_a_handle_bound_to_another_user(monkeypatch): + _patch_user_connection_env(monkeypatch, [_per_user_credential()]) + response = _call_as( + "POST", + "/credentials/copilot-cred/user_connection/poll", + json_body={"flow_handle": _flow_handle_for(user_id="user-b")}, + auth=_as_user("user-a"), + ) + assert response.status_code == 400 + + +def test_user_connection_poll_rejects_a_handle_bound_to_another_credential(monkeypatch): + table = _patch_user_connection_env(monkeypatch, [_per_user_credential()]) + with _github_http({_DEVICE_CODE_URL: _DEVICE_FLOW_START}): + start = _call_as("POST", "/credentials/copilot-cred/user_connection/start", auth=_as_user()) + assert start.status_code == 200 + forged = _flow_handle_for(credential_name="other-cred") + response = _call_as( + "POST", + "/credentials/copilot-cred/user_connection/poll", + json_body={"flow_handle": forged}, + auth=_as_user(), + ) + assert response.status_code == 400 + assert start.json()["flow_handle"] != forged + table.upsert.assert_not_awaited() + + +def test_user_connection_poll_rejects_an_expired_handle(monkeypatch): + import time as _time + + _patch_user_connection_env(monkeypatch, [_per_user_credential()]) + response = _call_as( + "POST", + "/credentials/copilot-cred/user_connection/poll", + json_body={"flow_handle": _flow_handle_for(expires_at=_time.time() - 1)}, + auth=_as_user(), + ) + assert response.status_code == 400 + + +def test_user_connection_poll_valid_handle_works_with_a_fresh_cache(monkeypatch): + """The handle carries the device code: a poll on a worker whose cache is empty + still completes the flow, no cross-worker state required.""" + table = _patch_user_connection_env(monkeypatch, [_per_user_credential()]) + with _github_http( + { + _DEVICE_CODE_URL: _DEVICE_FLOW_START, + _ACCESS_TOKEN_URL: {"access_token": "gho_1"}, + _COPILOT_TOKEN_URL: {"token": "copilot-tok", "expires_at": 4102444800}, + _GITHUB_USER_URL: {"login": "octo"}, + } + ): + start = _call_as("POST", "/credentials/copilot-cred/user_connection/start", auth=_as_user()) + assert start.status_code == 200, start.text + # simulate a different worker: swap in a brand new empty cache + from litellm.caching.dual_cache import DualCache + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "user_api_key_cache", DualCache()) + poll = _call_as( + "POST", + "/credentials/copilot-cred/user_connection/poll", + json_body={"flow_handle": start.json()["flow_handle"]}, + auth=_as_user(), + ) + assert poll.status_code == 200, poll.text + assert poll.json()["status"] == "connected" + table.upsert.assert_awaited_once() diff --git a/tests/unit/proxy/credential_endpoints/test_user_provider_credentials.py b/tests/unit/proxy/credential_endpoints/test_user_provider_credentials.py index e854bc00de09..939db2418355 100644 --- a/tests/unit/proxy/credential_endpoints/test_user_provider_credentials.py +++ b/tests/unit/proxy/credential_endpoints/test_user_provider_credentials.py @@ -12,6 +12,7 @@ decode_user_provider_credential, delete_user_provider_credential, delete_user_provider_credentials_for_credential, + drop_user_provider_credential_cache, get_user_provider_credential, invalidate_user_provider_credential_cache, upsert_user_provider_credential, @@ -153,7 +154,7 @@ async def test_invalidate_cache_forces_db_refetch(monkeypatch): cache = DualCache(redis_cache=_fake_redis(monkeypatch)) await aget_user_provider_tokens(prisma_client, cache, "user-a", ["copilot-cred"]) - await invalidate_user_provider_credential_cache(cache, "user-a", "copilot-cred") + await drop_user_provider_credential_cache(cache, "user-a", "copilot-cred") tokens = await aget_user_provider_tokens(prisma_client, cache, "user-a", ["copilot-cred"]) assert tokens == {"copilot-cred": "gho_new"} assert table.find_many.await_count == 2 @@ -187,13 +188,39 @@ async def test_disconnect_on_one_worker_invalidates_other_workers_via_redis(monk tokens_b = await aget_user_provider_tokens(prisma_client, cache_b, "user-a", ["copilot-cred"]) assert tokens_b == {"copilot-cred": "gho_secret"} - # worker A disconnects + # worker A disconnects: the key is overwritten with the not-connected tombstone await invalidate_user_provider_credential_cache(cache_a, "user-a", "copilot-cred") - # worker B must now see not-connected (DB returns no row on the second read) + # worker B sees the tombstone from Redis, no worker-local staleness, no DB read needed tokens_b_after = await aget_user_provider_tokens(prisma_client, cache_b, "user-a", ["copilot-cred"]) assert tokens_b_after == {} - assert table.find_many.await_count == 2 + assert table.find_many.await_count == 1 + + +@pytest.mark.asyncio +async def test_stale_fill_cannot_overwrite_a_disconnect_tombstone(monkeypatch): + """A token read racing a disconnect: the read's fill uses set-if-absent, so it + must lose to the tombstone the disconnect wrote.""" + from litellm.proxy.credential_endpoints import user_provider_credentials as upc + + redis = _fake_redis(monkeypatch) + cache = DualCache(redis_cache=redis) + + payload = GithubCopilotUserConnectionPayload(access_token="gho_secret", github_login="octo") + ciphertext = upc._encode(payload) + + async def _read_while_disconnecting(*args, **kwargs): + # the row was fetched before the disconnect landed; the fill that follows must + # lose to the tombstone instead of resurrecting the token + await invalidate_user_provider_credential_cache(cache, "user-a", "copilot-cred") + return [_row(credential_b64=ciphertext)] + + table = MagicMock() + table.find_many = AsyncMock(side_effect=_read_while_disconnecting) + prisma_client = _prisma(table) + + await aget_user_provider_tokens(prisma_client, cache, "user-a", ["copilot-cred"]) + assert await aget_user_provider_tokens(prisma_client, cache, "user-a", ["copilot-cred"]) == {} @pytest.mark.asyncio diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index c633e263f3a8..45f4c517d055 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -2970,8 +2970,6 @@ def test_add_headers_to_llm_call_by_model_group_existing_headers_in_data(): litellm.model_group_settings = original_model_group_settings -from typing import Optional - from fastapi.responses import Response from litellm.integrations.custom_logger import CustomLogger @@ -2982,7 +2980,7 @@ def test_add_headers_to_llm_call_by_model_group_existing_headers_in_data(): class TestCustomLogger(CustomLogger): def __init__(self): - self.standard_logging_object: Optional[StandardLoggingPayload] = None + self.standard_logging_object: StandardLoggingPayload | None = None super().__init__() async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -9045,6 +9043,112 @@ async def test_injects_tokens_for_per_user_deployments_only(self, monkeypatch): call_where = table.find_many.await_args.kwargs["where"] assert call_where == {"AND": ({"user_id": "user-a"}, {"credential_name": {"in": ["copilot-cred"]}})} + @pytest.mark.asyncio + async def test_service_key_cannot_override_a_per_user_credential(self, monkeypatch): + """Keys with no user_id must still hit the body check: discovery ran before the + missing-user return, so a litellm_credential_name override is a 400.""" + import pytest + + from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request + + self._env(monkeypatch) + router = MagicMock() + router.fallbacks = [] + router.context_window_fallbacks = [] + router.content_policy_fallbacks = [] + router.get_model_list = MagicMock( + return_value=[ + { + "model_name": "copilot-chat", + "litellm_params": { + "model": "github_copilot/gpt-4o", + "litellm_credential_name": "copilot-cred", + }, + } + ] + ) + router.get_deployment = MagicMock(return_value=None) + data = {"model": "copilot-chat", "litellm_credential_name": "shared-cred", "secret_fields": {}} + with pytest.raises(Exception) as exc: + await _resolve_user_provider_credentials_for_request( + data=data, + authenticated_user_id=None, + team_id=None, + llm_router=router, + ) + assert getattr(exc.value, "status_code", None) == 400 + + @pytest.mark.asyncio + async def test_service_key_without_override_leaves_data_unchanged(self, monkeypatch): + """Pin: no user_id and no override -> the function is a no-op.""" + from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request + + self._env(monkeypatch) + router = MagicMock() + router.fallbacks = [] + router.context_window_fallbacks = [] + router.content_policy_fallbacks = [] + router.get_model_list = MagicMock( + return_value=[ + { + "model_name": "copilot-chat", + "litellm_params": { + "model": "github_copilot/gpt-4o", + "litellm_credential_name": "copilot-cred", + }, + } + ] + ) + router.get_deployment = MagicMock(return_value=None) + data = {"model": "copilot-chat", "secret_fields": {}} + await _resolve_user_provider_credentials_for_request( + data=data, + authenticated_user_id=None, + team_id=None, + llm_router=router, + ) + assert data == {"model": "copilot-chat", "secret_fields": {}} + + @pytest.mark.asyncio + async def test_request_level_fallback_loads_per_user_tokens(self, monkeypatch): + """A request-level fallback naming a per-user group must surface its credential.""" + from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request + + table = self._env(monkeypatch) + router = MagicMock() + router.fallbacks = [] + router.context_window_fallbacks = [] + router.content_policy_fallbacks = [] + + def _get_model_list(model_name=None, team_id=None): + if model_name != "copilot-chat": + return [] + return [ + { + "model_name": "copilot-chat", + "litellm_params": { + "model": "github_copilot/gpt-4o", + "litellm_credential_name": "copilot-cred", + }, + } + ] + + router.get_model_list = MagicMock(side_effect=_get_model_list) + router.get_deployment = MagicMock(return_value=None) + data = { + "model": "gpt-4o", + "fallbacks": [{"gpt-4o": ["copilot-chat"]}], + "secret_fields": {}, + } + await _resolve_user_provider_credentials_for_request( + data=data, + authenticated_user_id="user-a", + team_id=None, + llm_router=router, + ) + table.find_many.assert_awaited() + assert data["secret_fields"]["user_provider_credentials_user_id"] == "user-a" + @pytest.mark.asyncio async def test_team_public_model_name_resolves_per_user_deployments(self, monkeypatch): """A caller hitting a team's public model name must get the same per-user diff --git a/ui/litellm-dashboard/src/components/model_add/GithubCopilotConnectModal.integration.test.tsx b/ui/litellm-dashboard/src/components/model_add/GithubCopilotConnectModal.integration.test.tsx index 72ff22b8e4ce..963ebab81d31 100644 --- a/ui/litellm-dashboard/src/components/model_add/GithubCopilotConnectModal.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/model_add/GithubCopilotConnectModal.integration.test.tsx @@ -16,6 +16,7 @@ const START_RESPONSE = { verification_uri: "https://github.com/login/device", expires_in: 900, interval: 5, + flow_handle: "fh-1", }; const renderModal = () => { @@ -60,6 +61,7 @@ describe("GithubCopilotConnectModal", () => { expect(networking.userConnectionPollCall).not.toHaveBeenCalled(); await advance(0.1); expect(networking.userConnectionPollCall).toHaveBeenCalledTimes(1); + expect(networking.userConnectionPollCall).toHaveBeenCalledWith("session-token", "copilot-per-user", "fh-1"); await advance(5); expect(networking.userConnectionPollCall).toHaveBeenCalledTimes(2); diff --git a/ui/litellm-dashboard/src/components/model_add/GithubCopilotConnectModal.tsx b/ui/litellm-dashboard/src/components/model_add/GithubCopilotConnectModal.tsx index 26d809deaf94..8c76b47b6dba 100644 --- a/ui/litellm-dashboard/src/components/model_add/GithubCopilotConnectModal.tsx +++ b/ui/litellm-dashboard/src/components/model_add/GithubCopilotConnectModal.tsx @@ -57,7 +57,7 @@ export default function GithubCopilotConnectModal({ dispatch({ type: "timed_out" }); return; } - userConnectionPollCall(accessToken, credentialName) + userConnectionPollCall(accessToken, credentialName, state.flowHandle) .then((response) => !cancelled && dispatch({ type: "polled", response })) .catch( (error: unknown) => !cancelled && dispatch({ type: "errored", message: extractProxyErrorMessage(error) }), diff --git a/ui/litellm-dashboard/src/components/model_add/UserConnectionsPanel.test.tsx b/ui/litellm-dashboard/src/components/model_add/UserConnectionsPanel.integration.test.tsx similarity index 100% rename from ui/litellm-dashboard/src/components/model_add/UserConnectionsPanel.test.tsx rename to ui/litellm-dashboard/src/components/model_add/UserConnectionsPanel.integration.test.tsx diff --git a/ui/litellm-dashboard/src/components/model_add/github_copilot_connect_flow.ts b/ui/litellm-dashboard/src/components/model_add/github_copilot_connect_flow.ts index f9532d7b0421..0e38fc91bb16 100644 --- a/ui/litellm-dashboard/src/components/model_add/github_copilot_connect_flow.ts +++ b/ui/litellm-dashboard/src/components/model_add/github_copilot_connect_flow.ts @@ -15,6 +15,7 @@ export type ConnectFlowState = readonly verificationUri: string; readonly expiresAt: number; readonly intervalSeconds: number; + readonly flowHandle: string; } | { readonly kind: "connected"; readonly attempt: number; readonly githubLogin: string } | { @@ -77,6 +78,7 @@ export const connectFlowReducer = (state: ConnectFlowState, event: ConnectFlowEv verificationUri: safeVerificationUri(event.response.verification_uri), expiresAt: event.now + event.response.expires_in * 1000, intervalSeconds: event.response.interval, + flowHandle: event.response.flow_handle, }; case "polled": return state.kind === "awaiting" ? afterPoll(state, event.response) : state; diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index d44dd361792b..2cb271b33a1d 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -2593,6 +2593,7 @@ export interface UserConnectionStartResponse { verification_uri: string; expires_in: number; interval: number; + flow_handle: string; } export type UserConnectionPollStatus = "pending" | "slow_down" | "expired" | "denied" | "no_copilot_seat" | "connected"; @@ -2618,8 +2619,12 @@ export const userConnectionStartCall = ( export const userConnectionPollCall = ( accessToken: string, credentialName: string, + flowHandle: string, ): Promise => - apiClient.post(`${userConnectionPath(credentialName)}/poll`, { accessToken }); + apiClient.post(`${userConnectionPath(credentialName)}/poll`, { + accessToken, + body: { flow_handle: flowHandle }, + }); export const userConnectionDeleteCall = (accessToken: string, credentialName: string): Promise => apiClient.delete(userConnectionPath(credentialName), { accessToken }); diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 10878be0b9ed..3fd79f65dd09 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -50284,6 +50284,11 @@ export interface components { /** Status */ status: string; }; + /** UserConnectionPollRequest */ + UserConnectionPollRequest: { + /** Flow Handle */ + flow_handle: string; + }; /** UserConnectionPollResponse */ UserConnectionPollResponse: { /** Github Login */ @@ -50300,6 +50305,8 @@ export interface components { UserConnectionStartResponse: { /** Expires In */ expires_in: number; + /** Flow Handle */ + flow_handle: string; /** Interval */ interval: number; /** User Code */ @@ -57619,7 +57626,11 @@ export interface operations { }; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + "application/json": components["schemas"]["UserConnectionPollRequest"]; + }; + }; responses: { /** @description Successful Response */ 200: { From fdfff3e9fd154c46b52da31d744ab92ea4a6b672 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 22:57:15 +0000 Subject: [PATCH 17/38] fix(lint): suppress MutableMapping param on session auth pinning --- litellm/llms/github_copilot/common_utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/litellm/llms/github_copilot/common_utils.py b/litellm/llms/github_copilot/common_utils.py index dcdaf0f8525a..fcd5a4cacadc 100644 --- a/litellm/llms/github_copilot/common_utils.py +++ b/litellm/llms/github_copilot/common_utils.py @@ -78,7 +78,10 @@ def get_copilot_default_headers(api_key: str) -> dict: } -def pin_session_authorization(headers: MutableMapping[str, str], session_token: str) -> None: +def pin_session_authorization( + headers: MutableMapping[str, str], # mutable-ok: the helper fixes caller-owned headers in place + session_token: str, +) -> None: """The stored per-user session token owns Authorization outright: drop every caller-supplied bearer header (any casing) so it can never displace the session's on the wire.""" From 58e99c6fa2e191391a57cddc85cda2be7cbd64a0 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 23:02:16 +0000 Subject: [PATCH 18/38] test(github_copilot): narrow pytest.raises to HTTPException for service-key override --- tests/unit/proxy/test_litellm_pre_call_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 45f4c517d055..b2fcf2e60e9b 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -9048,6 +9048,7 @@ async def test_service_key_cannot_override_a_per_user_credential(self, monkeypat """Keys with no user_id must still hit the body check: discovery ran before the missing-user return, so a litellm_credential_name override is a 400.""" import pytest + from fastapi import HTTPException from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request @@ -9069,7 +9070,7 @@ async def test_service_key_cannot_override_a_per_user_credential(self, monkeypat ) router.get_deployment = MagicMock(return_value=None) data = {"model": "copilot-chat", "litellm_credential_name": "shared-cred", "secret_fields": {}} - with pytest.raises(Exception) as exc: + with pytest.raises(HTTPException) as exc: await _resolve_user_provider_credentials_for_request( data=data, authenticated_user_id=None, From 8f732ffbda6c9586b0b525450f830110a6cce2d6 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 23:23:35 +0000 Subject: [PATCH 19/38] fix(github_copilot): fail closed on revoke errors, strip caller auth headers, widen fallback discovery --- litellm/llms/github_copilot/per_user_auth.py | 37 ++++++++ .../proxy/credential_endpoints/endpoints.py | 12 ++- .../user_provider_credentials.py | 11 ++- litellm/proxy/litellm_pre_call_utils.py | 87 ++++++++++++++----- .../llms/github_copilot/test_per_user_auth.py | 85 ++++++++++++++++++ .../credential_endpoints/test_endpoints.py | 20 +++++ .../unit/proxy/test_litellm_pre_call_utils.py | 79 +++++++++++++++++ .../github_copilot_connect_flow.test.ts | 9 +- 8 files changed, 311 insertions(+), 29 deletions(-) diff --git a/litellm/llms/github_copilot/per_user_auth.py b/litellm/llms/github_copilot/per_user_auth.py index 8dfa2a1ba2c8..98d34e23052d 100644 --- a/litellm/llms/github_copilot/per_user_auth.py +++ b/litellm/llms/github_copilot/per_user_auth.py @@ -653,6 +653,41 @@ def is_github_copilot_per_user_request(kwargs: Mapping[str, object]) -> bool: return isinstance(kwargs.get(GITHUB_COPILOT_USER_SESSION_KWARG_KEY), GithubCopilotUserSession) +def _without_authorization(headers: Mapping[str, str]) -> Mapping[str, str]: + return {k: v for k, v in headers.items() if k.lower() != "authorization"} + + +def _strip_caller_authorization( + kwargs: dict[str, object], # mutable-ok: kwargs is the request mutation channel +) -> None: + """Once a per-user session is attached, its token owns Authorization on the + wire. Caller-supplied bearer headers would be re-merged by the HTTP handler + after the transformations run, so they are rebuilt here without any + Authorization key; the caller's mappings are left untouched.""" + for key in ("extra_headers", "headers"): + value: Final = kwargs.get(key) + if isinstance(value, Mapping): + stripped: Final = _without_authorization(value) + if len(stripped) != len(value): + kwargs[key] = stripped # rebind-ok: kwargs is the request mutation channel + optional_params: Final = kwargs.get("optional_params") + if isinstance(optional_params, dict): + eh: Final = optional_params.get("extra_headers") + if isinstance(eh, Mapping): + kwargs["optional_params"] = { # rebind-ok: kwargs is the request mutation channel + **optional_params, + "extra_headers": _without_authorization(eh), + } + litellm_params: Final = kwargs.get("litellm_params") + if isinstance(litellm_params, dict): + eh2: Final = litellm_params.get("extra_headers") + if isinstance(eh2, Mapping): + kwargs["litellm_params"] = { # rebind-ok: kwargs is the request mutation channel + **litellm_params, + "extra_headers": _without_authorization(eh2), + } + + def attach_github_copilot_user_session( kwargs: dict[str, object], # mutable-ok: kwargs is the request mutation channel ) -> None: @@ -670,6 +705,7 @@ def attach_github_copilot_user_session( credential_name=credential_name, ) kwargs[GITHUB_COPILOT_USER_SESSION_KWARG_KEY] = session # rebind-ok: kwargs is the request mutation channel + _strip_caller_authorization(kwargs) async def aattach_github_copilot_user_session( @@ -689,3 +725,4 @@ async def aattach_github_copilot_user_session( credential_name=credential_name, ) kwargs[GITHUB_COPILOT_USER_SESSION_KWARG_KEY] = session # rebind-ok: kwargs is the request mutation channel + _strip_caller_authorization(kwargs) diff --git a/litellm/proxy/credential_endpoints/endpoints.py b/litellm/proxy/credential_endpoints/endpoints.py index 35a947bf55f8..47a22f683690 100644 --- a/litellm/proxy/credential_endpoints/endpoints.py +++ b/litellm/proxy/credential_endpoints/endpoints.py @@ -680,8 +680,18 @@ async def delete_user_connection( status_code=500, detail={"error": CommonProxyErrors.db_not_connected_error.value}, ) + # Tombstone first: while a token may still be cached, the DB row stays. + # If the tombstone write fails and the row were deleted, the stale token + # would keep working until its TTL with no connection left to re-check. + tombstoned: Final = await invalidate_user_provider_credential_cache( + user_api_key_cache, user_id, credential_name + ) + if not tombstoned: + raise HTTPException( + status_code=503, + detail={"error": "could not revoke cached connection, retry"}, + ) prior: Final = await delete_user_provider_credential(prisma_client, user_id, credential_name) - await invalidate_user_provider_credential_cache(user_api_key_cache, user_id, credential_name) if prior is not None: evict_copilot_user_session(user_id, prior.access_token) return UserConnectionDeleteResponse(status="disconnected") diff --git a/litellm/proxy/credential_endpoints/user_provider_credentials.py b/litellm/proxy/credential_endpoints/user_provider_credentials.py index 02445344e195..ee2969e7155d 100644 --- a/litellm/proxy/credential_endpoints/user_provider_credentials.py +++ b/litellm/proxy/credential_endpoints/user_provider_credentials.py @@ -172,21 +172,24 @@ async def invalidate_user_provider_credential_cache( cache: DualCache, user_id: str, credential_name: str, -) -> None: +) -> bool: """Write the not-connected tombstone rather than deleting: a read in flight before the disconnect must not fill the token back in behind it (fills are - set-if-absent).""" + set-if-absent). Returns False when Redis is attached and the write fails, so + the disconnect can refuse to drop the row while a stale token might linger.""" token_cache: Final = cache.redis_cache if token_cache is None: - return + return True try: await token_cache.async_set_cache( _cache_key(user_id, credential_name), _NOT_CONNECTED, ttl=GITHUB_COPILOT_USER_CREDENTIAL_CACHE_TTL_SECONDS, ) - except Exception: # noqa: BLE001 # a Redis outage must not fail a disconnect + except Exception: # noqa: BLE001 # caller decides whether a Redis outage aborts the disconnect verbose_proxy_logger.warning("invalidate_user_provider_credential_cache: Redis tombstone failed") + return False + return True @with_service_target("user_provider_connections") diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index efd4dd575dfd..adb526d3cfaf 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -2652,6 +2652,9 @@ async def add_litellm_data_to_request( authenticated_user_id=authenticated_user_id, team_id=user_api_key_dict.team_id, llm_router=llm_router, + router_settings=( + user_api_key_dict.router_settings if isinstance(user_api_key_dict.router_settings, dict) else None + ), ) ## ENFORCED PARAMS CHECK @@ -2699,40 +2702,79 @@ def _fallback_edges( return frozenset(targets) -def _request_fallback_groups( +_REQUEST_FALLBACK_KEYS: Final = ("fallbacks", "context_window_fallbacks", "content_policy_fallbacks") + + +def _fallback_lists( data: Mapping[str, object], + llm_router: litellm.Router, + router_settings: Mapping[str, object] | None, +) -> tuple[Sequence[object], ...]: + """Every fallback list the router can route this request through: the three + router-level lists, the same three keys from the request body (which replace + the router's when present, so the union is the safe superset), and the + key/team ``router_settings.fallbacks`` the proxy prefers over router-level + fallbacks in ``_configured_fallbacks``.""" + lists: Final[list[Sequence[object]]] = [] # mutable-ok: accumulates each fallback list found + for router_list in ( + llm_router.fallbacks, + llm_router.context_window_fallbacks, + llm_router.content_policy_fallbacks, + ): + if isinstance(router_list, list): + lists.append(router_list) + for key in _REQUEST_FALLBACK_KEYS: + entries: object = data.get(key) + if isinstance(entries, list): + lists.append(entries) + key_fallbacks: object = router_settings.get("fallbacks") if router_settings is not None else None + if isinstance(key_fallbacks, list): + lists.append(key_fallbacks) + return tuple(lists) + + +def _fallback_edges( + fallback_lists: Sequence[Sequence[object]], model_group: str, ) -> frozenset[str]: - """Request-level fallbacks (``fallbacks``, ``context_window_fallbacks``, - ``content_policy_fallbacks``) in the shapes routing accepts: a list of - target group names applying to the request, or a list of - ``{model_group: [targets]}`` maps matching this model.""" - targets: Final[set[str]] = set() # mutable-ok: accumulates request-level fallback targets - for key in ("fallbacks", "context_window_fallbacks", "content_policy_fallbacks"): - entries: Final = data.get(key) - if not isinstance(entries, list): - continue - for entry in entries: - if isinstance(entry, str): - targets.add(entry) - elif isinstance(entry, dict): - raw = entry.get(model_group) or entry.get("*") - values = raw if isinstance(raw, list) else ([raw] if raw is not None else []) - targets.update(value for value in values if isinstance(value, str)) + """Targets routing would pick for ``model_group`` out of each list, resolved + with the router's own matcher so exact, provider-prefixed, stripped and "*" + keys plus bare-string entries all count.""" + from litellm.router_utils.fallback_event_handlers import get_fallback_model_group + + targets: Final[set[str]] = set() # mutable-ok: accumulates fallback targets + for fallback_list in fallback_lists: + resolved: Final = get_fallback_model_group(fallbacks=fallback_list, model_group=model_group)[0] + values: Final = resolved if isinstance(resolved, list) else ([resolved] if resolved is not None else []) + targets.update(name for value in values if (name := _fallback_target_name(value)) is not None) return frozenset(targets) +def _fallback_target_name(value: object) -> str | None: + """The group a single fallback target names: a bare string, or the + ``{"model": "..."}`` advanced entry the router accepts.""" + if isinstance(value, str): + return value + if isinstance(value, dict): + model_value: Final = value.get("model") + return model_value if isinstance(model_value, str) else None + return None + + def _fallback_target_groups( llm_router: litellm.Router, + data: Mapping[str, object], + router_settings: Mapping[str, object] | None, model_group: str, ) -> frozenset[str]: - """Transitive closure over the router's fallback graph: a chain like - A -> B -> C must still surface C's per-user credential for a request to A.""" + """Transitive closure over every fallback graph that can route this request: + a chain like A -> B -> C must still surface C's per-user credential.""" + lists: Final = _fallback_lists(data, llm_router, router_settings) seen: Final[set[str]] = set() # mutable-ok: BFS visited set frontier: Final[list[str]] = [model_group] # mutable-ok: BFS work list while frontier: group = frontier.pop() - for target in _fallback_edges(llm_router, group): + for target in _fallback_edges(lists, group): if target not in seen and target != model_group: seen.add(target) frontier.append(target) @@ -2786,6 +2828,7 @@ async def _resolve_user_provider_credentials_for_request( authenticated_user_id: str | None, team_id: str | None, llm_router: litellm.Router | None, + router_settings: Mapping[str, object] | None = None, ) -> None: """Resolve the calling user's per-user provider connections into secret_fields. @@ -2799,9 +2842,7 @@ async def _resolve_user_provider_credentials_for_request( model: Final = data.get("model") if llm_router is None or not isinstance(model, str): return - model_groups: Final = frozenset( - {model} | _fallback_target_groups(llm_router, model) | _request_fallback_groups(data, model) - ) + model_groups: Final = frozenset({model} | _fallback_target_groups(llm_router, data, router_settings, model)) credential_names: Final = _per_user_credential_names_for_groups(llm_router, model_groups, team_id) if credential_names and "litellm_credential_name" in data: raise HTTPException( diff --git a/tests/unit/llms/github_copilot/test_per_user_auth.py b/tests/unit/llms/github_copilot/test_per_user_auth.py index 50b9b2682260..b4f8308e8a05 100644 --- a/tests/unit/llms/github_copilot/test_per_user_auth.py +++ b/tests/unit/llms/github_copilot/test_per_user_auth.py @@ -571,6 +571,91 @@ def test_e2e_completion_per_user_session_token_wins_over_caller_authorization(pe assert extra_headers["x-custom"] == "keep" +_EVIL_AUTH_HEADERS = {"authorization": "Bearer caller-evil", "x-keep": "1"} + + +def _assert_session_token_on_the_wire(requests): + upstream = [r for r in requests if r.url.host.endswith("githubcopilot.com")] + assert upstream, "expected the model request to the Copilot host" + assert upstream[0].headers["authorization"] == "Bearer copilot-token" + + +def test_e2e_completion_per_user_caller_authorization_cannot_override_session(per_user_credential): + """Caller-supplied Authorization must be stripped at the wrapper: chat's SDK + client owns its own httpx transport, so the capture point is the headers + handed to openai_chat_completions.completion.""" + caller_headers: dict[str, str] = dict(_EVIL_AUTH_HEADERS) + with ( + github_http(_exchange_responder()), + _no_authenticator(), + patch( + "litellm.main.openai_chat_completions.completion", + return_value=_chat_completion_response(), + ) as mock_completion, + ): + litellm.completion( + model="github_copilot/gpt-4o", + messages=[{"role": "user", "content": "hi"}], + litellm_credential_name="copilot-cred", + secret_fields=_secret_fields("user-a", "gho_a"), + extra_headers=caller_headers, + ) + extra_headers = mock_completion.call_args.kwargs["optional_params"]["extra_headers"] + assert extra_headers["Authorization"] == "Bearer copilot-token" + assert "authorization" not in extra_headers + assert extra_headers["x-keep"] == "1" + assert caller_headers == dict(_EVIL_AUTH_HEADERS), "the caller's dict must not be mutated" + + +def test_e2e_embedding_per_user_caller_authorization_cannot_override_session(per_user_credential): + with ( + github_http(_copilot_surface_responder(_embedding_response_payload())) as requests, + _no_authenticator(), + ): + litellm.embedding( + model="github_copilot/text-embedding-3-small", + input=["hi"], + litellm_credential_name="copilot-cred", + secret_fields=_secret_fields("user-a", "gho_a"), + extra_headers=dict(_EVIL_AUTH_HEADERS), + ) + _assert_session_token_on_the_wire(requests) + + +def test_e2e_responses_per_user_caller_authorization_cannot_override_session(per_user_credential): + with ( + github_http(_copilot_surface_responder(_responses_payload())) as requests, + _no_authenticator(), + ): + litellm.responses( + model="github_copilot/gpt-5.3-codex", + input=[{"role": "user", "content": "hi"}], + litellm_credential_name="copilot-cred", + secret_fields=_secret_fields("user-a", "gho_a"), + extra_headers=dict(_EVIL_AUTH_HEADERS), + ) + _assert_session_token_on_the_wire(requests) + + +@pytest.mark.asyncio +async def test_e2e_anthropic_messages_per_user_caller_authorization_cannot_override_session( + per_user_credential, +): + with ( + github_http(_copilot_surface_responder(_anthropic_messages_payload())) as requests, + _no_authenticator(), + ): + await litellm.anthropic.messages.acreate( + model="github_copilot/claude-haiku-4.5", + max_tokens=16, + messages=[{"role": "user", "content": "hi"}], + litellm_credential_name="copilot-cred", + secret_fields=_secret_fields("user-a", "gho_a"), + extra_headers=dict(_EVIL_AUTH_HEADERS), + ) + _assert_session_token_on_the_wire(requests) + + def test_e2e_embedding_per_user_credential(per_user_credential): with ( github_http(_copilot_surface_responder(_embedding_response_payload())) as requests, diff --git a/tests/unit/proxy/credential_endpoints/test_endpoints.py b/tests/unit/proxy/credential_endpoints/test_endpoints.py index ec91e25f1d4d..0f4f84c2f242 100644 --- a/tests/unit/proxy/credential_endpoints/test_endpoints.py +++ b/tests/unit/proxy/credential_endpoints/test_endpoints.py @@ -2060,3 +2060,23 @@ def test_user_connection_poll_valid_handle_works_with_a_fresh_cache(monkeypatch) assert poll.status_code == 200, poll.text assert poll.json()["status"] == "connected" table.upsert.assert_awaited_once() + + +def test_delete_user_connection_fails_closed_when_the_tombstone_write_fails(monkeypatch): + """Redis attached but its set() raises: the disconnect must not delete the row, + or the cached token would keep working until TTL with nothing left to revoke.""" + table = _patch_user_connection_env(monkeypatch, [_per_user_credential()], rows=[_connection_row()]) + from types import SimpleNamespace + + from litellm.proxy import proxy_server + + failing_redis = SimpleNamespace( + async_set_cache=AsyncMock(side_effect=ConnectionError("redis down")), + async_get_cache=AsyncMock(return_value=None), + async_delete_cache=AsyncMock(return_value=None), + ) + monkeypatch.setattr(proxy_server, "user_api_key_cache", DualCache(redis_cache=failing_redis)) + + response = _call_as("DELETE", "/credentials/copilot-cred/user_connection", auth=_as_user()) + assert response.status_code == 503, response.text + table.delete_many.assert_not_awaited() diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index b2fcf2e60e9b..db21d7c9dec8 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -9150,6 +9150,85 @@ def _get_model_list(model_name=None, team_id=None): table.find_many.assert_awaited() assert data["secret_fields"]["user_provider_credentials_user_id"] == "user-a" + @pytest.mark.asyncio + async def test_request_fallback_chain_loads_transitive_per_user_tokens(self, monkeypatch): + """A -> B -> C through request-level fallback dicts: the transitive target's + per-user credential must be discovered too.""" + from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request + + table = self._env(monkeypatch) + router = MagicMock() + router.fallbacks = [] + router.context_window_fallbacks = [] + router.content_policy_fallbacks = [] + + def _get_model_list(model_name=None, team_id=None): + if model_name != "copilot-chat": + return [] + return [ + { + "model_name": "copilot-chat", + "litellm_params": { + "model": "github_copilot/gpt-4o", + "litellm_credential_name": "copilot-cred", + }, + } + ] + + router.get_model_list = MagicMock(side_effect=_get_model_list) + router.get_deployment = MagicMock(return_value=None) + data = { + "model": "gpt-4o", + "fallbacks": [{"gpt-4o": ["backup-chat"]}, {"backup-chat": ["copilot-chat"]}], + "secret_fields": {}, + } + await _resolve_user_provider_credentials_for_request( + data=data, + authenticated_user_id="user-a", + team_id=None, + llm_router=router, + ) + table.find_many.assert_awaited() + assert data["secret_fields"]["user_provider_credentials_user_id"] == "user-a" + + @pytest.mark.asyncio + async def test_key_router_settings_fallback_loads_per_user_tokens(self, monkeypatch): + """The proxy prefers key/team router_settings.fallbacks over router-level + fallbacks (_configured_fallbacks), so discovery must read them too.""" + from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request + + table = self._env(monkeypatch) + router = MagicMock() + router.fallbacks = [] + router.context_window_fallbacks = [] + router.content_policy_fallbacks = [] + router.get_model_list = MagicMock( + side_effect=lambda model_name=None, team_id=None: ( + [ + { + "model_name": "copilot-chat", + "litellm_params": { + "model": "github_copilot/gpt-4o", + "litellm_credential_name": "copilot-cred", + }, + } + ] + if model_name == "copilot-chat" + else [] + ) + ) + router.get_deployment = MagicMock(return_value=None) + data = {"model": "gpt-4o", "secret_fields": {}} + await _resolve_user_provider_credentials_for_request( + data=data, + authenticated_user_id="user-a", + team_id=None, + llm_router=router, + router_settings={"fallbacks": [{"gpt-4o": [{"model": "copilot-chat"}]}]}, + ) + table.find_many.assert_awaited() + assert data["secret_fields"]["user_provider_credentials_user_id"] == "user-a" + @pytest.mark.asyncio async def test_team_public_model_name_resolves_per_user_deployments(self, monkeypatch): """A caller hitting a team's public model name must get the same per-user diff --git a/ui/litellm-dashboard/src/components/model_add/github_copilot_connect_flow.test.ts b/ui/litellm-dashboard/src/components/model_add/github_copilot_connect_flow.test.ts index 31a32bc0954f..72c2e783adfb 100644 --- a/ui/litellm-dashboard/src/components/model_add/github_copilot_connect_flow.test.ts +++ b/ui/litellm-dashboard/src/components/model_add/github_copilot_connect_flow.test.ts @@ -5,7 +5,13 @@ import { connectFlowReducer, type ConnectFlowState, initialConnectFlowState } fr const started = (verificationUri = "https://github.com/login/device"): ConnectFlowState => connectFlowReducer(initialConnectFlowState, { type: "started", - response: { user_code: "ABCD-1234", verification_uri: verificationUri, expires_in: 900, interval: 5 }, + response: { + user_code: "ABCD-1234", + verification_uri: verificationUri, + expires_in: 900, + interval: 5, + flow_handle: "fh-1", + }, now: 1_000, }); @@ -19,6 +25,7 @@ describe("connectFlowReducer", () => { verificationUri: "https://github.com/login/device", expiresAt: 901_000, intervalSeconds: 5, + flowHandle: "fh-1", }; expect(started()).toEqual(expected); }); From 458dd7bec809ac683b80a2ca5e8d901b4e8ae583 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 23:47:49 +0000 Subject: [PATCH 20/38] fix(github_copilot): verify revoke tombstone, bound fallback discovery, cover aliased fallbacks --- .../user_provider_credentials.py | 7 + litellm/proxy/litellm_pre_call_utils.py | 139 +++++++++++++----- .../credential_endpoints/test_endpoints.py | 22 +++ .../unit/proxy/test_litellm_pre_call_utils.py | 122 +++++++++++++++ 4 files changed, 255 insertions(+), 35 deletions(-) diff --git a/litellm/proxy/credential_endpoints/user_provider_credentials.py b/litellm/proxy/credential_endpoints/user_provider_credentials.py index ee2969e7155d..64bd6bd5d58e 100644 --- a/litellm/proxy/credential_endpoints/user_provider_credentials.py +++ b/litellm/proxy/credential_endpoints/user_provider_credentials.py @@ -189,6 +189,13 @@ async def invalidate_user_provider_credential_cache( except Exception: # noqa: BLE001 # caller decides whether a Redis outage aborts the disconnect verbose_proxy_logger.warning("invalidate_user_provider_credential_cache: Redis tombstone failed") return False + # async_set_cache swallows client errors internally, so a write that never + # landed looks identical to a success. The tombstone is only trusted when the + # key reads back as _NOT_CONNECTED; anything else means revoke failed. + readback: Final = await _try_cache_get(token_cache, _cache_key(user_id, credential_name)) + if readback != _NOT_CONNECTED: + verbose_proxy_logger.warning("invalidate_user_provider_credential_cache: tombstone not visible after write") + return False return True diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index adb526d3cfaf..e6a9c49ac7ff 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1,5 +1,6 @@ import asyncio import copy +import itertools import json import re import time @@ -7,7 +8,7 @@ from collections.abc import Mapping, MutableMapping, Sequence from datetime import datetime from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, cast +from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast from fastapi import HTTPException, Request from pydantic import TypeAdapter @@ -2682,33 +2683,15 @@ async def add_litellm_data_to_request( return data -def _fallback_edges( - llm_router: litellm.Router, - model_group: str, -) -> frozenset[str]: - targets: Final[set[str]] = set() # mutable-ok: accumulates fallback targets - for fallback_list in ( - llm_router.fallbacks, - llm_router.context_window_fallbacks, - llm_router.content_policy_fallbacks, - ): - for entry in fallback_list or (): - if not isinstance(entry, dict): - continue - for key in (model_group, "*"): - raw = entry.get(key) - values = raw if isinstance(raw, list) else ([raw] if raw is not None else []) - targets.update(value for value in values if isinstance(value, str)) - return frozenset(targets) - - _REQUEST_FALLBACK_KEYS: Final = ("fallbacks", "context_window_fallbacks", "content_policy_fallbacks") +_FALLBACK_DISCOVERY_LIMIT: Final = 256 def _fallback_lists( data: Mapping[str, object], llm_router: litellm.Router, router_settings: Mapping[str, object] | None, + team_router_settings: Mapping[str, object] | None, ) -> tuple[Sequence[object], ...]: """Every fallback list the router can route this request through: the three router-level lists, the same three keys from the request body (which replace @@ -2727,24 +2710,53 @@ def _fallback_lists( entries: object = data.get(key) if isinstance(entries, list): lists.append(entries) - key_fallbacks: object = router_settings.get("fallbacks") if router_settings is not None else None - if isinstance(key_fallbacks, list): - lists.append(key_fallbacks) + for settings in (router_settings, team_router_settings): + key_fallbacks: object = settings.get("fallbacks") if settings is not None else None + if isinstance(key_fallbacks, list): + lists.append(key_fallbacks) return tuple(lists) +_FallbackIndex: TypeAlias = tuple[Mapping[str, tuple[str, ...]], tuple[object, ...]] + + +def _index_fallback_list(fallback_list: Sequence[object]) -> _FallbackIndex: + """Index one fallback list once: an exact ``{source_key: targets}`` map for + bare keys, plus a capped tuple of entries only the router matcher can score + ("*" keys, provider-prefixed keys, bare-string generic targets).""" + exact: dict[str, tuple[str, ...]] = {} # mutable-ok: one entry per bare source key + fuzzy: list[object] = [] # mutable-ok: accumulates matcher-only entries + for entry in fallback_list: + if isinstance(entry, dict) and entry: + key = next(iter(entry)) + raw: Final = entry[key] + values: Final = raw if isinstance(raw, list) else ([raw] if raw is not None else []) + targets: Final = tuple(name for value in values if (name := _fallback_target_name(value)) is not None) + if "*" in key or "/" in key: + fuzzy.append(entry) + else: + exact[key] = targets + else: + fuzzy.append(entry) + return exact, tuple(fuzzy[:_FALLBACK_DISCOVERY_LIMIT]) + + def _fallback_edges( - fallback_lists: Sequence[Sequence[object]], + indexed_lists: Sequence[_FallbackIndex], model_group: str, ) -> frozenset[str]: - """Targets routing would pick for ``model_group`` out of each list, resolved - with the router's own matcher so exact, provider-prefixed, stripped and "*" - keys plus bare-string entries all count.""" + """Targets routing would pick for ``model_group``: exact-map hits on the group + and its provider-stripped suffix, plus the router matcher run over the small + fuzzy subset instead of every entry.""" from litellm.router_utils.fallback_event_handlers import get_fallback_model_group + stripped: Final = model_group.split("/", 1)[1] if "/" in model_group else None targets: Final[set[str]] = set() # mutable-ok: accumulates fallback targets - for fallback_list in fallback_lists: - resolved: Final = get_fallback_model_group(fallbacks=fallback_list, model_group=model_group)[0] + for exact, fuzzy in indexed_lists: + targets.update(exact.get(model_group, ())) + if stripped is not None: + targets.update(exact.get(stripped, ())) + resolved: Final = get_fallback_model_group(fallbacks=list(fuzzy), model_group=model_group)[0] values: Final = resolved if isinstance(resolved, list) else ([resolved] if resolved is not None else []) targets.update(name for value in values if (name := _fallback_target_name(value)) is not None) return frozenset(targets) @@ -2765,16 +2777,22 @@ def _fallback_target_groups( llm_router: litellm.Router, data: Mapping[str, object], router_settings: Mapping[str, object] | None, + team_router_settings: Mapping[str, object] | None, model_group: str, ) -> frozenset[str]: """Transitive closure over every fallback graph that can route this request: - a chain like A -> B -> C must still surface C's per-user credential.""" - lists: Final = _fallback_lists(data, llm_router, router_settings) + a chain like A -> B -> C must still surface C's per-user credential. Bounded + by _FALLBACK_DISCOVERY_LIMIT: discovery is advisory, and an undiscovered + target fails closed as the usual "not connected" 401.""" + indexed: Final = tuple( + _index_fallback_list(fallback_list) + for fallback_list in _fallback_lists(data, llm_router, router_settings, team_router_settings) + ) seen: Final[set[str]] = set() # mutable-ok: BFS visited set frontier: Final[list[str]] = [model_group] # mutable-ok: BFS work list - while frontier: + while frontier and len(seen) <= _FALLBACK_DISCOVERY_LIMIT: group = frontier.pop() - for target in _fallback_edges(lists, group): + for target in _fallback_edges(indexed, group): if target not in seen and target != model_group: seen.add(target) frontier.append(target) @@ -2823,6 +2841,30 @@ def _per_user_credential_names_for_groups( return tuple(names) +async def _team_router_settings(user_api_key_dict_team_id: str | None) -> Mapping[str, object] | None: + """The team's router_settings, which _configured_fallbacks prefers after the + key's (hierarchical Key > Team). Mirrors the cached get_team_object lookup the + proxy uses so no extra DB read lands on the hot path.""" + if not user_api_key_dict_team_id: + return None + try: + from litellm.proxy.auth.auth_checks import get_team_object + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache + + if prisma_client is None: + return None + team_obj: Final = await get_team_object( + team_id=user_api_key_dict_team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + settings: Final = getattr(team_obj, "router_settings", None) + return settings if isinstance(settings, Mapping) else None + except Exception: # noqa: BLE001 # advisory discovery must never break the request + return None + + async def _resolve_user_provider_credentials_for_request( data: dict[str, object], # mutable-ok: writes resolved credentials into the nested secret_fields dict authenticated_user_id: str | None, @@ -2842,7 +2884,34 @@ async def _resolve_user_provider_credentials_for_request( model: Final = data.get("model") if llm_router is None or not isinstance(model, str): return - model_groups: Final = frozenset({model} | _fallback_target_groups(llm_router, data, router_settings, model)) + from litellm.router_utils.common_utils import resolve_model_group_alias + + # Discovery must see the group the router actually routes to, so include the + # post-alias name from every alias map that may still rewrite data["model"] + # after this function (litellm.model_alias_map, router_settings.model_group_alias). + requested_groups: Final = frozenset( + target + for target in ( + model, + litellm.model_alias_map.get(model) if isinstance(litellm.model_alias_map, Mapping) else None, + ( + resolve_model_group_alias(router_settings.get("model_group_alias"), model) + if router_settings is not None + else None + ), + ) + if isinstance(target, str) and target + ) + team_router_settings: Final = await _team_router_settings(user_api_key_dict_team_id=team_id) + model_groups: Final = frozenset( + itertools.chain.from_iterable( + (requested_groups,) + + tuple( + _fallback_target_groups(llm_router, data, router_settings, team_router_settings, group) + for group in requested_groups + ) + ) + ) credential_names: Final = _per_user_credential_names_for_groups(llm_router, model_groups, team_id) if credential_names and "litellm_credential_name" in data: raise HTTPException( diff --git a/tests/unit/proxy/credential_endpoints/test_endpoints.py b/tests/unit/proxy/credential_endpoints/test_endpoints.py index 0f4f84c2f242..a268894e6209 100644 --- a/tests/unit/proxy/credential_endpoints/test_endpoints.py +++ b/tests/unit/proxy/credential_endpoints/test_endpoints.py @@ -2080,3 +2080,25 @@ def test_delete_user_connection_fails_closed_when_the_tombstone_write_fails(monk response = _call_as("DELETE", "/credentials/copilot-cred/user_connection", auth=_as_user()) assert response.status_code == 503, response.text table.delete_many.assert_not_awaited() + + +def test_delete_user_connection_fails_when_the_tombstone_write_silently_noops(monkeypatch): + """RedisCache.async_set_cache swallows client errors, so a set() that returns + without writing still looks successful. The disconnect must verify the + tombstone by reading the key back and refuse to delete the row.""" + table = _patch_user_connection_env(monkeypatch, [_per_user_credential()], rows=[_connection_row()]) + from types import SimpleNamespace + + from litellm.proxy import proxy_server + + store: dict = {} + silent_redis = SimpleNamespace( + async_set_cache=AsyncMock(return_value=None), # reports success, writes nothing + async_get_cache=AsyncMock(side_effect=lambda key: store.get(key)), + async_delete_cache=AsyncMock(return_value=None), + ) + monkeypatch.setattr(proxy_server, "user_api_key_cache", DualCache(redis_cache=silent_redis)) + + response = _call_as("DELETE", "/credentials/copilot-cred/user_connection", auth=_as_user()) + assert response.status_code == 503, response.text + table.delete_many.assert_not_awaited() diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index db21d7c9dec8..3681ff56e62d 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -9229,6 +9229,128 @@ async def test_key_router_settings_fallback_loads_per_user_tokens(self, monkeypa table.find_many.assert_awaited() assert data["secret_fields"]["user_provider_credentials_user_id"] == "user-a" + @pytest.mark.asyncio + async def test_fallback_discovery_bounds_matcher_calls_on_huge_lists(self, monkeypatch): + """30k unmatched fallback entries must not mean 30k matcher calls: the + index runs the router matcher only over a capped fuzzy subset.""" + import litellm.router_utils.fallback_event_handlers as feh + from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request + + self._env(monkeypatch) + entries_seen: list[int] = [] + real_matcher = feh.get_fallback_model_group + + def _spy(fallbacks, model_group): + entries_seen.append(len(fallbacks)) + return real_matcher(fallbacks=fallbacks, model_group=model_group) + + monkeypatch.setattr(feh, "get_fallback_model_group", _spy) + router = MagicMock() + router.fallbacks = [] + router.context_window_fallbacks = [] + router.content_policy_fallbacks = [] + router.get_model_list = MagicMock(return_value=[]) + router.get_deployment = MagicMock(return_value=None) + data = { + "model": "gpt-4o", + "fallbacks": [{"unmatched-%d" % i: ["x"]} for i in range(30_000)], + "secret_fields": {}, + } + await _resolve_user_provider_credentials_for_request( + data=data, + authenticated_user_id="user-a", + team_id=None, + llm_router=router, + ) + total = sum(entries_seen) + assert total <= 512, ( + f"matcher work must be bounded by the cap, got {total} entries across {len(entries_seen)} calls" + ) + + @pytest.mark.asyncio + async def test_model_alias_map_target_loads_per_user_tokens(self, monkeypatch): + """litellm.model_alias_map rewrites data['model'] after pre-call discovery: + the alias target's per-user credential must be discovered on the rewritten + name the router actually receives.""" + from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request + + table = self._env(monkeypatch) + monkeypatch.setattr(litellm, "model_alias_map", {"chat-alias": "copilot-chat"}) + router = MagicMock() + router.fallbacks = [] + router.context_window_fallbacks = [] + router.content_policy_fallbacks = [] + router.get_model_list = MagicMock( + side_effect=lambda model_name=None, team_id=None: ( + [ + { + "model_name": "copilot-chat", + "litellm_params": { + "model": "github_copilot/gpt-4o", + "litellm_credential_name": "copilot-cred", + }, + } + ] + if model_name == "copilot-chat" + else [] + ) + ) + router.get_deployment = MagicMock(return_value=None) + data = {"model": "chat-alias", "secret_fields": {}} + await _resolve_user_provider_credentials_for_request( + data=data, + authenticated_user_id="user-a", + team_id=None, + llm_router=router, + ) + table.find_many.assert_awaited() + assert data["secret_fields"]["user_provider_credentials_user_id"] == "user-a" + + @pytest.mark.asyncio + async def test_team_router_settings_fallback_loads_per_user_tokens(self, monkeypatch): + """Team router_settings fallbacks (hierarchical Key > Team source in + _get_hierarchical_router_settings) must be part of discovery.""" + from types import SimpleNamespace + + from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request + import litellm.proxy.auth.auth_checks as auth_checks + + table = self._env(monkeypatch) + monkeypatch.setattr( + auth_checks, + "get_team_object", + AsyncMock(return_value=SimpleNamespace(router_settings={"fallbacks": [{"gpt-4o": ["copilot-chat"]}]})), + ) + router = MagicMock() + router.fallbacks = [] + router.context_window_fallbacks = [] + router.content_policy_fallbacks = [] + router.get_model_list = MagicMock( + side_effect=lambda model_name=None, team_id=None: ( + [ + { + "model_name": "copilot-chat", + "litellm_params": { + "model": "github_copilot/gpt-4o", + "litellm_credential_name": "copilot-cred", + }, + } + ] + if model_name == "copilot-chat" + else [] + ) + ) + router.get_deployment = MagicMock(return_value=None) + data = {"model": "gpt-4o", "secret_fields": {}} + await _resolve_user_provider_credentials_for_request( + data=data, + authenticated_user_id="user-a", + team_id="team-1", + llm_router=router, + ) + table.find_many.assert_awaited() + assert data["secret_fields"]["user_provider_credentials_user_id"] == "user-a" + @pytest.mark.asyncio async def test_team_public_model_name_resolves_per_user_deployments(self, monkeypatch): """A caller hitting a team's public model name must get the same per-user From 318239982088d060ef9c5df62e35b3df3674c474 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 00:12:05 +0000 Subject: [PATCH 21/38] fix(github_copilot): make fallback discovery a router superset and guard credential override at deployment --- litellm/proxy/litellm_pre_call_utils.py | 27 +++- litellm/router.py | 23 +++ .../llms/github_copilot/test_per_user_auth.py | 49 ++++++ .../unit/proxy/test_litellm_pre_call_utils.py | 140 +++++++++++++++++- 4 files changed, 225 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index e6a9c49ac7ff..bf0116e68207 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -2722,7 +2722,8 @@ def _fallback_lists( def _index_fallback_list(fallback_list: Sequence[object]) -> _FallbackIndex: """Index one fallback list once: an exact ``{source_key: targets}`` map for - bare keys, plus a capped tuple of entries only the router matcher can score + bare keys (duplicate keys union their targets, a superset of the router's + first-wins resolution), plus the entries only the router matcher can score ("*" keys, provider-prefixed keys, bare-string generic targets).""" exact: dict[str, tuple[str, ...]] = {} # mutable-ok: one entry per bare source key fuzzy: list[object] = [] # mutable-ok: accumulates matcher-only entries @@ -2735,10 +2736,10 @@ def _index_fallback_list(fallback_list: Sequence[object]) -> _FallbackIndex: if "*" in key or "/" in key: fuzzy.append(entry) else: - exact[key] = targets + exact[key] = exact.get(key, ()) + targets else: fuzzy.append(entry) - return exact, tuple(fuzzy[:_FALLBACK_DISCOVERY_LIMIT]) + return exact, tuple(fuzzy) def _fallback_edges( @@ -2781,16 +2782,17 @@ def _fallback_target_groups( model_group: str, ) -> frozenset[str]: """Transitive closure over every fallback graph that can route this request: - a chain like A -> B -> C must still surface C's per-user credential. Bounded - by _FALLBACK_DISCOVERY_LIMIT: discovery is advisory, and an undiscovered - target fails closed as the usual "not connected" 401.""" + a chain like A -> B -> C must still surface C's per-user credential. Every + entry the router could match is indexed or matcher-scored, so discovery is + a faithful superset of what routing can reach; the only bound is the 400 on + caller-supplied fallback lists in the resolver.""" indexed: Final = tuple( _index_fallback_list(fallback_list) for fallback_list in _fallback_lists(data, llm_router, router_settings, team_router_settings) ) seen: Final[set[str]] = set() # mutable-ok: BFS visited set frontier: Final[list[str]] = [model_group] # mutable-ok: BFS work list - while frontier and len(seen) <= _FALLBACK_DISCOVERY_LIMIT: + while frontier: group = frontier.pop() for target in _fallback_edges(indexed, group): if target not in seen and target != model_group: @@ -2884,6 +2886,17 @@ async def _resolve_user_provider_credentials_for_request( model: Final = data.get("model") if llm_router is None or not isinstance(model, str): return + # The one caller-controlled input to discovery is the request body's own + # fallback lists, so the size bound lives here instead of on the scan: admin + # lists (router, key/team router_settings) are always read in full. + request_fallback_entries: Final = sum( + len(entries) for key in _REQUEST_FALLBACK_KEYS if isinstance(entries := data.get(key), list) + ) + if request_fallback_entries > _FALLBACK_DISCOVERY_LIMIT: + raise HTTPException( + status_code=400, + detail=(f"fallback lists in the request body cannot exceed {_FALLBACK_DISCOVERY_LIMIT} entries"), + ) from litellm.router_utils.common_utils import resolve_model_group_alias # Discovery must see the group the router actually routes to, so include the diff --git a/litellm/router.py b/litellm/router.py index bc0cf665f701..dbc4fd439a7a 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -4206,6 +4206,29 @@ def _update_kwargs_with_deployment( existing_tags.append(credential_tag) kwargs[metadata_variable_name]["tags"] = existing_tags + # A caller-supplied litellm_credential_name rides in kwargs and wins the + # ``{**litellm_params, **kwargs}`` merge downstream, so a per-user + # deployment's credential can be swapped for a shared one even when + # pre-call discovery missed this fallback hop. The deployment's own + # litellm_params still carry the configured name and auth type, so the + # check is exact: same name or no caller override is unaffected. + from litellm.llms.github_copilot.per_user_auth import ( + github_copilot_per_user_credential_name, + ) + + configured_per_user_name: Final = github_copilot_per_user_credential_name(deployment.get("litellm_params")) + caller_credential_name: Final = kwargs.get("litellm_credential_name") + if ( + configured_per_user_name is not None + and isinstance(caller_credential_name, str) + and caller_credential_name != configured_per_user_name + ): + raise litellm.BadRequestError( + message="litellm_credential_name cannot be overridden on a deployment that uses per-user GitHub OAuth", + model=deployment_model_name, + llm_provider="", + ) + kwargs["model_info"] = model_info if function_name == "_ageneric_api_call_with_fallbacks": diff --git a/tests/unit/llms/github_copilot/test_per_user_auth.py b/tests/unit/llms/github_copilot/test_per_user_auth.py index b4f8308e8a05..1c11bbb71101 100644 --- a/tests/unit/llms/github_copilot/test_per_user_auth.py +++ b/tests/unit/llms/github_copilot/test_per_user_auth.py @@ -1033,3 +1033,52 @@ async def test_shared_streaming_call_still_writes_response_cache(tmp_path, monke handler.should_store_result_in_cache(original_function=litellm.acompletion, kwargs=handler.request_kwargs) is True ) + + +def _per_user_router(): + return litellm.Router( + model_list=[ + { + "model_name": "copilot", + "litellm_params": { + "model": "github_copilot/gpt-4.1", + "litellm_credential_name": "copilot-cred", + }, + } + ] + ) + + +def test_per_user_deployment_rejects_a_caller_credential_override(per_user_credential): + """Caller kwargs win the litellm_params merge downstream, so the deployment + layer is the last place that still sees both names: a different name on a + per-user deployment must be refused there even if discovery missed the hop.""" + router = _per_user_router() + deployment: Final = router.get_model_list()[0] + kwargs: Final = {"litellm_credential_name": "shared-cred", "metadata": {}} + with pytest.raises(litellm.BadRequestError): + router._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs) + + +def test_per_user_deployment_allows_the_configured_credential_name(per_user_credential): + router = _per_user_router() + deployment: Final = router.get_model_list()[0] + kwargs: Final = {"litellm_credential_name": "copilot-cred", "metadata": {}} + router._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs) + assert kwargs["litellm_credential_name"] == "copilot-cred" + assert "model_info" in kwargs + + +def test_shared_deployment_keeps_caller_credential_override(): + from litellm.llms.github_copilot.authenticator import Authenticator + + with ( + patch.object(litellm, "credential_list", [_credential(auth_type="shared")]), + patch.object(Authenticator, "get_api_key", return_value="shared-token"), + ): + router = _per_user_router() + deployment: Final = router.get_model_list()[0] + kwargs: Final = {"litellm_credential_name": "other-cred", "metadata": {}} + router._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs) + assert kwargs["litellm_credential_name"] == "other-cred" + assert "model_info" in kwargs diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 3681ff56e62d..a0fb8c9ccddd 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -9231,8 +9231,9 @@ async def test_key_router_settings_fallback_loads_per_user_tokens(self, monkeypa @pytest.mark.asyncio async def test_fallback_discovery_bounds_matcher_calls_on_huge_lists(self, monkeypatch): - """30k unmatched fallback entries must not mean 30k matcher calls: the - index runs the router matcher only over a capped fuzzy subset.""" + """30k admin-configured unmatched entries go through the exact index: the + router matcher only ever sees the small wildcard subset, once per group + per list, instead of being scored against every entry.""" import litellm.router_utils.fallback_event_handlers as feh from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request @@ -9246,6 +9247,134 @@ def _spy(fallbacks, model_group): monkeypatch.setattr(feh, "get_fallback_model_group", _spy) router = MagicMock() + router.fallbacks = [{"unmatched-%d" % i: ["x"]} for i in range(30_000)] + [{"*": ["generic-x"]}] + router.context_window_fallbacks = [] + router.content_policy_fallbacks = [] + router.get_model_list = MagicMock(return_value=[]) + router.get_deployment = MagicMock(return_value=None) + data = {"model": "gpt-4o", "secret_fields": {}} + await _resolve_user_provider_credentials_for_request( + data=data, + authenticated_user_id="user-a", + team_id=None, + llm_router=router, + ) + total = sum(entries_seen) + assert total <= 8, ( + f"matcher must only see the 1 wildcard entry, got {total} entries across {len(entries_seen)} calls" + ) + + @pytest.mark.asyncio + async def test_duplicate_fallback_source_keys_union_targets(self, monkeypatch): + """Two entries with the same source key: the router picks the first, so + the per-user target on that first entry must be discovered even though + a later duplicate points elsewhere.""" + from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request + + table = self._env(monkeypatch) + router = MagicMock() + router.fallbacks = [{"gpt-4o": ["copilot-chat"]}, {"gpt-4o": ["other-chat"]}] + router.context_window_fallbacks = [] + router.content_policy_fallbacks = [] + router.get_model_list = MagicMock( + side_effect=lambda model_name=None, team_id=None: ( + [ + { + "model_name": "copilot-chat", + "litellm_params": { + "model": "github_copilot/gpt-4o", + "litellm_credential_name": "copilot-cred", + }, + } + ] + if model_name == "copilot-chat" + else [] + ) + ) + router.get_deployment = MagicMock(return_value=None) + data = {"model": "gpt-4o", "secret_fields": {}} + await _resolve_user_provider_credentials_for_request( + data=data, + authenticated_user_id="user-a", + team_id=None, + llm_router=router, + ) + table.find_many.assert_awaited() + assert data["secret_fields"]["user_provider_credentials_user_id"] == "user-a" + + @pytest.mark.asyncio + async def test_fallback_discovery_scans_past_the_wildcard_cap_position(self, monkeypatch): + """A wildcard match at position 290 of an admin list is still discovered: + admin lists are scanned in full, only the request body's lists are + capped.""" + from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request + + table = self._env(monkeypatch) + router = MagicMock() + entries = [{"azure/unmatched-%d" % i: ["x"]} for i in range(300)] + entries.insert(290, {"*": ["copilot-chat"]}) + router.fallbacks = entries + router.context_window_fallbacks = [] + router.content_policy_fallbacks = [] + router.get_model_list = MagicMock( + side_effect=lambda model_name=None, team_id=None: ( + [ + { + "model_name": "copilot-chat", + "litellm_params": { + "model": "github_copilot/gpt-4o", + "litellm_credential_name": "copilot-cred", + }, + } + ] + if model_name == "copilot-chat" + else [] + ) + ) + router.get_deployment = MagicMock(return_value=None) + data = {"model": "gpt-4o", "secret_fields": {}} + await _resolve_user_provider_credentials_for_request( + data=data, + authenticated_user_id="user-a", + team_id=None, + llm_router=router, + ) + table.find_many.assert_awaited() + assert data["secret_fields"]["user_provider_credentials_user_id"] == "user-a" + + @pytest.mark.asyncio + async def test_request_fallback_entries_over_the_limit_get_a_400(self, monkeypatch): + """The request body's fallback lists are the caller-controlled input to + discovery, so past 256 entries the request is refused instead of scanned.""" + from fastapi import HTTPException + + from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request + + self._env(monkeypatch) + router = MagicMock() + router.fallbacks = [] + router.context_window_fallbacks = [] + router.content_policy_fallbacks = [] + data = { + "model": "gpt-4o", + "fallbacks": [{"unmatched-%d" % i: ["x"]} for i in range(257)], + "secret_fields": {}, + } + with pytest.raises(HTTPException) as exc: + await _resolve_user_provider_credentials_for_request( + data=data, + authenticated_user_id="user-a", + team_id=None, + llm_router=router, + ) + assert getattr(exc.value, "status_code", None) == 400 + + @pytest.mark.asyncio + async def test_request_fallback_entries_at_the_limit_are_accepted(self, monkeypatch): + from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request + + self._env(monkeypatch) + router = MagicMock() router.fallbacks = [] router.context_window_fallbacks = [] router.content_policy_fallbacks = [] @@ -9253,7 +9382,7 @@ def _spy(fallbacks, model_group): router.get_deployment = MagicMock(return_value=None) data = { "model": "gpt-4o", - "fallbacks": [{"unmatched-%d" % i: ["x"]} for i in range(30_000)], + "fallbacks": [{"unmatched-%d" % i: ["x"]} for i in range(256)], "secret_fields": {}, } await _resolve_user_provider_credentials_for_request( @@ -9262,10 +9391,7 @@ def _spy(fallbacks, model_group): team_id=None, llm_router=router, ) - total = sum(entries_seen) - assert total <= 512, ( - f"matcher work must be bounded by the cap, got {total} entries across {len(entries_seen)} calls" - ) + assert data["secret_fields"] == {} @pytest.mark.asyncio async def test_model_alias_map_target_loads_per_user_tokens(self, monkeypatch): From c12bfabd4304b7f8bde5ec71b419d078670d4368 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 00:13:25 +0000 Subject: [PATCH 22/38] style(router): drop explanatory comment on per-user credential guard --- litellm/router.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index dbc4fd439a7a..86dc683861d6 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -4206,12 +4206,6 @@ def _update_kwargs_with_deployment( existing_tags.append(credential_tag) kwargs[metadata_variable_name]["tags"] = existing_tags - # A caller-supplied litellm_credential_name rides in kwargs and wins the - # ``{**litellm_params, **kwargs}`` merge downstream, so a per-user - # deployment's credential can be swapped for a shared one even when - # pre-call discovery missed this fallback hop. The deployment's own - # litellm_params still carry the configured name and auth type, so the - # check is exact: same name or no caller override is unaffected. from litellm.llms.github_copilot.per_user_auth import ( github_copilot_per_user_credential_name, ) From ab92ca5d524819ff8308cea3cecbb2ec7a2e943b Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 00:33:22 +0000 Subject: [PATCH 23/38] fix(github_copilot): gate fallback discovery on per-user credentials and bound aggregate targets --- litellm/proxy/litellm_pre_call_utils.py | 80 +++++++++-- .../llms/github_copilot/test_per_user_auth.py | 6 +- .../unit/proxy/test_litellm_pre_call_utils.py | 127 +++++++++++++++--- 3 files changed, 181 insertions(+), 32 deletions(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index bf0116e68207..486f144d77a4 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -2687,6 +2687,33 @@ async def add_litellm_data_to_request( _FALLBACK_DISCOVERY_LIMIT: Final = 256 +def _per_user_oauth_configured() -> bool: + """Whether any configured credential opted into per-user GitHub OAuth. + + Cheap in-memory precondition for all of the fallback discovery below: with + no per-user credential there is nothing to discover, so every code path in + this section (traversal, request-list size limit) must stay byte-identical + to shared-mode behavior.""" + from litellm.constants import ( + GITHUB_COPILOT_AUTH_TYPE_KEY, + GITHUB_COPILOT_PER_USER_AUTH_TYPE, + ) + + return any( + isinstance(values := getattr(credential, "credential_values", None), Mapping) + and values.get(GITHUB_COPILOT_AUTH_TYPE_KEY) == GITHUB_COPILOT_PER_USER_AUTH_TYPE + for credential in (litellm.credential_list or ()) + ) + + +def _fallback_entry_target_count(entry: object) -> int: + """Nested target names one fallback entry carries: a bare string counts 1, + a dict entry the sum of its value-list lengths.""" + if isinstance(entry, dict): + return sum(len(value) for value in entry.values() if isinstance(value, list)) + return 1 + + def _fallback_lists( data: Mapping[str, object], llm_router: litellm.Router, @@ -2742,24 +2769,44 @@ def _index_fallback_list(fallback_list: Sequence[object]) -> _FallbackIndex: return exact, tuple(fuzzy) +def _fuzzy_entry_matched(entry: object, resolved: object) -> bool: + """Whether a fuzzy entry is the one the router matcher returned for a group, + so its targets never get expanded a second time.""" + if isinstance(entry, dict) and entry: + value: Final = entry[next(iter(entry))] + return value is resolved or value == resolved + if isinstance(entry, str): + return resolved == [entry] + return False + + def _fallback_edges( indexed_lists: Sequence[_FallbackIndex], model_group: str, + fired_fuzzy: set[tuple[int, int]], # mutable-ok: records which fuzzy entries already expanded ) -> frozenset[str]: """Targets routing would pick for ``model_group``: exact-map hits on the group - and its provider-stripped suffix, plus the router matcher run over the small - fuzzy subset instead of every entry.""" + and its provider-stripped suffix, plus the router matcher run over the fuzzy + entries that have not fired yet (tracked in ``fired_fuzzy`` by + ``(list_index, entry_index)``), so each entry's targets are expanded once.""" from litellm.router_utils.fallback_event_handlers import get_fallback_model_group stripped: Final = model_group.split("/", 1)[1] if "/" in model_group else None targets: Final[set[str]] = set() # mutable-ok: accumulates fallback targets - for exact, fuzzy in indexed_lists: + for list_index, (exact, fuzzy) in enumerate(indexed_lists): targets.update(exact.get(model_group, ())) if stripped is not None: targets.update(exact.get(stripped, ())) - resolved: Final = get_fallback_model_group(fallbacks=list(fuzzy), model_group=model_group)[0] + pending: Final = tuple(index for index in range(len(fuzzy)) if (list_index, index) not in fired_fuzzy) + if not pending: + continue + resolved: Final = get_fallback_model_group( + fallbacks=[fuzzy[index] for index in pending], model_group=model_group + )[0] values: Final = resolved if isinstance(resolved, list) else ([resolved] if resolved is not None else []) targets.update(name for value in values if (name := _fallback_target_name(value)) is not None) + if resolved is not None: + fired_fuzzy.update((list_index, index) for index in pending if _fuzzy_entry_matched(fuzzy[index], resolved)) return frozenset(targets) @@ -2785,16 +2832,19 @@ def _fallback_target_groups( a chain like A -> B -> C must still surface C's per-user credential. Every entry the router could match is indexed or matcher-scored, so discovery is a faithful superset of what routing can reach; the only bound is the 400 on - caller-supplied fallback lists in the resolver.""" + caller-supplied fallback lists in the resolver. A fuzzy entry whose targets + already expanded once is marked fired in ``fired_fuzzy`` and never matched + or expanded again, so total target visits stay O(total targets).""" indexed: Final = tuple( _index_fallback_list(fallback_list) for fallback_list in _fallback_lists(data, llm_router, router_settings, team_router_settings) ) seen: Final[set[str]] = set() # mutable-ok: BFS visited set frontier: Final[list[str]] = [model_group] # mutable-ok: BFS work list + fired_fuzzy: Final[set[tuple[int, int]]] = set() # mutable-ok: fired (list, entry) indexes while frontier: group = frontier.pop() - for target in _fallback_edges(indexed, group): + for target in _fallback_edges(indexed, group, fired_fuzzy): if target not in seen and target != model_group: seen.add(target) frontier.append(target) @@ -2886,13 +2936,21 @@ async def _resolve_user_provider_credentials_for_request( model: Final = data.get("model") if llm_router is None or not isinstance(model, str): return + # With no per-user credential configured there is nothing to discover: no + # traversal, no size check, identical behavior to a shared-mode-only proxy. + if not _per_user_oauth_configured(): + return # The one caller-controlled input to discovery is the request body's own # fallback lists, so the size bound lives here instead of on the scan: admin - # lists (router, key/team router_settings) are always read in full. - request_fallback_entries: Final = sum( - len(entries) for key in _REQUEST_FALLBACK_KEYS if isinstance(entries := data.get(key), list) - ) - if request_fallback_entries > _FALLBACK_DISCOVERY_LIMIT: + # lists (router, key/team router_settings) are always read in full. The + # bound is on aggregate target names, not outer entries, so a single dict + # holding thousands of targets is counted honestly. + request_fallback_targets: Final = sum( + sum(_fallback_entry_target_count(entry) for entry in entries) + for key in _REQUEST_FALLBACK_KEYS + if isinstance(entries := data.get(key), list) + ) + if request_fallback_targets > _FALLBACK_DISCOVERY_LIMIT: raise HTTPException( status_code=400, detail=(f"fallback lists in the request body cannot exceed {_FALLBACK_DISCOVERY_LIMIT} entries"), diff --git a/tests/unit/llms/github_copilot/test_per_user_auth.py b/tests/unit/llms/github_copilot/test_per_user_auth.py index 1c11bbb71101..9b16d3c0aa79 100644 --- a/tests/unit/llms/github_copilot/test_per_user_auth.py +++ b/tests/unit/llms/github_copilot/test_per_user_auth.py @@ -1053,7 +1053,7 @@ def test_per_user_deployment_rejects_a_caller_credential_override(per_user_crede """Caller kwargs win the litellm_params merge downstream, so the deployment layer is the last place that still sees both names: a different name on a per-user deployment must be refused there even if discovery missed the hop.""" - router = _per_user_router() + router: Final = _per_user_router() deployment: Final = router.get_model_list()[0] kwargs: Final = {"litellm_credential_name": "shared-cred", "metadata": {}} with pytest.raises(litellm.BadRequestError): @@ -1061,7 +1061,7 @@ def test_per_user_deployment_rejects_a_caller_credential_override(per_user_crede def test_per_user_deployment_allows_the_configured_credential_name(per_user_credential): - router = _per_user_router() + router: Final = _per_user_router() deployment: Final = router.get_model_list()[0] kwargs: Final = {"litellm_credential_name": "copilot-cred", "metadata": {}} router._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs) @@ -1076,7 +1076,7 @@ def test_shared_deployment_keeps_caller_credential_override(): patch.object(litellm, "credential_list", [_credential(auth_type="shared")]), patch.object(Authenticator, "get_api_key", return_value="shared-token"), ): - router = _per_user_router() + router: Final = _per_user_router() deployment: Final = router.get_model_list()[0] kwargs: Final = {"litellm_credential_name": "other-cred", "metadata": {}} router._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs) diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index a0fb8c9ccddd..99f5deffeb9d 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -9238,28 +9238,28 @@ async def test_fallback_discovery_bounds_matcher_calls_on_huge_lists(self, monke from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request self._env(monkeypatch) - entries_seen: list[int] = [] - real_matcher = feh.get_fallback_model_group + entries_seen: Final[list[int]] = [] + real_matcher: Final = feh.get_fallback_model_group def _spy(fallbacks, model_group): entries_seen.append(len(fallbacks)) return real_matcher(fallbacks=fallbacks, model_group=model_group) monkeypatch.setattr(feh, "get_fallback_model_group", _spy) - router = MagicMock() + router: Final = MagicMock() router.fallbacks = [{"unmatched-%d" % i: ["x"]} for i in range(30_000)] + [{"*": ["generic-x"]}] router.context_window_fallbacks = [] router.content_policy_fallbacks = [] router.get_model_list = MagicMock(return_value=[]) router.get_deployment = MagicMock(return_value=None) - data = {"model": "gpt-4o", "secret_fields": {}} + data: Final = {"model": "gpt-4o", "secret_fields": {}} await _resolve_user_provider_credentials_for_request( data=data, authenticated_user_id="user-a", team_id=None, llm_router=router, ) - total = sum(entries_seen) + total: Final = sum(entries_seen) assert total <= 8, ( f"matcher must only see the 1 wildcard entry, got {total} entries across {len(entries_seen)} calls" ) @@ -9271,8 +9271,8 @@ async def test_duplicate_fallback_source_keys_union_targets(self, monkeypatch): a later duplicate points elsewhere.""" from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request - table = self._env(monkeypatch) - router = MagicMock() + table: Final = self._env(monkeypatch) + router: Final = MagicMock() router.fallbacks = [{"gpt-4o": ["copilot-chat"]}, {"gpt-4o": ["other-chat"]}] router.context_window_fallbacks = [] router.content_policy_fallbacks = [] @@ -9292,7 +9292,7 @@ async def test_duplicate_fallback_source_keys_union_targets(self, monkeypatch): ) ) router.get_deployment = MagicMock(return_value=None) - data = {"model": "gpt-4o", "secret_fields": {}} + data: Final = {"model": "gpt-4o", "secret_fields": {}} await _resolve_user_provider_credentials_for_request( data=data, authenticated_user_id="user-a", @@ -9309,9 +9309,9 @@ async def test_fallback_discovery_scans_past_the_wildcard_cap_position(self, mon capped.""" from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request - table = self._env(monkeypatch) - router = MagicMock() - entries = [{"azure/unmatched-%d" % i: ["x"]} for i in range(300)] + table: Final = self._env(monkeypatch) + router: Final = MagicMock() + entries: Final = [{"azure/unmatched-%d" % i: ["x"]} for i in range(300)] entries.insert(290, {"*": ["copilot-chat"]}) router.fallbacks = entries router.context_window_fallbacks = [] @@ -9332,7 +9332,7 @@ async def test_fallback_discovery_scans_past_the_wildcard_cap_position(self, mon ) ) router.get_deployment = MagicMock(return_value=None) - data = {"model": "gpt-4o", "secret_fields": {}} + data: Final = {"model": "gpt-4o", "secret_fields": {}} await _resolve_user_provider_credentials_for_request( data=data, authenticated_user_id="user-a", @@ -9351,13 +9351,13 @@ async def test_request_fallback_entries_over_the_limit_get_a_400(self, monkeypat from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request self._env(monkeypatch) - router = MagicMock() + router: Final = MagicMock() router.fallbacks = [] router.context_window_fallbacks = [] router.content_policy_fallbacks = [] - data = { + data: Final = { "model": "gpt-4o", - "fallbacks": [{"unmatched-%d" % i: ["x"]} for i in range(257)], + "fallbacks": [{"gpt-4o": ["target-%d" % i for i in range(257)]}], "secret_fields": {}, } with pytest.raises(HTTPException) as exc: @@ -9374,15 +9374,56 @@ async def test_request_fallback_entries_at_the_limit_are_accepted(self, monkeypa from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request self._env(monkeypatch) - router = MagicMock() + router: Final = MagicMock() router.fallbacks = [] router.context_window_fallbacks = [] router.content_policy_fallbacks = [] router.get_model_list = MagicMock(return_value=[]) router.get_deployment = MagicMock(return_value=None) - data = { + data: Final = { + "model": "gpt-4o", + "fallbacks": [{"gpt-4o": ["target-%d" % i for i in range(256)]}], + "secret_fields": {}, + } + await _resolve_user_provider_credentials_for_request( + data=data, + authenticated_user_id="user-a", + team_id=None, + llm_router=router, + ) + assert data["secret_fields"] == {} + + @pytest.mark.asyncio + async def test_shared_only_credentials_skip_discovery_and_the_limit(self, monkeypatch): + """With no per-user credential configured the request's fallback lists + are never scanned and the size limit never fires: a proxy that never + opted into per-user OAuth keeps byte-identical behavior.""" + import litellm.router_utils.fallback_event_handlers as feh + from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request + + self._env(monkeypatch) + monkeypatch.setattr( + litellm, + "credential_list", + [CredentialItem(credential_name="shared-cred", credential_values={"api_key": "k"}, credential_info={})], + ) + matcher_calls: Final[list[object]] = [] + real_matcher: Final = feh.get_fallback_model_group + + def _spy(fallbacks, model_group): + matcher_calls.append(model_group) + return real_matcher(fallbacks=fallbacks, model_group=model_group) + + monkeypatch.setattr(feh, "get_fallback_model_group", _spy) + router: Final = MagicMock() + router.fallbacks = [{"gpt-4o": ["copilot-chat"]}] + router.context_window_fallbacks = [] + router.content_policy_fallbacks = [] + router.get_model_list = MagicMock(return_value=[]) + router.get_deployment = MagicMock(return_value=None) + data: Final = { "model": "gpt-4o", - "fallbacks": [{"unmatched-%d" % i: ["x"]} for i in range(256)], + "fallbacks": [{"unmatched-%d" % i: ["x"]} for i in range(1000)], "secret_fields": {}, } await _resolve_user_provider_credentials_for_request( @@ -9391,8 +9432,58 @@ async def test_request_fallback_entries_at_the_limit_are_accepted(self, monkeypa team_id=None, llm_router=router, ) + assert matcher_calls == [] + router.get_model_list.assert_not_called() assert data["secret_fields"] == {} + @pytest.mark.asyncio + async def test_wildcard_targets_expand_once_not_per_group(self, monkeypatch): + """An admin ``{"*": [5000 targets]}`` chain must expand the entry once: + without fired-entry tracking each discovered group would re-expand the + same 5000 targets.""" + from litellm.proxy import litellm_pre_call_utils as lpc + from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request + + self._env(monkeypatch) + target_visits: Final[list[int]] = [] + real_edges: Final = lpc._fallback_edges + + def _counting_edges(indexed_lists, model_group, fired_fuzzy): + result: Final = real_edges(indexed_lists, model_group, fired_fuzzy) + target_visits.append(len(result)) + return result + + monkeypatch.setattr(lpc, "_fallback_edges", _counting_edges) + router: Final = MagicMock() + router.fallbacks = [{"*": ["target-%d" % i for i in range(4999)] + ["copilot-chat"]}] + router.context_window_fallbacks = [] + router.content_policy_fallbacks = [] + router.get_model_list = MagicMock( + side_effect=lambda model_name=None, team_id=None: ( + [ + { + "model_name": "copilot-chat", + "litellm_params": { + "model": "github_copilot/gpt-4o", + "litellm_credential_name": "copilot-cred", + }, + } + ] + if model_name == "copilot-chat" + else [] + ) + ) + router.get_deployment = MagicMock(return_value=None) + data: Final = {"model": "gpt-4o", "secret_fields": {}} + await _resolve_user_provider_credentials_for_request( + data=data, + authenticated_user_id="user-a", + team_id=None, + llm_router=router, + ) + total: Final = sum(target_visits) + assert total <= 5100, f"targets must expand once, got {total} visits" + @pytest.mark.asyncio async def test_model_alias_map_target_loads_per_user_tokens(self, monkeypatch): """litellm.model_alias_map rewrites data['model'] after pre-call discovery: From f9af01cc69f691b4ca53bca3787e7ffded2522ca Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 00:57:37 +0000 Subject: [PATCH 24/38] fix(github_copilot): count every fallback mapping toward the discovery limit --- .../github_copilot/chat/transformation.py | 6 +-- .../embedding/transformation.py | 6 +-- litellm/llms/github_copilot/per_user_auth.py | 4 +- litellm/proxy/litellm_pre_call_utils.py | 32 +++++++-------- .../unit/proxy/test_litellm_pre_call_utils.py | 39 ++++++++++++------- 5 files changed, 49 insertions(+), 38 deletions(-) diff --git a/litellm/llms/github_copilot/chat/transformation.py b/litellm/llms/github_copilot/chat/transformation.py index fba975fc971f..2aae6a0b36ff 100644 --- a/litellm/llms/github_copilot/chat/transformation.py +++ b/litellm/llms/github_copilot/chat/transformation.py @@ -1,7 +1,7 @@ import json import os from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Final +from typing import TYPE_CHECKING, Final, cast import httpx @@ -87,9 +87,7 @@ def _transform_messages( for message in messages: if message.get("role") == "system": # Convert system message to assistant message - transformed_message = message.copy() - transformed_message["role"] = "assistant" - transformed_messages.append(transformed_message) + transformed_messages.append(cast(AllMessageValues, {**message, "role": "assistant"})) else: transformed_messages.append(message) diff --git a/litellm/llms/github_copilot/embedding/transformation.py b/litellm/llms/github_copilot/embedding/transformation.py index a8c4dc10e78f..225d32bda388 100644 --- a/litellm/llms/github_copilot/embedding/transformation.py +++ b/litellm/llms/github_copilot/embedding/transformation.py @@ -65,9 +65,9 @@ def validate_environment( """ user_session: Final = require_github_copilot_user_session(litellm_params) # pyright: ignore[reportUnknownArgumentType] # litellm_params arrives as an untyped request dict if user_session is not None: - merged_headers: Final = {**get_copilot_default_headers(user_session.token), **headers} - pin_session_authorization(merged_headers, user_session.token) - return merged_headers + session_headers: Final = {**get_copilot_default_headers(user_session.token), **headers} + pin_session_authorization(session_headers, user_session.token) + return session_headers try: # Get GitHub Copilot API key via OAuth api_key = self.authenticator.get_api_key() diff --git a/litellm/llms/github_copilot/per_user_auth.py b/litellm/llms/github_copilot/per_user_auth.py index 98d34e23052d..8f09c58f7db4 100644 --- a/litellm/llms/github_copilot/per_user_auth.py +++ b/litellm/llms/github_copilot/per_user_auth.py @@ -665,9 +665,9 @@ def _strip_caller_authorization( after the transformations run, so they are rebuilt here without any Authorization key; the caller's mappings are left untouched.""" for key in ("extra_headers", "headers"): - value: Final = kwargs.get(key) + value = kwargs.get(key) if isinstance(value, Mapping): - stripped: Final = _without_authorization(value) + stripped = _without_authorization(value) if len(stripped) != len(value): kwargs[key] = stripped # rebind-ok: kwargs is the request mutation channel optional_params: Final = kwargs.get("optional_params") diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 486f144d77a4..6ca5d9886c6b 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -2707,10 +2707,12 @@ def _per_user_oauth_configured() -> bool: def _fallback_entry_target_count(entry: object) -> int: - """Nested target names one fallback entry carries: a bare string counts 1, - a dict entry the sum of its value-list lengths.""" + """Work units one fallback entry carries, floored so nothing counts as zero: + a bare string or any non-dict counts 1; a dict entry counts one unit per + key plus one per target in its value lists, so empty lists and scalar + values still cost what the router scan costs.""" if isinstance(entry, dict): - return sum(len(value) for value in entry.values() if isinstance(value, list)) + return sum(max(1, len(value)) if isinstance(value, list) else 1 for value in entry.values()) return 1 @@ -2757,9 +2759,9 @@ def _index_fallback_list(fallback_list: Sequence[object]) -> _FallbackIndex: for entry in fallback_list: if isinstance(entry, dict) and entry: key = next(iter(entry)) - raw: Final = entry[key] - values: Final = raw if isinstance(raw, list) else ([raw] if raw is not None else []) - targets: Final = tuple(name for value in values if (name := _fallback_target_name(value)) is not None) + raw = entry[key] + values = raw if isinstance(raw, list) else ([raw] if raw is not None else []) + targets = tuple(name for value in values if (name := _fallback_target_name(value)) is not None) if "*" in key or "/" in key: fuzzy.append(entry) else: @@ -2797,13 +2799,11 @@ def _fallback_edges( targets.update(exact.get(model_group, ())) if stripped is not None: targets.update(exact.get(stripped, ())) - pending: Final = tuple(index for index in range(len(fuzzy)) if (list_index, index) not in fired_fuzzy) + pending = tuple(index for index in range(len(fuzzy)) if (list_index, index) not in fired_fuzzy) if not pending: continue - resolved: Final = get_fallback_model_group( - fallbacks=[fuzzy[index] for index in pending], model_group=model_group - )[0] - values: Final = resolved if isinstance(resolved, list) else ([resolved] if resolved is not None else []) + resolved = get_fallback_model_group(fallbacks=[fuzzy[index] for index in pending], model_group=model_group)[0] + values = resolved if isinstance(resolved, list) else ([resolved] if resolved is not None else []) targets.update(name for value in values if (name := _fallback_target_name(value)) is not None) if resolved is not None: fired_fuzzy.update((list_index, index) for index in pending if _fuzzy_entry_matched(fuzzy[index], resolved)) @@ -2863,9 +2863,9 @@ def _per_user_credential_names_for_groups( names: Final[list[str]] = [] # mutable-ok: accumulates one name per per-user deployment for group in model_groups: - by_name: Final = llm_router.get_model_list(model_name=group, team_id=team_id) or () - deployment_id_match: Final = llm_router.get_deployment(model_id=group) - deployments: Final[Sequence[object]] = ( + by_name = llm_router.get_model_list(model_name=group, team_id=team_id) or () + deployment_id_match = llm_router.get_deployment(model_id=group) + deployments: Sequence[object] = ( *by_name, *((deployment_id_match,) if deployment_id_match is not None else ()), ) @@ -2882,10 +2882,10 @@ def _per_user_credential_names_for_groups( ) if not isinstance(credential_name_obj, str) or not credential_name_obj or credential_name_obj in names: continue - credential: Final = CredentialAccessor.find_credential(credential_name_obj) + credential = CredentialAccessor.find_credential(credential_name_obj) if credential is None: continue - values: Final = cast( # cast-ok: credential_values is a plain dict at runtime + values = cast( # cast-ok: credential_values is a plain dict at runtime Mapping[object, object], credential.credential_values ) if values.get(GITHUB_COPILOT_AUTH_TYPE_KEY) == GITHUB_COPILOT_PER_USER_AUTH_TYPE: diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 99f5deffeb9d..aac26ac8e3da 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -9343,9 +9343,20 @@ async def test_fallback_discovery_scans_past_the_wildcard_cap_position(self, mon assert data["secret_fields"]["user_provider_credentials_user_id"] == "user-a" @pytest.mark.asyncio - async def test_request_fallback_entries_over_the_limit_get_a_400(self, monkeypatch): + @pytest.mark.parametrize( + "entries", + [ + [{"gpt-4o": ["target-%d" % i for i in range(257)]}], + [{"azure/unmatched-%d" % i: []} for i in range(257)], + [{("k%d" % i): [] for i in range(257)}], + [{"m%d" % i: "t"} for i in range(257)], + ], + ids=["nested targets", "empty-list mappings", "one dict of empty lists", "scalar mappings"], + ) + async def test_request_fallback_entries_over_the_limit_get_a_400(self, monkeypatch, entries): """The request body's fallback lists are the caller-controlled input to - discovery, so past 256 entries the request is refused instead of scanned.""" + discovery, so past 256 counted work units (a key counts even when its + value list is empty or a scalar) the request is refused instead of scanned.""" from fastapi import HTTPException from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request @@ -9355,11 +9366,7 @@ async def test_request_fallback_entries_over_the_limit_get_a_400(self, monkeypat router.fallbacks = [] router.context_window_fallbacks = [] router.content_policy_fallbacks = [] - data: Final = { - "model": "gpt-4o", - "fallbacks": [{"gpt-4o": ["target-%d" % i for i in range(257)]}], - "secret_fields": {}, - } + data: Final = {"model": "gpt-4o", "fallbacks": entries, "secret_fields": {}} with pytest.raises(HTTPException) as exc: await _resolve_user_provider_credentials_for_request( data=data, @@ -9370,7 +9377,17 @@ async def test_request_fallback_entries_over_the_limit_get_a_400(self, monkeypat assert getattr(exc.value, "status_code", None) == 400 @pytest.mark.asyncio - async def test_request_fallback_entries_at_the_limit_are_accepted(self, monkeypatch): + @pytest.mark.parametrize( + "entries", + [ + [{"gpt-4o": ["target-%d" % i for i in range(256)]}], + [{"azure/unmatched-%d" % i: []} for i in range(256)], + [{("k%d" % i): [] for i in range(256)}], + [{"m%d" % i: "t"} for i in range(256)], + ], + ids=["nested targets", "empty-list mappings", "one dict of empty lists", "scalar mappings"], + ) + async def test_request_fallback_entries_at_the_limit_are_accepted(self, monkeypatch, entries): from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request self._env(monkeypatch) @@ -9380,11 +9397,7 @@ async def test_request_fallback_entries_at_the_limit_are_accepted(self, monkeypa router.content_policy_fallbacks = [] router.get_model_list = MagicMock(return_value=[]) router.get_deployment = MagicMock(return_value=None) - data: Final = { - "model": "gpt-4o", - "fallbacks": [{"gpt-4o": ["target-%d" % i for i in range(256)]}], - "secret_fields": {}, - } + data: Final = {"model": "gpt-4o", "fallbacks": entries, "secret_fields": {}} await _resolve_user_provider_credentials_for_request( data=data, authenticated_user_id="user-a", From 4e0a12c217094ad24d9fa87c1a31a027ce59c44d Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 01:04:32 +0000 Subject: [PATCH 25/38] fix(github_copilot): suppress banned typing cast imports with per-line reasons --- litellm/llms/github_copilot/chat/transformation.py | 2 +- litellm/proxy/litellm_pre_call_utils.py | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/litellm/llms/github_copilot/chat/transformation.py b/litellm/llms/github_copilot/chat/transformation.py index 2aae6a0b36ff..5930c6c94049 100644 --- a/litellm/llms/github_copilot/chat/transformation.py +++ b/litellm/llms/github_copilot/chat/transformation.py @@ -1,7 +1,7 @@ import json import os from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Final, cast +from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # spread dict loses the role literal without a cast import httpx diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 6ca5d9886c6b..94d4bfe1a272 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -8,7 +8,14 @@ from collections.abc import Mapping, MutableMapping, Sequence from datetime import datetime from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast +from typing import ( + TYPE_CHECKING, + Any, + Final, + Literal, + TypeAlias, + cast, # noqa: TID251 # narrows untyped deployment and credential payloads +) from fastapi import HTTPException, Request from pydantic import TypeAdapter From c578aeb3be2ac61041b691a745c1f4ecd98b6a8f Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 01:40:36 +0000 Subject: [PATCH 26/38] test(github_copilot): cover per-user 401 cooldown skip through the router failure callback --- .../router_utils/test_cooldown_handlers.py | 168 +++++++++++++++--- 1 file changed, 142 insertions(+), 26 deletions(-) diff --git a/tests/unit/router_utils/test_cooldown_handlers.py b/tests/unit/router_utils/test_cooldown_handlers.py index 4632744572fe..a23696e52b2f 100644 --- a/tests/unit/router_utils/test_cooldown_handlers.py +++ b/tests/unit/router_utils/test_cooldown_handlers.py @@ -2,6 +2,7 @@ import importlib import random import time +from typing import Final from unittest.mock import MagicMock, patch import httpx @@ -576,6 +577,7 @@ def _vcr_outcome_gate(request, vcr): yield record_vcr_outcome(request, vcr) + @pytest.fixture(scope="function") def setup_and_teardown(): """ @@ -596,9 +598,11 @@ def setup_and_teardown(): loop.close() asyncio.set_event_loop(None) + def _make_router(model_list: list, **kwargs) -> Router: return Router(model_list=model_list, **kwargs) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestDeploymentLevelAllowedFails: def test_deployment_level_allowed_fails_overrides_router_level(self): @@ -671,6 +675,7 @@ def test_deployment_level_allowed_fails_does_not_affect_other_deployments(self): "secondary has no deployment-level policy; with allowed_fails=10 it should not cool down on first failure" ) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestDeploymentLevelAllowedFailsPolicyByExceptionType: def test_rate_limit_error_triggers_cooldown_with_zero_threshold(self): @@ -745,6 +750,7 @@ def test_internal_server_error_respects_per_exception_threshold(self): ) assert should_cooldown is True, "Should cooldown after exceeding InternalServerErrorAllowedFails=5" + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestExceptionTypeCountersTrackedIndependently: def test_cache_key_suffix_separates_exception_type_counters(self): @@ -795,6 +801,7 @@ def test_cache_key_suffix_separates_exception_type_counters(self): assert generic_counter_after == 1, "generic counter should now be 1" assert rl_counter_after == 3, "RateLimitError counter must remain unchanged after InternalServerError" + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestCooldownCacheTTLCorrection: def _make_cooldown_cache(self) -> CooldownCache: @@ -896,6 +903,7 @@ async def test_async_expired_entry_evicted(self): assert active == [], "Expired entry must not appear in async active cooldowns" assert cc.in_memory_cache.get_cache(key) is None + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestFallbackDeploymentCooldown: def test_trigger_cooldown_for_failed_deployment_calls_set_cooldown(self): @@ -1086,6 +1094,7 @@ def test_trigger_cooldown_prefers_model_info_cooldown_time_over_litellm_params(s call_kwargs = mock_set_cooldown.call_args[1] assert call_kwargs["time_to_cooldown"] == 15.0, "model_info.cooldown_time must take priority" + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestSingleDeploymentModelGroupProtection: def test_generic_allowed_fails_does_not_bypass_single_deployment_protection(self): @@ -1145,6 +1154,7 @@ def test_named_exception_policy_still_overrides_single_deployment_protection(sel ) assert should_cooldown is True, "explicit per-exception-type policy must still cool down a solo deployment" + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestShouldCooldownBasedOnAllowedFailsPolicyFalsyZero: def test_router_level_policy_of_zero_is_not_swallowed_by_allowed_fails(self): @@ -1173,6 +1183,7 @@ def test_router_level_policy_of_zero_is_not_swallowed_by_allowed_fails(self): ) assert should_cooldown is True, "RateLimitErrorAllowedFails=0 must cool down after the first failure" + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestResolveAllowedFailsFromPolicyFallsThrough: def test_none_value_on_first_match_falls_through_to_next_type(self): @@ -1190,6 +1201,7 @@ def test_none_value_on_first_match_falls_through_to_next_type(self): result = _resolve_allowed_fails_from_policy(policy=policy, exception=exc) assert result == 3, "must fall through to BadRequestErrorAllowedFails when the more specific field is unset" + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestDeploymentCallbackOnFailureCooldownTimePrecedence: def test_model_info_cooldown_time_used_in_primary_sync_path(self): @@ -1282,11 +1294,7 @@ def _router( }, "model_info": { "id": model_id, - **( - {"allowed_fails_policy": allowed_fails_policy} - if allowed_fails_policy is not None - else {} - ), + **({"allowed_fails_policy": allowed_fails_policy} if allowed_fails_policy is not None else {}), }, } ] @@ -1409,6 +1417,85 @@ async def test_oauth_deployment_still_cools_down_on_provider_failures(self, stat assert get_deployment_failures_for_current_minute(router, model_id) == 1 assert get_cooldown_deployments(router, parent_otel_span=None) == [model_id] + +@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") +class TestPerUserCopilotAuthFailureCooldown: + @staticmethod + def _router(model_id: str, litellm_params: dict[str, object]) -> Router: + return _make_router( + model_list=[ + { + "model_name": "copilot", + "litellm_params": { + "model": "github_copilot/gpt-4o", + **litellm_params, + }, + "model_info": {"id": model_id}, + } + ] + ) + + @staticmethod + def _callback(router: Router, model_id: str, exception: Exception) -> bool: + return router.deployment_callback_on_failure( + kwargs={ + "exception": exception, + "litellm_params": {"model_info": {"id": model_id}}, + }, + completion_response=None, + start_time=0, + end_time=1, + ) + + @pytest.mark.asyncio + async def test_per_user_oauth_caller_auth_failure_does_not_cooldown( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + from litellm.constants import GITHUB_COPILOT_AUTH_TYPE_KEY, GITHUB_COPILOT_PER_USER_AUTH_TYPE + + model_id: Final = "per-user-oauth" + monkeypatch.setattr( + litellm, + "credential_list", + [ + CredentialItem( + credential_name="copilot-per-user", + credential_values={GITHUB_COPILOT_AUTH_TYPE_KEY: GITHUB_COPILOT_PER_USER_AUTH_TYPE}, + credential_info={"custom_llm_provider": "github_copilot"}, + ) + ], + ) + router: Final = self._router(model_id, {"litellm_credential_name": "copilot-per-user"}) + exception: Final = litellm.CallerCredentialAuthenticationError( + message="reconnect", llm_provider="github_copilot", model="" + ) + + result: Final = self._callback(router, model_id, exception) + + assert result is False + assert get_cooldown_deployments(router, parent_otel_span=None) == [] + + @pytest.mark.asyncio + async def test_shared_mode_copilot_auth_failure_still_cools_down(self) -> None: + from litellm.llms.github_copilot.authenticator import Authenticator + + model_id: Final = "shared-mode" + with ( + patch.object(Authenticator, "get_api_key", return_value="shared-copilot-token"), + patch.object(Authenticator, "get_api_base", return_value="https://api.githubcopilot.com"), + ): + router: Final = self._router(model_id, {"api_key": "sk-shared"}) + self._callback( + router, + model_id, + litellm.AuthenticationError("upstream rejected the shared token", "github_copilot", "gpt-4o"), + ) + + assert get_deployment_failures_for_current_minute(router, model_id) == 1 + assert get_cooldown_deployments(router, parent_otel_span=None) == [model_id] + + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestNewAllowedFailsPolicyFields: def test_service_unavailable_error_matched_by_policy(self): @@ -1460,6 +1547,7 @@ def test_allowed_fails_policy_model_accepts_new_fields(self): assert policy.BadGatewayErrorAllowedFails == 2 assert policy.NotFoundErrorAllowedFails == 1 + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestRouterLevelGetAllowedFailsFromPolicy: """Router.get_allowed_fails_from_policy must handle all AllowedFailsPolicy fields.""" @@ -1495,6 +1583,7 @@ def test_unmatched_exception_returns_none(self): exc = litellm.RateLimitError("429", "openai", "gpt-4") assert router.get_allowed_fails_from_policy(exc) is None + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_router_cooldown_event_callback_no_deployment(): @@ -1519,6 +1608,7 @@ async def test_router_cooldown_event_callback_no_deployment(): # Assert that the router's get_deployment method was called mock_router.get_deployment.assert_called_once_with(model_id="test-deployment") + @pytest.fixture def testing_litellm_router(): return Router( @@ -1541,6 +1631,7 @@ def testing_litellm_router(): ] ) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_should_run_cooldown_logic(testing_litellm_router): testing_litellm_router.disable_cooldowns = True @@ -1555,6 +1646,7 @@ def test_should_run_cooldown_logic(testing_litellm_router): testing_litellm_router.provider_default_deployment_ids = ["test_deployment"] assert _should_run_cooldown_logic(testing_litellm_router, "test_deployment", 500, Exception("Test")) is False + @pytest.fixture def single_deployment_router(): """A router with one deployment whose model_info.id is the lookup-able @@ -1571,6 +1663,7 @@ def single_deployment_router(): ] ) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_should_run_cooldown_logic_generic_bad_request_excluded_by_default( single_deployment_router, @@ -1582,6 +1675,7 @@ def test_should_run_cooldown_logic_generic_bad_request_excluded_by_default( exc = litellm.BadRequestError("bad request", "openai", "gpt-5-mini") assert _should_run_cooldown_logic(single_deployment_router, "dep-1", 400, exc) is False + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_should_run_cooldown_logic_router_level_policy_does_not_override_bad_request_exclusion( single_deployment_router, @@ -1595,6 +1689,7 @@ def test_should_run_cooldown_logic_router_level_policy_does_not_override_bad_req single_deployment_router.allowed_fails_policy = AllowedFailsPolicy(BadRequestErrorAllowedFails=5) assert _should_run_cooldown_logic(single_deployment_router, "dep-1", 400, exc) is False + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_should_run_cooldown_logic_explicit_deployment_level_policy_overrides_content_policy_exclusion( single_deployment_router, @@ -1606,6 +1701,7 @@ def test_should_run_cooldown_logic_explicit_deployment_level_policy_overrides_co deployment_dict["model_info"]["allowed_fails_policy"] = {"ContentPolicyViolationErrorAllowedFails": 0} assert _should_run_cooldown_logic(single_deployment_router, "dep-1", 400, exc) is True + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestHasExplicitAllowedFailsPolicyForException: def test_no_policy_anywhere_returns_false(self, single_deployment_router): @@ -1636,6 +1732,7 @@ def test_none_deployment_returns_false(self, single_deployment_router): single_deployment_router.allowed_fails_policy = AllowedFailsPolicy(RateLimitErrorAllowedFails=3) assert _has_explicit_allowed_fails_policy_for_exception(single_deployment_router, None, exc) is False + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_should_cooldown_deployment_rate_limit_error(testing_litellm_router): """ @@ -1645,6 +1742,7 @@ def test_should_cooldown_deployment_rate_limit_error(testing_litellm_router): _exception = litellm.exceptions.RateLimitError("Rate limit", "openai", "gpt-5-mini") assert _should_cooldown_deployment(testing_litellm_router, "test_deployment", 429, _exception) is True + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_should_cooldown_deployment_auth_limit_error(testing_litellm_router): """ @@ -1654,6 +1752,7 @@ def test_should_cooldown_deployment_auth_limit_error(testing_litellm_router): _exception = litellm.exceptions.AuthenticationError("Unauthorized", "openai", "gpt-5-mini") assert _should_cooldown_deployment(testing_litellm_router, "test_deployment", 401, _exception) is True + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.parametrize("exception_status", (401, 402)) def test_is_cooldown_required_for_account_errors(testing_litellm_router, exception_status): @@ -1666,6 +1765,7 @@ def test_is_cooldown_required_for_account_errors(testing_litellm_router, excepti is True ) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.parametrize("allowed_fails", (None, 0)) def test_single_deployment_402_does_not_cooldown( @@ -1694,6 +1794,7 @@ def test_single_deployment_402_does_not_cooldown( is False ) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_single_deployment_402_respects_router_allowed_fails_policy() -> None: assert ( @@ -1719,6 +1820,7 @@ def test_single_deployment_402_respects_router_allowed_fails_policy() -> None: is True ) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_single_deployment_402_respects_deployment_allowed_fails_policy() -> None: assert ( @@ -1746,6 +1848,7 @@ def test_single_deployment_402_respects_deployment_allowed_fails_policy() -> Non is True ) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_multi_deployment_402_cools_down(testing_litellm_router: Router) -> None: assert ( @@ -1762,6 +1865,7 @@ def test_multi_deployment_402_cools_down(testing_litellm_router: Router) -> None is True ) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_should_cooldown_deployment(testing_litellm_router): @@ -1812,6 +1916,7 @@ async def test_should_cooldown_deployment(testing_litellm_router): # expect this to fail since it's now 51% of requests are failing assert _should_cooldown_deployment(testing_litellm_router, deployment_id, 500, Exception("Test")) is True + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_should_cooldown_deployment_allowed_fails_set_on_router(): @@ -1838,6 +1943,7 @@ async def test_should_cooldown_deployment_allowed_fails_set_on_router(): assert _should_cooldown_deployment(router, "test_deployment", 500, Exception("Test")) is True + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_increment_deployment_successes_for_current_minute_does_not_write_to_redis( testing_litellm_router, @@ -1879,12 +1985,14 @@ def test_increment_deployment_successes_for_current_minute_does_not_write_to_red ) assert testing_litellm_router.cache.in_memory_cache.get_cache("test_deployment:successes") is not None + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_cast_exception_status_to_int(): assert cast_exception_status_to_int(200) == 200 assert cast_exception_status_to_int("404") == 404 assert cast_exception_status_to_int("invalid") == 500 + @pytest.fixture def router(): return Router( @@ -1899,6 +2007,7 @@ def router(): ] ) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @patch("litellm.router_utils.cooldown_handlers.get_deployment_successes_for_current_minute") @patch("litellm.router_utils.cooldown_handlers.get_deployment_failures_for_current_minute") @@ -1918,6 +2027,7 @@ def test_should_cooldown_high_traffic_all_fails(mock_failures, mock_successes, r assert should_cooldown is True, "Should cooldown when all requests fail with sufficient traffic" + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @patch("litellm.router_utils.cooldown_handlers.get_deployment_successes_for_current_minute") @patch("litellm.router_utils.cooldown_handlers.get_deployment_failures_for_current_minute") @@ -1935,6 +2045,7 @@ def test_no_cooldown_low_traffic(mock_failures, mock_successes, router): assert should_cooldown is False, "Should not cooldown when traffic is below threshold" + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @patch("litellm.router_utils.cooldown_handlers.get_deployment_successes_for_current_minute") @patch("litellm.router_utils.cooldown_handlers.get_deployment_failures_for_current_minute") @@ -1954,6 +2065,7 @@ def test_cooldown_rate_limit(mock_failures, mock_successes, router): assert should_cooldown is False, "Should not cooldown on rate limit error for single deployment models" + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @patch("litellm.router_utils.cooldown_handlers.get_deployment_successes_for_current_minute") @patch("litellm.router_utils.cooldown_handlers.get_deployment_failures_for_current_minute") @@ -1971,6 +2083,7 @@ def test_mixed_success_failure(mock_failures, mock_successes, router): assert should_cooldown is False, "Should not cooldown when failure rate is below threshold" + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_is_cooldown_required_empty_string_exception_status(testing_litellm_router): """ @@ -1984,6 +2097,7 @@ def test_is_cooldown_required_empty_string_exception_status(testing_litellm_rout assert result is False, "Should not require cooldown when exception_status is empty string" + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_should_cooldown_deployment_minimum_request_threshold(testing_litellm_router): """ @@ -2031,6 +2145,7 @@ def test_should_cooldown_deployment_minimum_request_threshold(testing_litellm_ro f"Should cooldown when we have {DEFAULT_FAILURE_THRESHOLD_MINIMUM_REQUESTS} failed requests (100% failure rate)" ) + @pytest.mark.asyncio async def test_dynamic_cooldowns(): """ @@ -2072,6 +2187,7 @@ async def test_dynamic_cooldowns(): assert "cooldown_time" in tmp_mock.call_args[0][0]["litellm_params"] assert tmp_mock.call_args[0][0]["litellm_params"]["cooldown_time"] == 0 + @pytest.mark.asyncio async def test_cooldown_time_zero_uses_zero_not_default(): """ @@ -2116,6 +2232,7 @@ async def test_cooldown_time_zero_uses_zero_not_default(): healthy_deployments, _ = await router._async_get_healthy_deployments(model="gpt-3.5-turbo", parent_otel_span=None) assert len(healthy_deployments) == 1 + def test_should_run_cooldown_logic_early_exit_on_zero_cooldown(): """ Unit test for _should_run_cooldown_logic to verify early exit when time_to_cooldown is 0 @@ -2171,6 +2288,7 @@ def test_should_run_cooldown_logic_early_exit_on_zero_cooldown(): ) assert result is True, "Should run cooldown logic when time_to_cooldown is positive" + @pytest.mark.parametrize("num_deployments", [1, 2]) def test_single_deployment_no_cooldowns(num_deployments: int): """ @@ -2236,9 +2354,7 @@ async def test_single_deployment_no_cooldowns_test_prod(): num_retries=0, ) - with patch.object( - router.cooldown_cache, "add_deployment_to_cooldown", new=MagicMock() - ) as mock_client: + with patch.object(router.cooldown_cache, "add_deployment_to_cooldown", new=MagicMock()) as mock_client: try: await router.acompletion( model="gpt-3.5-turbo", @@ -2339,11 +2455,10 @@ async def test_high_traffic_cooldowns_all_healthy_deployments(): raise e print("model_stats: ", model_stats) - cooldown_list = await async_get_cooldown_deployments( - litellm_router_instance=router, parent_otel_span=None - ) + cooldown_list = await async_get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None) assert len(cooldown_list) == 0 + @pytest.mark.asyncio() async def test_high_traffic_cooldowns_one_bad_deployment(): """ @@ -2407,7 +2522,6 @@ async def test_high_traffic_cooldowns_one_bad_deployment(): mock_response = "hi" elif bad_deployment_id == model_id: if num_failures / total_requests <= 0.6: - mock_response = "litellm.InternalServerError" elif num_failures / total_requests <= 0.25: @@ -2435,11 +2549,10 @@ async def test_high_traffic_cooldowns_one_bad_deployment(): raise e print("model_stats: ", model_stats) - cooldown_list = await async_get_cooldown_deployments( - litellm_router_instance=router, parent_otel_span=None - ) + cooldown_list = await async_get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None) assert len(cooldown_list) == 1 + @pytest.mark.asyncio() async def test_high_traffic_cooldowns_one_rate_limited_deployment(): """ @@ -2503,7 +2616,6 @@ async def test_high_traffic_cooldowns_one_rate_limited_deployment(): mock_response = "hi" elif bad_deployment_id == model_id: if num_failures / total_requests <= 0.6: - mock_response = "litellm.RateLimitError" elif num_failures / total_requests <= 0.25: @@ -2534,11 +2646,10 @@ async def test_high_traffic_cooldowns_one_rate_limited_deployment(): raise e print("model_stats: ", model_stats) - cooldown_list = await async_get_cooldown_deployments( - litellm_router_instance=router, parent_otel_span=None - ) + cooldown_list = await async_get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None) assert len(cooldown_list) == 1 + def test_router_fallbacks_with_cooldowns_and_model_id(): """ Test that after a RateLimitError, the router can still route subsequent @@ -2574,6 +2685,7 @@ def test_router_fallbacks_with_cooldowns_and_model_id(): ) assert response is not None + @pytest.mark.asyncio() async def test_router_fallbacks_with_cooldowns_and_dynamic_credentials(): """ @@ -2617,7 +2729,10 @@ async def test_router_fallbacks_with_cooldowns_and_dynamic_credentials(): @pytest.mark.parametrize( "exception,status", [ - (litellm.CallerCredentialAuthenticationError(message="reconnect", llm_provider="github_copilot", model=""), 401), + ( + litellm.CallerCredentialAuthenticationError(message="reconnect", llm_provider="github_copilot", model=""), + 401, + ), (litellm.CallerCredentialRateLimitError(message="slow down", llm_provider="github_copilot", model=""), 429), ], ) @@ -2643,9 +2758,7 @@ def test_per_user_session_upstream_errors_never_cool_down_the_shared_deployment( (litellm.InternalServerError("copilot 500", "github_copilot", "gpt-4o"), 500), ): assert ( - _should_run_cooldown_logic( - single_deployment_router, "dep-1", status, exc, request_kwargs=session_kwargs - ) + _should_run_cooldown_logic(single_deployment_router, "dep-1", status, exc, request_kwargs=session_kwargs) is False ) @@ -2654,7 +2767,10 @@ def test_shared_mode_upstream_429_still_cools_down_the_deployment(single_deploym """Regression: without the session marker the same 429 is a deployment-health signal and cools down exactly as before.""" exc = litellm.RateLimitError("copilot 429", "github_copilot", "gpt-4o") - assert _should_run_cooldown_logic( - single_deployment_router, "dep-1", 429, exc, request_kwargs={"model": "github_copilot/gpt-4o"} - ) is True + assert ( + _should_run_cooldown_logic( + single_deployment_router, "dep-1", 429, exc, request_kwargs={"model": "github_copilot/gpt-4o"} + ) + is True + ) assert _should_run_cooldown_logic(single_deployment_router, "dep-1", 429, exc) is True From 5ab7f4ae31b76c1cb12e1f1df657c2437e6023c3 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 01:47:38 +0000 Subject: [PATCH 27/38] test(github_copilot): cover per-user 401 cooldown skip through router and fallback paths --- .../router_utils/test_cooldown_handlers.py | 51 ++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/tests/unit/router_utils/test_cooldown_handlers.py b/tests/unit/router_utils/test_cooldown_handlers.py index bbf806f7db59..5940253f0629 100644 --- a/tests/unit/router_utils/test_cooldown_handlers.py +++ b/tests/unit/router_utils/test_cooldown_handlers.py @@ -2,6 +2,7 @@ import importlib import random import time +from collections.abc import Mapping from typing import Final from unittest.mock import MagicMock, patch @@ -1452,7 +1453,7 @@ async def test_fallback_api_key_auth_failure_still_cools_down(self) -> None: @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestPerUserCopilotAuthFailureCooldown: @staticmethod - def _router(model_id: str, litellm_params: dict[str, object]) -> Router: + def _router(model_id: str, litellm_params: Mapping[str, object]) -> Router: return _make_router( model_list=[ { @@ -1523,6 +1524,54 @@ async def test_shared_mode_copilot_auth_failure_still_cools_down(self) -> None: assert get_deployment_failures_for_current_minute(router, model_id) == 1 assert get_cooldown_deployments(router, parent_otel_span=None) == [model_id] + @pytest.mark.asyncio + async def test_fallback_per_user_caller_auth_failure_does_not_cooldown( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + from litellm.constants import GITHUB_COPILOT_AUTH_TYPE_KEY, GITHUB_COPILOT_PER_USER_AUTH_TYPE + + model_id: Final = "fallback-per-user" + monkeypatch.setattr( + litellm, + "credential_list", + [ + CredentialItem( + credential_name="copilot-per-user", + credential_values={GITHUB_COPILOT_AUTH_TYPE_KEY: GITHUB_COPILOT_PER_USER_AUTH_TYPE}, + credential_info={"custom_llm_provider": "github_copilot"}, + ) + ], + ) + router: Final = self._router(model_id, {"litellm_credential_name": "copilot-per-user"}) + exception: Final = litellm.CallerCredentialAuthenticationError( + message="reconnect", llm_provider="github_copilot", model="" + ) + exception.failed_deployment_id = model_id + + _trigger_cooldown_for_failed_deployment(litellm_router=router, kwargs={}, exception=exception) + + assert get_cooldown_deployments(router, parent_otel_span=None) == [] + + @pytest.mark.asyncio + async def test_fallback_shared_mode_auth_failure_still_cools_down(self) -> None: + from litellm.llms.github_copilot.authenticator import Authenticator + + model_id: Final = "fallback-shared" + with ( + patch.object(Authenticator, "get_api_key", return_value="shared-copilot-token"), + patch.object(Authenticator, "get_api_base", return_value="https://api.githubcopilot.com"), + ): + router: Final = self._router(model_id, {"api_key": "sk-shared"}) + exception: Final = litellm.AuthenticationError( + "upstream rejected the shared token", "github_copilot", "gpt-4o" + ) + exception.failed_deployment_id = model_id + + _trigger_cooldown_for_failed_deployment(litellm_router=router, kwargs={}, exception=exception) + + assert get_cooldown_deployments(router, parent_otel_span=None) == [model_id] + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestNewAllowedFailsPolicyFields: From 66e5da34827e6e306c38345df15a67b97b1d51ae Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 02:18:09 +0000 Subject: [PATCH 28/38] fix(github_copilot): cover auto-router discovery, per-user fallback cooldown and connect cache race --- .../proxy/credential_endpoints/endpoints.py | 9 ++- .../user_provider_credentials.py | 24 +++++++ litellm/proxy/litellm_pre_call_utils.py | 26 +++++++ litellm/router_utils/cooldown_handlers.py | 14 ++++ .../test_user_provider_credentials.py | 49 ++++++++++++++ .../unit/proxy/test_litellm_pre_call_utils.py | 67 +++++++++++++++++++ .../router_utils/test_cooldown_handlers.py | 65 +++++++++++++++++- 7 files changed, 249 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/credential_endpoints/endpoints.py b/litellm/proxy/credential_endpoints/endpoints.py index 47a22f683690..72f77ddd5385 100644 --- a/litellm/proxy/credential_endpoints/endpoints.py +++ b/litellm/proxy/credential_endpoints/endpoints.py @@ -67,10 +67,10 @@ decode_user_provider_credential, delete_user_provider_credential, delete_user_provider_credentials_for_credential, - drop_user_provider_credential_cache, invalidate_user_provider_credential_cache, list_user_provider_credentials, list_user_provider_credentials_for_credential, + set_user_provider_credential_cache, upsert_user_provider_credential, ) @@ -640,14 +640,17 @@ async def poll_user_connection( except litellm.CallerCredentialAuthenticationError: return UserConnectionPollResponse(status="no_copilot_seat") github_login: Final = await afetch_github_login(github_token) + connection_payload: Final = GithubCopilotUserConnectionPayload( + access_token=github_token, github_login=github_login + ) await upsert_user_provider_credential( prisma_client, user_id, credential_name, _GITHUB_COPILOT_PROVIDER, - GithubCopilotUserConnectionPayload(access_token=github_token, github_login=github_login), + connection_payload, ) - await drop_user_provider_credential_cache(user_api_key_cache, user_id, credential_name) + await set_user_provider_credential_cache(user_api_key_cache, user_id, credential_name, connection_payload) return UserConnectionPollResponse(status="connected", github_login=github_login) except litellm.CallerCredentialRateLimitError as e: raise HTTPException(status_code=429, detail=str(e)) diff --git a/litellm/proxy/credential_endpoints/user_provider_credentials.py b/litellm/proxy/credential_endpoints/user_provider_credentials.py index 64bd6bd5d58e..08f24fcfabd6 100644 --- a/litellm/proxy/credential_endpoints/user_provider_credentials.py +++ b/litellm/proxy/credential_endpoints/user_provider_credentials.py @@ -216,6 +216,30 @@ async def drop_user_provider_credential_cache( verbose_proxy_logger.warning("drop_user_provider_credential_cache: Redis delete failed") +@with_service_target("user_provider_connections") +async def set_user_provider_credential_cache( + cache: DualCache, + user_id: str, + credential_name: str, + payload: GithubCopilotUserConnectionPayload, +) -> None: + """Connect path: overwrite the key with the new connection's ciphertext so a + stale set-if-absent fill from a pre-connect read cannot resurrect a + not-connected marker. Falls back to a plain delete when the write raises.""" + token_cache: Final = cache.redis_cache + if token_cache is None: + return + try: + await token_cache.async_set_cache( + _cache_key(user_id, credential_name), + _encode(payload), + ttl=GITHUB_COPILOT_USER_CREDENTIAL_CACHE_TTL_SECONDS, + ) + except Exception: # noqa: BLE001 # a Redis outage must not fail a connect + verbose_proxy_logger.warning("set_user_provider_credential_cache: Redis set failed; falling back to delete") + await drop_user_provider_credential_cache(cache, user_id, credential_name) + + @with_service_target("user_provider_connections") async def aget_user_provider_tokens( prisma_client: "PrismaClient", diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 385f3e51e843..81d00a958985 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -2716,6 +2716,24 @@ def _per_user_oauth_configured() -> bool: ) +def _all_per_user_credential_names() -> tuple[str, ...]: + """Every credential name configured for per-user GitHub OAuth, so one DB + query can prefetch all of the caller's connections when an auto-router + deployment may route to a Copilot group at request time.""" + from litellm.constants import ( + GITHUB_COPILOT_AUTH_TYPE_KEY, + GITHUB_COPILOT_PER_USER_AUTH_TYPE, + ) + + return tuple( + credential.credential_name + for credential in (litellm.credential_list or ()) + if getattr(credential, "credential_name", None) + and isinstance(values := getattr(credential, "credential_values", None), Mapping) + and values.get(GITHUB_COPILOT_AUTH_TYPE_KEY) == GITHUB_COPILOT_PER_USER_AUTH_TYPE + ) + + def _fallback_entry_target_count(entry: object) -> int: """Work units one fallback entry carries, floored so nothing counts as zero: a bare string or any non-dict counts 1; a dict entry counts one unit per @@ -2870,6 +2888,7 @@ def _per_user_credential_names_for_groups( GITHUB_COPILOT_AUTH_TYPE_KEY, GITHUB_COPILOT_PER_USER_AUTH_TYPE, ) + from litellm.router_utils.auto_router_model_naming import classify_strategy_router_model names: Final[list[str]] = [] # mutable-ok: accumulates one name per per-user deployment for group in model_groups: @@ -2885,6 +2904,13 @@ def _per_user_credential_names_for_groups( if isinstance(deployment, Mapping) else getattr(deployment, "litellm_params", None) ) + deployment_model: object = ( + litellm_params.get("model") + if isinstance(litellm_params, Mapping) + else getattr(litellm_params, "model", None) + ) + if isinstance(deployment_model, str) and classify_strategy_router_model(deployment_model) == "semantic": + return _all_per_user_credential_names() credential_name_obj: object = ( litellm_params.get("litellm_credential_name") if isinstance(litellm_params, Mapping) diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index 9948b8bb35c0..a5f6277d2d74 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -326,6 +326,20 @@ def _should_run_cooldown_logic( ) return False + if deployment is not None: + failed_deployment: Final = litellm_router_instance.get_deployment(model_id=deployment) + if failed_deployment is not None: + from litellm.llms.github_copilot.per_user_auth import ( + github_copilot_per_user_credential_name, + ) + + failed_litellm_params: Final = getattr(failed_deployment, "litellm_params", None) + if github_copilot_per_user_credential_name(failed_litellm_params) is not None: + verbose_router_logger.debug( + "Should Not Run Cooldown Logic: the failed deployment authenticates per caller" + ) + return False + ######################################################### # If time_to_cooldown is 0 or 0.0000000, don't run cooldown logic ######################################################### diff --git a/tests/unit/proxy/credential_endpoints/test_user_provider_credentials.py b/tests/unit/proxy/credential_endpoints/test_user_provider_credentials.py index 939db2418355..e389ee57dd4e 100644 --- a/tests/unit/proxy/credential_endpoints/test_user_provider_credentials.py +++ b/tests/unit/proxy/credential_endpoints/test_user_provider_credentials.py @@ -15,6 +15,7 @@ drop_user_provider_credential_cache, get_user_provider_credential, invalidate_user_provider_credential_cache, + set_user_provider_credential_cache, upsert_user_provider_credential, ) @@ -285,3 +286,51 @@ async def test_without_redis_no_worker_local_entries_are_kept(): await invalidate_user_provider_credential_cache(cache_b, "user-a", "copilot-cred") assert await aget_user_provider_tokens(prisma_client, cache_a, "user-a", ["copilot-cred"]) == {} assert table.find_many.await_count == 3 + + +@pytest.mark.asyncio +async def test_connect_overwrites_a_not_connected_tombstone(monkeypatch): + """Connect writes the new connection over any cached tombstone so the next + read sees the token immediately, no TTL wait.""" + from litellm.proxy.credential_endpoints import user_provider_credentials as upc + + redis = _fake_redis(monkeypatch) + cache = DualCache(redis_cache=redis) + await invalidate_user_provider_credential_cache(cache, "user-a", "copilot-cred") + + payload = GithubCopilotUserConnectionPayload(access_token="gho_secret", github_login="octo") + await set_user_provider_credential_cache(cache, "user-a", "copilot-cred", payload) + + table = MagicMock() + table.find_many = AsyncMock(return_value=[]) + prisma_client = _prisma(table) + assert await aget_user_provider_tokens(prisma_client, cache, "user-a", ["copilot-cred"]) == { + "copilot-cred": "gho_secret" + } + table.find_many.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_stale_not_connected_fill_cannot_overwrite_a_connect_write(monkeypatch): + """A read that fetched "not connected" before the poll saved the row fills + with set-if-absent; the connect's plain overwrite must win so the marker + never lingers for the TTL.""" + from litellm.proxy.credential_endpoints import user_provider_credentials as upc + + redis = _fake_redis(monkeypatch) + cache = DualCache(redis_cache=redis) + + payload = GithubCopilotUserConnectionPayload(access_token="gho_secret", github_login="octo") + + async def _read_while_connecting(*args, **kwargs): + await set_user_provider_credential_cache(cache, "user-a", "copilot-cred", payload) + return [] + + table = MagicMock() + table.find_many = AsyncMock(side_effect=_read_while_connecting) + prisma_client = _prisma(table) + + await aget_user_provider_tokens(prisma_client, cache, "user-a", ["copilot-cred"]) + assert await aget_user_provider_tokens(prisma_client, cache, "user-a", ["copilot-cred"]) == { + "copilot-cred": "gho_secret" + } diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 1fd60af957c6..dc74301e81a4 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -9509,6 +9509,73 @@ def _spy(fallbacks, model_group): router.get_model_list.assert_not_called() assert data["secret_fields"] == {} + @pytest.mark.asyncio + async def test_auto_router_deployment_loads_every_per_user_connection(self, monkeypatch): + """An auto-router deployment can pick a Copilot group at request time, so + discovery must fall back to every per-user credential the caller has.""" + from litellm.proxy.credential_endpoints import user_provider_credentials as upc + from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request + + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "sk-test-master") + table: Final = self._env(monkeypatch) + ciphertext: Final = upc._encode( + upc.GithubCopilotUserConnectionPayload(access_token="gho_secret", github_login="octo") + ) + table.find_many = AsyncMock( + return_value=[SimpleNamespace(user_id="user-a", credential_name="copilot-cred", credential_b64=ciphertext)] + ) + router: Final = MagicMock() + router.fallbacks = [] + router.context_window_fallbacks = [] + router.content_policy_fallbacks = [] + router.get_model_list = MagicMock( + return_value=[ + { + "model_name": "auto-chat", + "litellm_params": {"model": "auto_router/copilot-picker"}, + } + ] + ) + router.get_deployment = MagicMock(return_value=None) + data: Final = {"model": "auto-chat", "secret_fields": {}} + await _resolve_user_provider_credentials_for_request( + data=data, + authenticated_user_id="user-a", + team_id=None, + llm_router=router, + ) + assert data["secret_fields"]["user_provider_credentials"] == {"copilot-cred": "gho_secret"} + call_where: Final = table.find_many.await_args.kwargs["where"] + assert call_where == {"AND": ({"user_id": "user-a"}, {"credential_name": {"in": ["copilot-cred"]}})} + + @pytest.mark.asyncio + async def test_non_per_user_group_loads_no_credentials(self, monkeypatch): + from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request + + table: Final = self._env(monkeypatch) + router: Final = MagicMock() + router.fallbacks = [] + router.context_window_fallbacks = [] + router.content_policy_fallbacks = [] + router.get_model_list = MagicMock( + return_value=[ + { + "model_name": "shared-chat", + "litellm_params": {"model": "github_copilot/gpt-4o", "litellm_credential_name": "shared-cred"}, + } + ] + ) + router.get_deployment = MagicMock(return_value=None) + data: Final = {"model": "shared-chat", "secret_fields": {}} + await _resolve_user_provider_credentials_for_request( + data=data, + authenticated_user_id="user-a", + team_id=None, + llm_router=router, + ) + table.find_many.assert_not_awaited() + assert data["secret_fields"] == {} + @pytest.mark.asyncio async def test_wildcard_targets_expand_once_not_per_group(self, monkeypatch): """An admin ``{"*": [5000 targets]}`` chain must expand the entry once: diff --git a/tests/unit/router_utils/test_cooldown_handlers.py b/tests/unit/router_utils/test_cooldown_handlers.py index 5940253f0629..3a5097d5b1f9 100644 --- a/tests/unit/router_utils/test_cooldown_handlers.py +++ b/tests/unit/router_utils/test_cooldown_handlers.py @@ -1453,7 +1453,11 @@ async def test_fallback_api_key_auth_failure_still_cools_down(self) -> None: @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestPerUserCopilotAuthFailureCooldown: @staticmethod - def _router(model_id: str, litellm_params: Mapping[str, object]) -> Router: + def _router( + model_id: str, + litellm_params: Mapping[str, object], + allowed_fails_policy: Mapping[str, int] | None = None, + ) -> Router: return _make_router( model_list=[ { @@ -1462,7 +1466,12 @@ def _router(model_id: str, litellm_params: Mapping[str, object]) -> Router: "model": "github_copilot/gpt-4o", **litellm_params, }, - "model_info": {"id": model_id}, + "model_info": { + "id": model_id, + "allowed_fails_policy": dict(allowed_fails_policy) + if allowed_fails_policy is not None + else None, + }, } ] ) @@ -1572,6 +1581,58 @@ async def test_fallback_shared_mode_auth_failure_still_cools_down(self) -> None: assert get_cooldown_deployments(router, parent_otel_span=None) == [model_id] + @pytest.mark.asyncio + async def test_fallback_rate_limit_on_per_user_deployment_does_not_cooldown( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + from litellm.constants import GITHUB_COPILOT_AUTH_TYPE_KEY, GITHUB_COPILOT_PER_USER_AUTH_TYPE + + model_id: Final = "fallback-per-user-429" + monkeypatch.setattr( + litellm, + "credential_list", + [ + CredentialItem( + credential_name="copilot-per-user", + credential_values={GITHUB_COPILOT_AUTH_TYPE_KEY: GITHUB_COPILOT_PER_USER_AUTH_TYPE}, + credential_info={"custom_llm_provider": "github_copilot"}, + ) + ], + ) + router: Final = self._router( + model_id, + {"litellm_credential_name": "copilot-per-user"}, + allowed_fails_policy={"RateLimitErrorAllowedFails": 0}, + ) + exception: Final = litellm.RateLimitError("copilot 429", "github_copilot", "gpt-4o") + exception.failed_deployment_id = model_id + + _trigger_cooldown_for_failed_deployment(litellm_router=router, kwargs={}, exception=exception) + + assert get_cooldown_deployments(router, parent_otel_span=None) == [] + + @pytest.mark.asyncio + async def test_fallback_rate_limit_on_shared_deployment_still_cools_down(self) -> None: + from litellm.llms.github_copilot.authenticator import Authenticator + + model_id: Final = "fallback-shared-429" + with ( + patch.object(Authenticator, "get_api_key", return_value="shared-copilot-token"), + patch.object(Authenticator, "get_api_base", return_value="https://api.githubcopilot.com"), + ): + router: Final = self._router( + model_id, + {"api_key": "sk-shared"}, + allowed_fails_policy={"RateLimitErrorAllowedFails": 0}, + ) + exception: Final = litellm.RateLimitError("copilot 429", "github_copilot", "gpt-4o") + exception.failed_deployment_id = model_id + + _trigger_cooldown_for_failed_deployment(litellm_router=router, kwargs={}, exception=exception) + + assert get_cooldown_deployments(router, parent_otel_span=None) == [model_id] + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestNewAllowedFailsPolicyFields: From 46648adf78bad84a16db7a7bf5610d0b3a0065a6 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 02:44:23 +0000 Subject: [PATCH 29/38] fix(github_copilot): cover every strategy router kind and verify the connect cache write --- .../proxy/credential_endpoints/endpoints.py | 9 +++- .../user_provider_credentials.py | 29 +++++++++--- litellm/proxy/litellm_pre_call_utils.py | 2 +- .../credential_endpoints/test_endpoints.py | 36 +++++++++++++++ .../test_user_provider_credentials.py | 44 ++++++++++++++++++- .../unit/proxy/test_litellm_pre_call_utils.py | 14 +++++- 6 files changed, 123 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/credential_endpoints/endpoints.py b/litellm/proxy/credential_endpoints/endpoints.py index 72f77ddd5385..c2b8dabbe755 100644 --- a/litellm/proxy/credential_endpoints/endpoints.py +++ b/litellm/proxy/credential_endpoints/endpoints.py @@ -650,7 +650,14 @@ async def poll_user_connection( _GITHUB_COPILOT_PROVIDER, connection_payload, ) - await set_user_provider_credential_cache(user_api_key_cache, user_id, credential_name, connection_payload) + if not await set_user_provider_credential_cache( + user_api_key_cache, user_id, credential_name, connection_payload + ): + raise HTTPException( + status_code=503, + detail="GitHub connection saved, but the cache could not be refreshed; " + "requests may be rejected for up to 60 seconds", + ) return UserConnectionPollResponse(status="connected", github_login=github_login) except litellm.CallerCredentialRateLimitError as e: raise HTTPException(status_code=429, detail=str(e)) diff --git a/litellm/proxy/credential_endpoints/user_provider_credentials.py b/litellm/proxy/credential_endpoints/user_provider_credentials.py index 08f24fcfabd6..6d9827b6e720 100644 --- a/litellm/proxy/credential_endpoints/user_provider_credentials.py +++ b/litellm/proxy/credential_endpoints/user_provider_credentials.py @@ -222,22 +222,41 @@ async def set_user_provider_credential_cache( user_id: str, credential_name: str, payload: GithubCopilotUserConnectionPayload, -) -> None: +) -> bool: """Connect path: overwrite the key with the new connection's ciphertext so a stale set-if-absent fill from a pre-connect read cannot resurrect a - not-connected marker. Falls back to a plain delete when the write raises.""" + not-connected marker. The overwrite is verified by decoding the key back + (async_set_cache swallows write errors); on a mismatch the key is deleted, + and False means a stale entry survived both attempts.""" token_cache: Final = cache.redis_cache if token_cache is None: - return + return True + key: Final = _cache_key(user_id, credential_name) try: await token_cache.async_set_cache( - _cache_key(user_id, credential_name), + key, _encode(payload), ttl=GITHUB_COPILOT_USER_CREDENTIAL_CACHE_TTL_SECONDS, ) + readback: object = await _try_cache_get(token_cache, key) + if isinstance(readback, str) and decode_user_provider_credential(readback) == payload: + return True + verbose_proxy_logger.warning( + "set_user_provider_credential_cache: Redis overwrite not visible; falling back to delete" + ) except Exception: # noqa: BLE001 # a Redis outage must not fail a connect verbose_proxy_logger.warning("set_user_provider_credential_cache: Redis set failed; falling back to delete") - await drop_user_provider_credential_cache(cache, user_id, credential_name) + try: + await token_cache.async_delete_cache(key) + except Exception: # noqa: BLE001 # a Redis outage must not fail a connect + verbose_proxy_logger.warning("set_user_provider_credential_cache: Redis delete failed") + after_delete: object = await _try_cache_get(token_cache, key) + if after_delete is None: + return True + if isinstance(after_delete, str) and decode_user_provider_credential(after_delete) == payload: + return True + verbose_proxy_logger.warning("set_user_provider_credential_cache: stale entry survived delete") + return False @with_service_target("user_provider_connections") diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 81d00a958985..afc5dd59757d 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -2909,7 +2909,7 @@ def _per_user_credential_names_for_groups( if isinstance(litellm_params, Mapping) else getattr(litellm_params, "model", None) ) - if isinstance(deployment_model, str) and classify_strategy_router_model(deployment_model) == "semantic": + if isinstance(deployment_model, str) and classify_strategy_router_model(deployment_model) is not None: return _all_per_user_credential_names() credential_name_obj: object = ( litellm_params.get("litellm_credential_name") diff --git a/tests/unit/proxy/credential_endpoints/test_endpoints.py b/tests/unit/proxy/credential_endpoints/test_endpoints.py index a268894e6209..a0f17b6a0a80 100644 --- a/tests/unit/proxy/credential_endpoints/test_endpoints.py +++ b/tests/unit/proxy/credential_endpoints/test_endpoints.py @@ -2102,3 +2102,39 @@ def test_delete_user_connection_fails_when_the_tombstone_write_silently_noops(mo response = _call_as("DELETE", "/credentials/copilot-cred/user_connection", auth=_as_user()) assert response.status_code == 503, response.text table.delete_many.assert_not_awaited() + + +def test_user_connection_poll_reports_503_when_the_cache_overwrite_cannot_land(monkeypatch): + """The connect save is durable, so a stale not-connected marker that survives + both the overwrite and the delete must surface as a retryable 503 rather + than a false "connected".""" + table = _patch_user_connection_env(monkeypatch, [_per_user_credential()]) + from types import SimpleNamespace + + from litellm.proxy import proxy_server + + stale_redis = SimpleNamespace( + async_set_cache=AsyncMock(return_value=None), + async_get_cache=AsyncMock(return_value="stale-value"), + async_delete_cache=AsyncMock(return_value=None), + ) + monkeypatch.setattr(proxy_server, "user_api_key_cache", DualCache(redis_cache=stale_redis)) + + with _github_http( + { + _DEVICE_CODE_URL: _DEVICE_FLOW_START, + _ACCESS_TOKEN_URL: {"access_token": "gho_1"}, + _COPILOT_TOKEN_URL: {"token": "copilot-tok", "expires_at": 4102444800}, + _GITHUB_USER_URL: {"login": "octo"}, + } + ): + start = _call_as("POST", "/credentials/copilot-cred/user_connection/start", auth=_as_user()) + poll = _call_as( + "POST", + "/credentials/copilot-cred/user_connection/poll", + json_body={"flow_handle": start.json()["flow_handle"]}, + auth=_as_user(), + ) + assert poll.status_code == 503, poll.text + assert "cache could not be refreshed" in poll.text + table.upsert.assert_awaited_once() diff --git a/tests/unit/proxy/credential_endpoints/test_user_provider_credentials.py b/tests/unit/proxy/credential_endpoints/test_user_provider_credentials.py index e389ee57dd4e..dd8456fc035a 100644 --- a/tests/unit/proxy/credential_endpoints/test_user_provider_credentials.py +++ b/tests/unit/proxy/credential_endpoints/test_user_provider_credentials.py @@ -299,7 +299,7 @@ async def test_connect_overwrites_a_not_connected_tombstone(monkeypatch): await invalidate_user_provider_credential_cache(cache, "user-a", "copilot-cred") payload = GithubCopilotUserConnectionPayload(access_token="gho_secret", github_login="octo") - await set_user_provider_credential_cache(cache, "user-a", "copilot-cred", payload) + assert await set_user_provider_credential_cache(cache, "user-a", "copilot-cred", payload) table = MagicMock() table.find_many = AsyncMock(return_value=[]) @@ -323,7 +323,7 @@ async def test_stale_not_connected_fill_cannot_overwrite_a_connect_write(monkeyp payload = GithubCopilotUserConnectionPayload(access_token="gho_secret", github_login="octo") async def _read_while_connecting(*args, **kwargs): - await set_user_provider_credential_cache(cache, "user-a", "copilot-cred", payload) + assert await set_user_provider_credential_cache(cache, "user-a", "copilot-cred", payload) return [] table = MagicMock() @@ -334,3 +334,43 @@ async def _read_while_connecting(*args, **kwargs): assert await aget_user_provider_tokens(prisma_client, cache, "user-a", ["copilot-cred"]) == { "copilot-cred": "gho_secret" } + + +@pytest.mark.asyncio +async def test_set_reports_connected_without_redis(): + """No Redis configured: there is no stale entry to defend against, so the + cache refresh succeeds by definition.""" + payload = GithubCopilotUserConnectionPayload(access_token="gho_secret", github_login="octo") + assert await set_user_provider_credential_cache(DualCache(), "user-a", "copilot-cred", payload) + + +@pytest.mark.asyncio +async def test_set_reports_connected_when_delete_clears_a_dropped_write(): + """async_set_cache swallows errors: a set that reports success but writes + nothing is verified by read-back, and a working delete still resolves the + stale entry.""" + store: dict = {} + silent_redis = SimpleNamespace( + async_set_cache=AsyncMock(return_value=None), + async_get_cache=AsyncMock(side_effect=lambda key: store.get(key)), + async_delete_cache=AsyncMock(side_effect=lambda key: store.pop(key, None)), + ) + cache = DualCache(redis_cache=silent_redis) + payload = GithubCopilotUserConnectionPayload(access_token="gho_secret", github_login="octo") + assert await set_user_provider_credential_cache(cache, "user-a", "copilot-cred", payload) + + +@pytest.mark.asyncio +async def test_set_reports_failure_when_a_stale_entry_survives_both_attempts(): + """A write silently dropped while a not-connected marker stays cached means + the caller cannot trust the connect: the helper reports failure instead of + claiming success.""" + stale = "stale-value" + silent_redis = SimpleNamespace( + async_set_cache=AsyncMock(return_value=None), + async_get_cache=AsyncMock(return_value=stale), + async_delete_cache=AsyncMock(return_value=None), + ) + cache = DualCache(redis_cache=silent_redis) + payload = GithubCopilotUserConnectionPayload(access_token="gho_secret", github_login="octo") + assert not await set_user_provider_credential_cache(cache, "user-a", "copilot-cred", payload) diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index dc74301e81a4..54e4175fb5ca 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -9510,7 +9510,17 @@ def _spy(fallbacks, model_group): assert data["secret_fields"] == {} @pytest.mark.asyncio - async def test_auto_router_deployment_loads_every_per_user_connection(self, monkeypatch): + @pytest.mark.parametrize( + "auto_router_model", + [ + "auto_router/copilot-picker", + "auto_router/complexity_router", + "auto_router/adaptive_router", + "auto_router/quality_router", + ], + ids=["semantic", "complexity", "adaptive", "quality"], + ) + async def test_auto_router_deployment_loads_every_per_user_connection(self, monkeypatch, auto_router_model: str): """An auto-router deployment can pick a Copilot group at request time, so discovery must fall back to every per-user credential the caller has.""" from litellm.proxy.credential_endpoints import user_provider_credentials as upc @@ -9532,7 +9542,7 @@ async def test_auto_router_deployment_loads_every_per_user_connection(self, monk return_value=[ { "model_name": "auto-chat", - "litellm_params": {"model": "auto_router/copilot-picker"}, + "litellm_params": {"model": auto_router_model}, } ] ) From 4cd0cea334ae8171106ac6e0bf44d2036406d7b1 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 07:43:57 +0000 Subject: [PATCH 30/38] fix merge conflict marker remnant in credential endpoint tests --- tests/unit/proxy/credential_endpoints/test_endpoints.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/proxy/credential_endpoints/test_endpoints.py b/tests/unit/proxy/credential_endpoints/test_endpoints.py index 0e6337002684..a221df0cb8f9 100644 --- a/tests/unit/proxy/credential_endpoints/test_endpoints.py +++ b/tests/unit/proxy/credential_endpoints/test_endpoints.py @@ -2101,7 +2101,6 @@ def test_user_connection_poll_reports_503_when_the_cache_overwrite_cannot_land(m assert poll.status_code == 503, poll.text assert "cache could not be refreshed" in poll.text table.upsert.assert_awaited_once() -||||||| dc59dd123b def _labeled_credential(name: str = "openai-prod", display_name: str | None = "Prod OpenAI") -> CredentialItem: From 0408828bd2357270e42bf5dfe0e4337d4e5b8a07 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 08:12:09 +0000 Subject: [PATCH 31/38] fix(github_copilot): do not purge user connections on label-only credential updates --- .../proxy/credential_endpoints/endpoints.py | 2 +- .../credential_endpoints/test_endpoints.py | 59 +++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/credential_endpoints/endpoints.py b/litellm/proxy/credential_endpoints/endpoints.py index 3a98a685affe..eff48a533b66 100644 --- a/litellm/proxy/credential_endpoints/endpoints.py +++ b/litellm/proxy/credential_endpoints/endpoints.py @@ -924,7 +924,7 @@ async def update_credential( Mapping[object, object], merged_credential.credential_values ) still_per_user: Final = merged_values.get(GITHUB_COPILOT_AUTH_TYPE_KEY) == GITHUB_COPILOT_PER_USER_AUTH_TYPE - if credential.credential_name != credential_name or not still_per_user: + if not still_per_user: await _purge_user_connections_for_credential(credential_name) return {"success": True, "message": "Credential updated successfully"} diff --git a/tests/unit/proxy/credential_endpoints/test_endpoints.py b/tests/unit/proxy/credential_endpoints/test_endpoints.py index a221df0cb8f9..7dcb718e0e47 100644 --- a/tests/unit/proxy/credential_endpoints/test_endpoints.py +++ b/tests/unit/proxy/credential_endpoints/test_endpoints.py @@ -1483,6 +1483,7 @@ def _patch_user_connection_env(monkeypatch, credentials, rows=()): table.upsert = AsyncMock(return_value=None) table.delete_many = AsyncMock(return_value=None) prisma_client.db.litellm_userprovidercredentials = table + prisma_client.writer_db = prisma_client.db monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma_client) cache = DualCache() monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", cache) @@ -1617,6 +1618,7 @@ def test_deleting_the_credential_purges_user_connections(monkeypatch, credential table.find_many = AsyncMock(return_value=rows) table.delete_many = AsyncMock(return_value=None) prisma_client.db.litellm_userprovidercredentials = table + prisma_client.writer_db = prisma_client.db prisma_client.db.litellm_credentialstable.find_unique = AsyncMock(return_value=None) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma_client) monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", DualCache()) @@ -1862,6 +1864,63 @@ def test_proxy_admin_can_patch_a_credential_to_per_user_oauth(self, restore_cred assert response.status_code == 200, response.text +def test_label_only_patch_does_not_purge_user_connections(): + """A PATCH that only changes display_name sends no credential_name in the + body, and that must not read as a rename that purges every user's stored + connection.""" + stored = CredentialItem( + credential_name="copilot-cred", + credential_values={"github_copilot_auth_type": "per_user_oauth"}, + credential_info={"custom_llm_provider": "github_copilot"}, + ) + with _repository_holding(stored): + from litellm.proxy import proxy_server + + table = MagicMock() + table.find_many = AsyncMock(return_value=[_connection_row()]) + table.delete_many = AsyncMock(return_value=None) + proxy_server.prisma_client.db.litellm_userprovidercredentials = table + proxy_server.prisma_client.writer_db = proxy_server.prisma_client.db + + response = _patch_credential( + "copilot-cred", + {"display_name": "Renamed Copilot", "credential_info": {"custom_llm_provider": "github_copilot"}}, + ) + + assert response.status_code == 200, response.text + table.find_many.assert_not_awaited() + table.delete_many.assert_not_awaited() + + +def test_switching_away_from_per_user_oauth_purges_user_connections(): + stored = CredentialItem( + credential_name="copilot-cred", + credential_values={"github_copilot_auth_type": "per_user_oauth"}, + credential_info={"custom_llm_provider": "github_copilot"}, + ) + with _repository_holding(stored): + from litellm.proxy import proxy_server + + table = MagicMock() + table.find_many = AsyncMock(return_value=[_connection_row()]) + table.delete_many = AsyncMock(return_value=None) + proxy_server.prisma_client.db.litellm_userprovidercredentials = table + proxy_server.prisma_client.writer_db = proxy_server.prisma_client.db + + response = _patch_credential( + "copilot-cred", + { + "credential_name": "copilot-cred", + "credential_values": {}, + "credential_values_to_delete": ["github_copilot_auth_type"], + "credential_info": {"custom_llm_provider": "github_copilot"}, + }, + ) + + assert response.status_code == 200, response.text + table.delete_many.assert_awaited_with(where={"credential_name": "copilot-cred"}) + + def test_pending_device_flow_survives_disconnect_via_stateless_handle(monkeypatch): """The device flow is stateless: no worker-local entry exists to clear on disconnect, so a still-valid issued handle polled on any worker completes.""" From e4e1cf27c174a68ba81e473342e77131a6ea6800 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 08:12:09 +0000 Subject: [PATCH 32/38] fix(github_copilot): read user provider credentials from the writer engine --- .../user_provider_credentials.py | 2 +- .../test_user_provider_credentials.py | 20 +++++++++++++++++++ .../unit/proxy/test_litellm_pre_call_utils.py | 2 ++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/credential_endpoints/user_provider_credentials.py b/litellm/proxy/credential_endpoints/user_provider_credentials.py index 6d9827b6e720..47bb19e9b2ec 100644 --- a/litellm/proxy/credential_endpoints/user_provider_credentials.py +++ b/litellm/proxy/credential_endpoints/user_provider_credentials.py @@ -54,7 +54,7 @@ class _UserProviderCredentialsRepository(PrismaTableRepository["prisma_models.Li def _table( prisma_client: "PrismaClient", ) -> TableActions["prisma_models.LiteLLM_UserProviderCredentials"]: - return _UserProviderCredentialsRepository(prisma_client).table + return _UserProviderCredentialsRepository(prisma_client, use_writer=True).table def _cache_key(user_id: str, credential_name: str) -> str: diff --git a/tests/unit/proxy/credential_endpoints/test_user_provider_credentials.py b/tests/unit/proxy/credential_endpoints/test_user_provider_credentials.py index dd8456fc035a..f86db9df0cc4 100644 --- a/tests/unit/proxy/credential_endpoints/test_user_provider_credentials.py +++ b/tests/unit/proxy/credential_endpoints/test_user_provider_credentials.py @@ -23,6 +23,7 @@ def _prisma(table=None): prisma_client = MagicMock() prisma_client.db.litellm_userprovidercredentials = table or MagicMock() + prisma_client.writer_db = prisma_client.db return prisma_client @@ -374,3 +375,22 @@ async def test_set_reports_failure_when_a_stale_entry_survives_both_attempts(): cache = DualCache(redis_cache=silent_redis) payload = GithubCopilotUserConnectionPayload(access_token="gho_secret", github_login="octo") assert not await set_user_provider_credential_cache(cache, "user-a", "copilot-cred", payload) + + +@pytest.mark.asyncio +async def test_reads_hit_the_writer_engine_not_a_stale_replica(): + """With DATABASE_URL_READ_REPLICA set, prisma_client.db is the reader. A row + saved by connect exists only on the writer, so every lookup must route to + writer_db or a fresh connection 401s as not-connected.""" + reader_table = MagicMock() + reader_table.find_many = AsyncMock(return_value=[]) + writer_table = MagicMock() + writer_table.find_many = AsyncMock(return_value=[]) + prisma_client = MagicMock() + prisma_client.db.litellm_userprovidercredentials = reader_table + prisma_client.writer_db.litellm_userprovidercredentials = writer_table + + await aget_user_provider_tokens(prisma_client, DualCache(), "user-a", ["copilot-cred"]) + + writer_table.find_many.assert_awaited_once() + reader_table.find_many.assert_not_awaited() diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 54e4175fb5ca..2ba736d8ae08 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -9074,6 +9074,7 @@ def _env(self, monkeypatch): table = MagicMock() table.find_many = AsyncMock(return_value=[]) prisma_client.db.litellm_userprovidercredentials = table + prisma_client.writer_db = prisma_client.db monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma_client) monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", DualCache()) return table @@ -9904,6 +9905,7 @@ def _per_user_env(monkeypatch, credential_names=("copilot-cred",)): table = MagicMock() table.find_many = AsyncMock(return_value=[]) prisma_client.db.litellm_userprovidercredentials = table + prisma_client.writer_db = prisma_client.db monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma_client) monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", DualCache()) monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "sk-test-master") From 55097b022dd3c4d22528cabc6fe53b7fba0c5eff Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 08:12:09 +0000 Subject: [PATCH 33/38] test(github_copilot): move per-user auth type form test to the integration tier --- ...vider_specific_fields.integration.test.tsx | 70 +++++++++++++++++++ .../provider_specific_fields.test.tsx | 40 ----------- 2 files changed, 70 insertions(+), 40 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/add_model/provider_specific_fields.integration.test.tsx diff --git a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.integration.test.tsx new file mode 100644 index 000000000000..1c1a668a982d --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.integration.test.tsx @@ -0,0 +1,70 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it, vi } from "vitest"; +import { useFormContext } from "react-hook-form"; +import { Providers } from "../provider_info_helpers"; +import { GITHUB_COPILOT_AUTH_TYPE_KEY } from "./provider_auth_types"; +import type { MountedFormValues } from "../common_components/MountedFormField"; +import { MountedFormHost } from "../../../tests/mounted-form-host"; +import ProviderSpecificFields from "./provider_specific_fields"; + +vi.mock("../networking", async () => { + const actual = await vi.importActual("../networking"); + return { + ...actual, + getProviderCreateMetadata: vi.fn().mockResolvedValue([ + { + provider: "GITHUB_COPILOT", + provider_display_name: Providers.GITHUB_COPILOT, + litellm_provider: "github_copilot", + default_model_placeholder: "github_copilot/chat", + credential_fields: [ + { key: "api_base", label: "API Base", field_type: "text" }, + { key: "api_key", label: "API Key", field_type: "password" }, + ], + }, + ]), + }; +}); + +const createQueryClient = () => + new QueryClient({ + defaultOptions: { + queries: { + retry: false, + gcTime: 0, + }, + }, + }); + +const GitHubCopilotAuthTypeProbe = () => { + const { watch } = useFormContext(); + return {String(watch(GITHUB_COPILOT_AUTH_TYPE_KEY) ?? "")}; +}; + +describe("ProviderSpecificFields", () => { + it("offers per-user GitHub OAuth as a credential-only model auth type without mounting its fixed value", async () => { + const onCreateCredential = vi.fn(); + const queryClient = createQueryClient(); + render( + + + + + + , + ); + + expect(await screen.findByRole("combobox", { name: "Auth Type:" })).toHaveTextContent("Per-user GitHub OAuth"); + expect(screen.getByRole("button", { name: "Create credential" })).toBeInTheDocument(); + expect(screen.getByTestId("github-copilot-auth-type")).toBeEmptyDOMElement(); + await userEvent.click(screen.getByRole("button", { name: "Create credential" })); + expect(onCreateCredential).toHaveBeenCalledWith("per_user_oauth"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.test.tsx b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.test.tsx index 641e0e2c1d26..09d48f828e13 100644 --- a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.test.tsx @@ -6,7 +6,6 @@ import { beforeAll, describe, expect, it, vi } from "vitest"; import { useFormContext } from "react-hook-form"; import { chooseSelectOption } from "@/../tests/test-utils"; import { Providers } from "../provider_info_helpers"; -import { GITHUB_COPILOT_AUTH_TYPE_KEY } from "./provider_auth_types"; import type { MountedFormValues } from "../common_components/MountedFormField"; import { MountedFormHost } from "../../../tests/mounted-form-host"; import ProviderSpecificFields from "./provider_specific_fields"; @@ -148,16 +147,6 @@ vi.mock("../networking", async () => { { key: "api_key", label: "Delegated Access Token", field_type: "password" }, ], }, - { - provider: "GITHUB_COPILOT", - provider_display_name: Providers.GITHUB_COPILOT, - litellm_provider: "github_copilot", - default_model_placeholder: "github_copilot/chat", - credential_fields: [ - { key: "api_base", label: "API Base", field_type: "text" }, - { key: "api_key", label: "API Key", field_type: "password" }, - ], - }, ]), }; }); @@ -193,11 +182,6 @@ const VertexCredentialsProbe = () => { return {String(watch("vertex_credentials") ?? "")}; }; -const GitHubCopilotAuthTypeProbe = () => { - const { watch } = useFormContext(); - return {String(watch(GITHUB_COPILOT_AUTH_TYPE_KEY) ?? "")}; -}; - const ValidationForm = ({ children }: { readonly children: ReactNode }) => { const form = useFormContext(); return ( @@ -551,30 +535,6 @@ describe("ProviderSpecificFields", () => { expect(screen.getByLabelText("Delegated Access Token")).toBeInTheDocument(); }); - it("offers per-user GitHub OAuth as a credential-only model auth type without mounting its fixed value", async () => { - const onCreateCredential = vi.fn(); - const queryClient = createQueryClient(); - render( - - - - - - , - ); - - expect(await screen.findByRole("combobox", { name: "Auth Type:" })).toHaveTextContent("Per-user GitHub OAuth"); - expect(screen.getByRole("button", { name: "Create credential" })).toBeInTheDocument(); - expect(screen.getByTestId("github-copilot-auth-type")).toBeEmptyDOMElement(); - await userEvent.click(screen.getByRole("button", { name: "Create credential" })); - expect(onCreateCredential).toHaveBeenCalledWith("per_user_oauth"); - }); - it("switches Microsoft 365 Copilot fields to a static delegated access token", async () => { const user = userEvent.setup(); const queryClient = createQueryClient(); From 55a9346814af1ebdf99d8c06e5ac188500a23ff5 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 08:36:21 +0000 Subject: [PATCH 34/38] test(github_copilot): annotate new credential regression test locals as Final --- .../proxy/credential_endpoints/test_endpoints.py | 15 ++++++++------- .../test_user_provider_credentials.py | 7 ++++--- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/unit/proxy/credential_endpoints/test_endpoints.py b/tests/unit/proxy/credential_endpoints/test_endpoints.py index 7dcb718e0e47..3c4e8b20f724 100644 --- a/tests/unit/proxy/credential_endpoints/test_endpoints.py +++ b/tests/unit/proxy/credential_endpoints/test_endpoints.py @@ -2,6 +2,7 @@ import json from contextlib import contextmanager +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -12,11 +13,11 @@ from fastapi.testclient import TestClient import litellm +from litellm.models.credentials import CredentialSource from litellm.proxy._types import LiteLLM_UserTable, LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.credential_endpoints.endpoints import get_llm_router from litellm.proxy.proxy_server import app -from litellm.models.credentials import CredentialSource from litellm.types.utils import CredentialItem client = TestClient(app) @@ -1868,7 +1869,7 @@ def test_label_only_patch_does_not_purge_user_connections(): """A PATCH that only changes display_name sends no credential_name in the body, and that must not read as a rename that purges every user's stored connection.""" - stored = CredentialItem( + stored: Final = CredentialItem( credential_name="copilot-cred", credential_values={"github_copilot_auth_type": "per_user_oauth"}, credential_info={"custom_llm_provider": "github_copilot"}, @@ -1876,13 +1877,13 @@ def test_label_only_patch_does_not_purge_user_connections(): with _repository_holding(stored): from litellm.proxy import proxy_server - table = MagicMock() + table: Final = MagicMock() table.find_many = AsyncMock(return_value=[_connection_row()]) table.delete_many = AsyncMock(return_value=None) proxy_server.prisma_client.db.litellm_userprovidercredentials = table proxy_server.prisma_client.writer_db = proxy_server.prisma_client.db - response = _patch_credential( + response: Final = _patch_credential( "copilot-cred", {"display_name": "Renamed Copilot", "credential_info": {"custom_llm_provider": "github_copilot"}}, ) @@ -1893,7 +1894,7 @@ def test_label_only_patch_does_not_purge_user_connections(): def test_switching_away_from_per_user_oauth_purges_user_connections(): - stored = CredentialItem( + stored: Final = CredentialItem( credential_name="copilot-cred", credential_values={"github_copilot_auth_type": "per_user_oauth"}, credential_info={"custom_llm_provider": "github_copilot"}, @@ -1901,13 +1902,13 @@ def test_switching_away_from_per_user_oauth_purges_user_connections(): with _repository_holding(stored): from litellm.proxy import proxy_server - table = MagicMock() + table: Final = MagicMock() table.find_many = AsyncMock(return_value=[_connection_row()]) table.delete_many = AsyncMock(return_value=None) proxy_server.prisma_client.db.litellm_userprovidercredentials = table proxy_server.prisma_client.writer_db = proxy_server.prisma_client.db - response = _patch_credential( + response: Final = _patch_credential( "copilot-cred", { "credential_name": "copilot-cred", diff --git a/tests/unit/proxy/credential_endpoints/test_user_provider_credentials.py b/tests/unit/proxy/credential_endpoints/test_user_provider_credentials.py index f86db9df0cc4..b549ecc67a92 100644 --- a/tests/unit/proxy/credential_endpoints/test_user_provider_credentials.py +++ b/tests/unit/proxy/credential_endpoints/test_user_provider_credentials.py @@ -1,6 +1,7 @@ """Tests for the per-user provider credentials DB/cache module.""" from types import SimpleNamespace +from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest @@ -382,11 +383,11 @@ async def test_reads_hit_the_writer_engine_not_a_stale_replica(): """With DATABASE_URL_READ_REPLICA set, prisma_client.db is the reader. A row saved by connect exists only on the writer, so every lookup must route to writer_db or a fresh connection 401s as not-connected.""" - reader_table = MagicMock() + reader_table: Final = MagicMock() reader_table.find_many = AsyncMock(return_value=[]) - writer_table = MagicMock() + writer_table: Final = MagicMock() writer_table.find_many = AsyncMock(return_value=[]) - prisma_client = MagicMock() + prisma_client: Final = MagicMock() prisma_client.db.litellm_userprovidercredentials = reader_table prisma_client.writer_db.litellm_userprovidercredentials = writer_table From 503fa8bd706448ed28cd37fe5595a73a2759f908 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:13:58 +0000 Subject: [PATCH 35/38] fix(github_copilot): type per-user OAuth JSON and DB boundaries --- .../litellm_core_utils/credential_accessor.py | 4 +- .../get_llm_provider_logic.py | 27 ++-- .../pass_through/messages/handler.py | 7 +- .../github_copilot/chat/transformation.py | 13 +- litellm/llms/github_copilot/common_utils.py | 4 +- .../embedding/transformation.py | 19 ++- .../github_copilot/messages/transformation.py | 15 +- litellm/llms/github_copilot/per_user_auth.py | 69 +++++---- .../responses/transformation.py | 25 +++- litellm/main.py | 121 ++++++++-------- litellm/models/credentials.py | 6 +- .../common_utils/credential_hydration.py | 9 +- .../proxy/credential_endpoints/endpoints.py | 20 +-- .../user_provider_credentials.py | 46 +++++- litellm/proxy/litellm_pre_call_utils.py | 131 +++++++++++++----- litellm/proxy/proxy_server.py | 65 ++++----- litellm/responses/main.py | 26 ++-- litellm/router.py | 76 ++++++---- .../clientside_credential_handler.py | 13 +- litellm/utils.py | 40 +++--- 20 files changed, 474 insertions(+), 262 deletions(-) diff --git a/litellm/litellm_core_utils/credential_accessor.py b/litellm/litellm_core_utils/credential_accessor.py index 7b4e8240b69b..95e36a64ef70 100644 --- a/litellm/litellm_core_utils/credential_accessor.py +++ b/litellm/litellm_core_utils/credential_accessor.py @@ -15,7 +15,9 @@ def find_credential(credential_name: str) -> CredentialItem | None: ) @staticmethod - def get_credential_values(credential_name: str) -> dict: + def get_credential_values( + credential_name: str, + ) -> dict[str, object]: # mutable-ok: callers merge the returned credential map """Safe accessor for credentials.""" credential: Final = CredentialAccessor.find_credential(credential_name) diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 256c06b658d7..8f5a90f85bb7 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -240,7 +240,7 @@ def get_llm_provider( model.split("/", 1)[0] in litellm.provider_list and model.split("/", 1)[0] not in litellm.model_list_set and len(model.split("/")) - > 1 # handle edge case where user passes in `litellm --model mistral` https://github.com/BerriAI/litellm/issues/1351 + > 1 # handle edge case where user passes in `litellm --model mistral` https://github.com/BerriAI/litellm/is ): return _get_openai_compatible_provider_info( model=model, @@ -411,7 +411,7 @@ def get_llm_provider( raise Exception(f"dynamic_api_key needs to be a string. dynamic_api_key={dynamic_api_key}") return model, custom_llm_provider, dynamic_api_key, api_base - # check if model in known model provider list -> for huggingface models, raise exception as they don't have a fixed provider (can be togetherai, anyscale, baseten, runpod, et.) + # check if model in known model provider list -> for huggingface models, raise exception as they don't have a f ## openai - chatcompletion + text completion if ( model in litellm.open_ai_chat_completion_models @@ -609,7 +609,7 @@ def _get_openai_compatible_provider_info( return model, custom_llm_provider, dynamic_api_key, api_base if custom_llm_provider == "perplexity": - # perplexity is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.perplexity.ai + # perplexity is openai compatible, we just need to set this to custom_openai and have the api_base be https://ap ( api_base, dynamic_api_key, @@ -617,7 +617,7 @@ def _get_openai_compatible_provider_info( elif custom_llm_provider == "aiohttp_openai": return model, "aiohttp_openai", api_key, api_base elif custom_llm_provider == "anyscale": - # anyscale is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.endpoints.anyscale.com/v1 + # anyscale is openai compatible, we just need to set this to custom_openai and have the api_base be https://api api_base = api_base or get_secret_str("ANYSCALE_API_BASE") or "https://api.endpoints.anyscale.com/v1" dynamic_api_key = api_key or get_secret_str("ANYSCALE_API_KEY") elif custom_llm_provider == "deepinfra": @@ -644,7 +644,7 @@ def _get_openai_compatible_provider_info( ) model = split_mantle_region_prefix(model)[1] # rebind-ok: the prefix is routing only, not a Mantle model id elif custom_llm_provider == "nvidia_nim": - # nvidia_nim is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.endpoints.anyscale.com/v1 + # nvidia_nim is openai compatible, we just need to set this to custom_openai and have the api_base be https://ap api_base = api_base or get_secret("NVIDIA_NIM_API_BASE") or "https://integrate.api.nvidia.com/v1" dynamic_api_key = api_key or get_secret_str("NVIDIA_NIM_API_KEY") elif custom_llm_provider == "nvidia_riva": @@ -692,11 +692,11 @@ def _get_openai_compatible_provider_info( dynamic_api_key = api_key or get_secret_str("AI21_API_KEY") custom_llm_provider = "ai21_chat" elif custom_llm_provider == "volcengine": - # volcengine is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.endpoints.anyscale.com/v1 + # volcengine is openai compatible, we just need to set this to custom_openai and have the api_base be https://ap api_base = api_base or get_secret("VOLCENGINE_API_BASE") or "https://ark.cn-beijing.volces.com/api/v3" dynamic_api_key = api_key or get_secret_str("VOLCENGINE_API_KEY") elif custom_llm_provider == "codestral": - # codestral is openai compatible, we just need to set this to custom_openai and have the api_base be https://codestral.mistral.ai/v1 + # codestral is openai compatible, we just need to set this to custom_openai and have the api_base be https://cod api_base = api_base or get_secret("CODESTRAL_API_BASE") or "https://codestral.mistral.ai/v1" dynamic_api_key = api_key or get_secret_str("CODESTRAL_API_KEY") elif custom_llm_provider == "hosted_vllm": @@ -724,7 +724,7 @@ def _get_openai_compatible_provider_info( dynamic_api_key, ) = litellm.LMStudioChatConfig().get_openai_compatible_provider_info(api_base, api_key) elif custom_llm_provider == "deepseek": - # deepseek is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.deepseek.com/v1 + # deepseek is openai compatible, we just need to set this to custom_openai and have the api_base be https://api api_base = api_base or get_secret("DEEPSEEK_API_BASE") or "https://api.deepseek.com/beta" dynamic_api_key = api_key or get_secret_str("DEEPSEEK_API_KEY") @@ -733,7 +733,7 @@ def _get_openai_compatible_provider_info( dynamic_api_key = api_key or get_secret_str("TENCENT_API_KEY") elif custom_llm_provider == "fireworks_ai": - # fireworks is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.fireworks.ai/inference/v1 + # fireworks is openai compatible, we just need to set this to custom_openai and have the api_base be https://api ( api_base, dynamic_api_key, @@ -801,11 +801,16 @@ def _get_openai_compatible_provider_info( github_copilot_user_session_from, ) - user_session: Final = github_copilot_user_session_from(litellm_params) # pyright: ignore[reportUnknownArgumentType] # litellm_params arrives as an untyped request dict + user_session: Final = github_copilot_user_session_from( + cast("dict[str, object]", litellm_params) # cast-ok: litellm_params arrives as an untyped request dict + ) if user_session is not None: api_base = user_session.api_base # rebind-ok: resolves provider args in place dynamic_api_key = user_session.token # rebind-ok: resolves provider args in place - elif github_copilot_per_user_credential_name(litellm_params) is not None: # pyright: ignore[reportUnknownArgumentType] # litellm_params arrives as an untyped request dict + elif ( + github_copilot_per_user_credential_name(cast("dict[str, object]", litellm_params)) # cast-ok: litellm_param + is not None + ): # per-user deployments resolve no token at provider-info time; the # caller's session supplies base + key per request api_base = api_base or DEFAULT_GITHUB_COPILOT_API_BASE # rebind-ok: resolves provider args in place diff --git a/litellm/llms/anthropic/pass_through/messages/handler.py b/litellm/llms/anthropic/pass_through/messages/handler.py index ec5e2a7646a1..47744d6e93b4 100644 --- a/litellm/llms/anthropic/pass_through/messages/handler.py +++ b/litellm/llms/anthropic/pass_through/messages/handler.py @@ -7,7 +7,7 @@ import asyncio import contextvars -from collections.abc import AsyncIterator, Coroutine, Iterator +from collections.abc import AsyncIterator, Coroutine, Iterator, Sequence from functools import partial from typing import Any, Final, cast @@ -630,7 +630,10 @@ def anthropic_messages_handler( return base_llm_http_handler.anthropic_messages_handler( model=model, messages=cast( # cast-ok: anthropic messages are dicts at runtime - "list[dict]", strip_provider_specific_fields_from_anthropic_messages(messages) + "list[dict[str, object]]", + strip_provider_specific_fields_from_anthropic_messages( + cast(Sequence[object], messages) # cast-ok: anthropic payloads arrive as untyped dicts + ), ), anthropic_messages_provider_config=anthropic_messages_provider_config, anthropic_messages_optional_request_params=dict(anthropic_messages_optional_request_params), diff --git a/litellm/llms/github_copilot/chat/transformation.py b/litellm/llms/github_copilot/chat/transformation.py index 5930c6c94049..9553327db970 100644 --- a/litellm/llms/github_copilot/chat/transformation.py +++ b/litellm/llms/github_copilot/chat/transformation.py @@ -87,7 +87,9 @@ def _transform_messages( for message in messages: if message.get("role") == "system": # Convert system message to assistant message - transformed_messages.append(cast(AllMessageValues, {**message, "role": "assistant"})) + transformed_messages.append( + cast(AllMessageValues, {**message, "role": "assistant"}) # cast-ok: dict matches the union + ) else: transformed_messages.append(message) @@ -111,10 +113,13 @@ def validate_environment( api_key: str | None = None, api_base: str | None = None, ) -> dict: - parent_headers: Final = super().validate_environment( - headers, model, messages, optional_params, litellm_params, api_key, api_base + parent_headers: Final = cast( # cast-ok: the parent validate_environment returns a plain dict at runtime + "dict[str, str]", + super().validate_environment(headers, model, messages, optional_params, litellm_params, api_key, api_base), + ) + user_session: Final = require_github_copilot_user_session( + cast("dict[str, object]", litellm_params) # cast-ok: litellm_params arrives as an untyped request dict ) - user_session: Final = require_github_copilot_user_session(litellm_params) # pyright: ignore[reportUnknownArgumentType] # litellm_params arrives as an untyped request dict session_token: Final = user_session.token if user_session is not None else None validated_headers: Final = {**self._copilot_headers(session_token), **parent_headers} if session_token is not None: diff --git a/litellm/llms/github_copilot/common_utils.py b/litellm/llms/github_copilot/common_utils.py index fcd5a4cacadc..515b7bf4fb5b 100644 --- a/litellm/llms/github_copilot/common_utils.py +++ b/litellm/llms/github_copilot/common_utils.py @@ -58,7 +58,9 @@ class GetAPIKeyError(GithubCopilotError): pass -def get_copilot_default_headers(api_key: str) -> dict: +def get_copilot_default_headers( + api_key: str, +) -> dict[str, str]: # mutable-ok: callers merge caller headers into the returned dict """ Get default headers for GitHub Copilot Responses API. diff --git a/litellm/llms/github_copilot/embedding/transformation.py b/litellm/llms/github_copilot/embedding/transformation.py index 225d32bda388..a6250afdd038 100644 --- a/litellm/llms/github_copilot/embedding/transformation.py +++ b/litellm/llms/github_copilot/embedding/transformation.py @@ -8,7 +8,12 @@ """ import os -from typing import TYPE_CHECKING, Any, Final +from typing import ( + TYPE_CHECKING, + Any, + Final, + cast, # noqa: TID251 # narrows untyped request dicts at the session boundary +) import httpx from pydantic import ConfigDict, TypeAdapter @@ -63,9 +68,13 @@ def validate_environment( """ Validate environment and set up headers for GitHub Copilot API. """ - user_session: Final = require_github_copilot_user_session(litellm_params) # pyright: ignore[reportUnknownArgumentType] # litellm_params arrives as an untyped request dict + user_session: Final = require_github_copilot_user_session( + cast("dict[str, object]", litellm_params) # cast-ok: litellm_params arrives as an untyped request dict + ) if user_session is not None: - session_headers: Final = {**get_copilot_default_headers(user_session.token), **headers} + session_headers: Final = cast( # cast-ok: both spreads are str-valued header dicts + "dict[str, str]", {**get_copilot_default_headers(user_session.token), **headers} + ) pin_session_authorization(session_headers, user_session.token) return session_headers try: @@ -108,7 +117,9 @@ def get_complete_url( """ Get the complete URL for GitHub Copilot Embedding API endpoint. """ - user_session: Final = require_github_copilot_user_session(litellm_params) # pyright: ignore[reportUnknownArgumentType] # litellm_params arrives as an untyped request dict + user_session: Final = require_github_copilot_user_session( + cast("dict[str, object]", litellm_params) # cast-ok: litellm_params arrives as an untyped request dict + ) effective_api_base = ( (user_session.api_base if user_session is not None else None) or api_base diff --git a/litellm/llms/github_copilot/messages/transformation.py b/litellm/llms/github_copilot/messages/transformation.py index 96e141c9f50c..be138755f854 100644 --- a/litellm/llms/github_copilot/messages/transformation.py +++ b/litellm/llms/github_copilot/messages/transformation.py @@ -1,4 +1,4 @@ -from typing import Any, Final +from typing import Any, Final, cast # noqa: TID251 # narrows untyped request dicts at the session boundary from litellm.exceptions import AuthenticationError from litellm.llms.anthropic.pass_through.messages.transformation import ( @@ -70,7 +70,9 @@ def validate_anthropic_messages_environment( # session, never the caller-supplied api_base. rstrip so a # tenant-specific base with a trailing slash does not yield a # double-slash URL once "/v1/messages" is appended downstream. - user_session: Final = require_github_copilot_user_session(litellm_params) # pyright: ignore[reportUnknownArgumentType] # litellm_params arrives as an untyped request dict + user_session: Final = require_github_copilot_user_session( + cast("dict[str, object]", litellm_params) # cast-ok: litellm_params arrives as an untyped request dict + ) dynamic_api_base: Final = ( user_session.api_base if user_session is not None @@ -93,7 +95,10 @@ def validate_anthropic_messages_environment( if key not in headers: headers[key] = value if user_session is not None: - pin_session_authorization(headers, user_session.token) + pin_session_authorization( + cast("dict[str, str]", headers), # cast-ok: headers is a str-valued request dict at runtime + user_session.token, + ) headers["openai-intent"] = "messages-proxy" headers["x-interaction-type"] = "messages-proxy" @@ -127,7 +132,9 @@ def get_complete_url( reuse it to avoid a second authenticator read, falling back to a fresh resolution only if it was not provided. """ - user_session: Final = require_github_copilot_user_session(litellm_params) + user_session: Final = require_github_copilot_user_session( + cast("dict[str, object]", litellm_params) # cast-ok: litellm_params arrives as an untyped request dict + ) resolved: Final = ( (user_session.api_base if user_session is not None else None) or api_base diff --git a/litellm/llms/github_copilot/per_user_auth.py b/litellm/llms/github_copilot/per_user_auth.py index 8f09c58f7db4..8cd09f6fa8f3 100644 --- a/litellm/llms/github_copilot/per_user_auth.py +++ b/litellm/llms/github_copilot/per_user_auth.py @@ -9,7 +9,7 @@ import os import threading import time -from collections.abc import Mapping +from collections.abc import Callable, Mapping from dataclasses import dataclass, field from typing import ( Final, @@ -323,12 +323,17 @@ def _do_exchange(github_token: str, credential_name: str, cache_key: str) -> Git return _session_from_response(response, credential_name, cache_key) -async def _do_aexchange(github_token: str, credential_name: str, cache_key: str) -> GithubCopilotUserSession: - from litellm.llms.custom_httpx.http_handler import get_async_httpx_client +def _new_async_github_client() -> _AsyncGitHubClient: + from litellm.llms.custom_httpx import http_handler - client: Final = cast( # cast-ok: pins the untyped litellm client to the local Protocol - _AsyncGitHubClient, get_async_httpx_client(_LLM_PROVIDER) - ) + return cast( # cast-ok: pins the untyped litellm client factory to the local Protocol + Callable[[str], _AsyncGitHubClient], + getattr(http_handler, "get_async_httpx_client"), # noqa: B009 # getattr keeps the untyped factory out of membe + )(_LLM_PROVIDER) + + +async def _do_aexchange(github_token: str, credential_name: str, cache_key: str) -> GithubCopilotUserSession: + client: Final = _new_async_github_client() try: response: Final = await client.get( DEFAULT_GITHUB_API_KEY_URL, @@ -392,11 +397,7 @@ async def aexchange_github_token( async def astart_device_flow() -> GithubCopilotDeviceFlowStart: - from litellm.llms.custom_httpx.http_handler import get_async_httpx_client - - client: Final = cast( # cast-ok: pins the untyped litellm client to the local Protocol - _AsyncGitHubClient, get_async_httpx_client(_LLM_PROVIDER) - ) + client: Final = _new_async_github_client() device_code_url: Final = os.getenv("GITHUB_COPILOT_DEVICE_CODE_URL", DEFAULT_GITHUB_DEVICE_CODE_URL) client_id: Final = os.getenv("GITHUB_COPILOT_CLIENT_ID", DEFAULT_GITHUB_CLIENT_ID) try: @@ -435,11 +436,7 @@ async def astart_device_flow() -> GithubCopilotDeviceFlowStart: async def apoll_device_flow(device_code: str) -> GithubCopilotDeviceFlowPoll: - from litellm.llms.custom_httpx.http_handler import get_async_httpx_client - - client: Final = cast( # cast-ok: pins the untyped litellm client to the local Protocol - _AsyncGitHubClient, get_async_httpx_client(_LLM_PROVIDER) - ) + client: Final = _new_async_github_client() access_token_url: Final = os.getenv("GITHUB_COPILOT_ACCESS_TOKEN_URL", DEFAULT_GITHUB_ACCESS_TOKEN_URL) client_id: Final = os.getenv("GITHUB_COPILOT_CLIENT_ID", DEFAULT_GITHUB_CLIENT_ID) try: @@ -504,11 +501,7 @@ def _poll_payload_from_response(response: httpx.Response) -> GithubCopilotDevice async def afetch_github_login(github_token: str) -> str: - from litellm.llms.custom_httpx.http_handler import get_async_httpx_client - - client: Final = cast( # cast-ok: pins the untyped litellm client to the local Protocol - _AsyncGitHubClient, get_async_httpx_client(_LLM_PROVIDER) - ) + client: Final = _new_async_github_client() try: response: Final = await client.get( "https://api.github.com/user", @@ -653,10 +646,18 @@ def is_github_copilot_per_user_request(kwargs: Mapping[str, object]) -> bool: return isinstance(kwargs.get(GITHUB_COPILOT_USER_SESSION_KWARG_KEY), GithubCopilotUserSession) -def _without_authorization(headers: Mapping[str, str]) -> Mapping[str, str]: +def _without_authorization(headers: Mapping[str, object]) -> dict[str, object]: return {k: v for k, v in headers.items() if k.lower() != "authorization"} +def _str_keyed_mapping(value: object) -> Mapping[str, object] | None: + return ( + cast("Mapping[str, object]", value) # cast-ok: request mappings are str-keyed + if isinstance(value, Mapping) + else None + ) + + def _strip_caller_authorization( kwargs: dict[str, object], # mutable-ok: kwargs is the request mutation channel ) -> None: @@ -667,21 +668,33 @@ def _strip_caller_authorization( for key in ("extra_headers", "headers"): value = kwargs.get(key) if isinstance(value, Mapping): - stripped = _without_authorization(value) - if len(stripped) != len(value): + stripped = _without_authorization( + cast("Mapping[str, object]", value) # cast-ok: Mapping-checked request header dict + ) + if len(stripped) != len( + cast("Mapping[object, object]", value) # cast-ok: Mapping-checked request header dict + ): kwargs[key] = stripped # rebind-ok: kwargs is the request mutation channel optional_params: Final = kwargs.get("optional_params") if isinstance(optional_params, dict): - eh: Final = optional_params.get("extra_headers") - if isinstance(eh, Mapping): + eh: Final = _str_keyed_mapping( + cast("dict[str, object]", optional_params).get( # cast-ok: optional_params is a plain str-keyed dict + "extra_headers" + ) + ) + if eh is not None: kwargs["optional_params"] = { # rebind-ok: kwargs is the request mutation channel **optional_params, "extra_headers": _without_authorization(eh), } litellm_params: Final = kwargs.get("litellm_params") if isinstance(litellm_params, dict): - eh2: Final = litellm_params.get("extra_headers") - if isinstance(eh2, Mapping): + eh2: Final = _str_keyed_mapping( + cast("dict[str, object]", litellm_params).get( # cast-ok: litellm_params is a plain str-keyed dict + "extra_headers" + ) + ) + if eh2 is not None: kwargs["litellm_params"] = { # rebind-ok: kwargs is the request mutation channel **litellm_params, "extra_headers": _without_authorization(eh2), diff --git a/litellm/llms/github_copilot/responses/transformation.py b/litellm/llms/github_copilot/responses/transformation.py index 0cb91ced5138..471a8b292468 100644 --- a/litellm/llms/github_copilot/responses/transformation.py +++ b/litellm/llms/github_copilot/responses/transformation.py @@ -9,7 +9,12 @@ """ import os -from typing import TYPE_CHECKING, Any, Final +from typing import ( + TYPE_CHECKING, + Any, + Final, + cast, # noqa: TID251 # narrows untyped request dicts at the session boundary +) import litellm from litellm._logging import verbose_logger @@ -201,7 +206,9 @@ def validate_environment( - copilot-vision-request if vision content detected - User-provided extra_headers (merged with priority) """ - user_session: Final = require_github_copilot_user_session(litellm_params) # pyright: ignore[reportUnknownArgumentType] # litellm_params arrives as an untyped request dict + user_session: Final = require_github_copilot_user_session( + cast("dict[str, object]", litellm_params) # cast-ok: litellm_params arrives as an untyped request dict + ) default_headers: Final = get_copilot_default_headers(user_session.token) if user_session is not None else None try: # Get GitHub Copilot API key via OAuth @@ -217,7 +224,9 @@ def validate_environment( copilot_headers: Final = default_headers or get_copilot_default_headers(api_key) # Merge with existing headers (user's extra_headers take priority) - merged_headers: Final = {**copilot_headers, **headers} + merged_headers: Final = cast( # cast-ok: both spreads are str-valued header dicts + "dict[str, str]", {**copilot_headers, **headers} + ) if user_session is not None: pin_session_authorization(merged_headers, user_session.token) @@ -254,7 +263,9 @@ def get_complete_url( """ Get the complete URL for GitHub Copilot Responses API endpoint. """ - user_session: Final = require_github_copilot_user_session(litellm_params) # pyright: ignore[reportUnknownArgumentType] # litellm_params arrives as an untyped request dict + user_session: Final = require_github_copilot_user_session( + cast("dict[str, object]", litellm_params) # cast-ok: litellm_params arrives as an untyped request dict + ) effective_api_base = ( (user_session.api_base if user_session is not None else None) or api_base @@ -347,9 +358,9 @@ def _get_initiator(self, input_param: str | ResponseInputParam) -> str: return "user" # If input is a list, analyze items - if isinstance(input_param, list): # pyright: ignore[reportUnnecessaryIsInstance] # items arrive from untyped request params + if isinstance(input_param, list): # pyright: ignore[reportUnnecessaryIsInstance] # items arrive from untyped r for item in input_param: - if not isinstance(item, dict): # pyright: ignore[reportUnnecessaryIsInstance] # items arrive from untyped request params + if not isinstance(item, dict): # pyright: ignore[reportUnnecessaryIsInstance] # items arrive from unty continue # Check if item has no role (agent-initiated) @@ -358,7 +369,7 @@ def _get_initiator(self, input_param: str | ResponseInputParam) -> str: # Check if role is assistant (agent-initiated) role = item.get("role") - if isinstance(role, str) and role.lower() == "assistant": # pyright: ignore[reportUnnecessaryIsInstance] # role arrives from untyped request params + if isinstance(role, str) and role.lower() == "assistant": # pyright: ignore[reportUnnecessaryIsInstance] # role return "agent" # Default to user-initiated diff --git a/litellm/main.py b/litellm/main.py index f24243b459c6..93795f7e0be2 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -642,11 +642,16 @@ async def acompletion( "enable_json_schema_validation": enable_json_schema_validation, } if custom_llm_provider is None: - _supplemental: Final = {k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS if k in kwargs} + _supplemental: Final = cast( # cast-ok: kwargs values are request-scoped objects + "dict[str, object]", {k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS if k in kwargs} + ) _, custom_llm_provider, _, _ = get_llm_provider( model=model, custom_llm_provider=custom_llm_provider, - api_base=kwargs.get("api_base") or base_url, + api_base=cast( # cast-ok: api_base arrives as a str in the request kwargs + "str | None", kwargs.get("api_base") + ) + or base_url, litellm_params=(GenericLiteLLMParams.model_validate(_supplemental) if _supplemental else None), ) @@ -1476,7 +1481,7 @@ def _complete_azure(ctx: CompletionDispatchContext) -> _CompletionDispatchResult }, ) - return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contra def _complete_azure_text(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: @@ -1857,7 +1862,7 @@ def _complete_text_completion_openai( original_response=_response, additional_args={"headers": headers}, ) - return _response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + return _response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contr def _complete_fireworks_ai( @@ -2136,7 +2141,7 @@ def _complete_groq(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: timeout: Final = ctx.timeout api_base = ( - api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there + api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api b or litellm.api_base or get_secret("GROQ_API_BASE") or "https://api.groq.com/openai/v1" @@ -2145,7 +2150,7 @@ def _complete_groq(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: # set API KEY api_key = ( api_key - or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there + or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the or litellm.groq_key or get_secret("GROQ_API_KEY") ) @@ -2175,7 +2180,7 @@ def _complete_groq(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: headers=headers, encoding=_get_encoding(), api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alph client=client, ) @@ -2421,7 +2426,7 @@ def _complete_aiohttp_openai( timeout: Final = ctx.timeout api_base = ( - api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there + api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api b or litellm.api_base or get_secret("OPENAI_BASE_URL") or get_secret("OPENAI_API_BASE") @@ -2430,7 +2435,7 @@ def _complete_aiohttp_openai( # set API KEY api_key = ( api_key - or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there + or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the or litellm.openai_key or get_secret("OPENAI_API_KEY") ) @@ -2628,13 +2633,13 @@ def _complete_custom_openai( organization or litellm.organization or get_secret("OPENAI_ORGANIZATION") - or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 + or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/opena ) openai.organization = organization # set API KEY api_key = ( api_key - or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there + or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the or litellm.openai_key or get_secret("OPENAI_API_KEY") ) @@ -2657,12 +2662,14 @@ def _complete_custom_openai( require_github_copilot_user_session, ) - user_session: Final = require_github_copilot_user_session(litellm_params) # pyright: ignore[reportUnknownArgumentType] # litellm_params arrives as an untyped request dict + user_session: Final = require_github_copilot_user_session(litellm_params) copilot_headers: Final = get_copilot_default_headers( user_session.token if user_session is not None else Authenticator().get_api_key() ) if extra_headers: - copilot_headers.update(extra_headers) + copilot_headers.update( + cast("dict[str, str]", extra_headers) # cast-ok: extra_headers is a str-valued request dict + ) if user_session is not None: pin_session_authorization(copilot_headers, user_session.token) extra_headers = copilot_headers @@ -2747,7 +2754,7 @@ def _complete_custom_openai( additional_args={"headers": outbound_headers}, ) - return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contra def _complete_mistral(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: @@ -2897,7 +2904,7 @@ def _complete_anthropic_text( headers=headers, encoding=_get_encoding(), api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alph ) @@ -3014,7 +3021,7 @@ def _complete_nlp_cloud(ctx: CompletionDispatchContext) -> _CompletionDispatchRe original_response=response, ) - return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contra def _complete_aleph_alpha(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: @@ -3052,7 +3059,7 @@ def _complete_aleph_alpha(ctx: CompletionDispatchContext) -> _CompletionDispatch encoding=_get_encoding(), default_max_tokens_to_sample=litellm.max_tokens, api_key=aleph_alpha_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alph ) if "stream" in optional_params and optional_params["stream"] is True: @@ -3063,7 +3070,7 @@ def _complete_aleph_alpha(ctx: CompletionDispatchContext) -> _CompletionDispatch custom_llm_provider="aleph_alpha", logging_obj=logging, ) - return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch def _complete_cohere_chat(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: @@ -3127,7 +3134,7 @@ def _complete_cohere_chat(ctx: CompletionDispatchContext) -> _CompletionDispatch encoding=_get_encoding(), api_key=cohere_key, provider_config=provider_config, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alph ) @@ -3351,7 +3358,7 @@ def _complete_oobabooga(ctx: CompletionDispatchContext) -> _CompletionDispatchRe custom_llm_provider="oobabooga", logging_obj=logging, ) - return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch def _complete_databricks(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: @@ -3401,7 +3408,7 @@ def _complete_databricks(ctx: CompletionDispatchContext) -> _CompletionDispatchR headers=headers, encoding=_get_encoding(), api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph client=client, ) except Exception as e: @@ -3531,7 +3538,7 @@ def _complete_openrouter(ctx: CompletionDispatchContext) -> _CompletionDispatchR headers=headers, encoding=_get_encoding(), api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alph client=client, ) ## LOGGING @@ -3636,7 +3643,7 @@ def _complete_vercel_ai_gateway( headers=headers, encoding=_get_encoding(), api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alph client=client, ) ## LOGGING @@ -3952,7 +3959,7 @@ def _complete_vertex_ai(ctx: CompletionDispatchContext) -> _CompletionDispatchRe custom_llm_provider="vertex_ai", logging_obj=logging, ) - return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch def _complete_predibase(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: @@ -4060,8 +4067,8 @@ def _complete_text_completion_codestral( ) if "stream" in optional_params and optional_params["stream"] is True and acompletion is False: - return _model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract - return _model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + return _model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the disp + return _model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch def _complete_text_completion_inception( @@ -4125,7 +4132,7 @@ def _complete_text_completion_inception( original_response=_response, additional_args={"headers": headers}, ) - return _response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + return _response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contr def _complete_sagemaker_chat( @@ -4160,7 +4167,7 @@ def _complete_sagemaker_chat( headers=headers, encoding=_get_encoding(), api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alph client=client, ) @@ -4301,7 +4308,7 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu headers=headers, encoding=_get_encoding(), api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph client=client, ) else: @@ -4435,7 +4442,7 @@ def _complete_watsonx_text( headers=headers, encoding=_get_encoding(), api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alph client=client, ) @@ -4512,7 +4519,7 @@ def _complete_ollama(ctx: CompletionDispatchContext) -> _CompletionDispatchResul headers=headers, encoding=_get_encoding(), api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alph client=client, ) @@ -4554,7 +4561,7 @@ def _complete_ollama_chat(ctx: CompletionDispatchContext) -> _CompletionDispatch headers=headers, encoding=_get_encoding(), api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alph client=client, ) @@ -4630,7 +4637,7 @@ def _complete_cloudflare(ctx: CompletionDispatchContext) -> _CompletionDispatchR headers=headers, encoding=_get_encoding(), api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alph ) @@ -5061,7 +5068,7 @@ def _complete_custom_providers( logging_obj=logging, ) - return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contra def _complete_langgraph(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: @@ -5347,7 +5354,7 @@ def completion( # Cast tools to Optional[Iterable[ToolParam]] for type checking tools_for_mcp: Final = cast(Iterable[ToolParam] | None, tools) if LiteLLM_Proxy_MCP_Handler.should_use_litellm_mcp_gateway(tools=tools_for_mcp): - return acompletion_with_mcp( # pyright: ignore[reportReturnType] # MCP path returns a coroutine that acompletion() awaits; completion()'s sync return type omits it + return acompletion_with_mcp( # pyright: ignore[reportReturnType] # MCP path returns a coroutine that acomp model=model, messages=messages, functions=functions, @@ -5532,12 +5539,12 @@ def completion( logging: Final[LiteLLMLoggingObj] = cast(LiteLLMLoggingObj, litellm_logging_obj) fallbacks = fallbacks or litellm.model_fallbacks if fallbacks is not None: - return completion_with_fallbacks( # pyright: ignore[reportReturnType] # fallback runner is untyped; resolves to ModelResponse|CustomStreamWrapper at runtime + return completion_with_fallbacks( # pyright: ignore[reportReturnType] # fallback runner is untyped; resolv **args ) if model_list is not None: deployments: Final = [m["litellm_params"] for m in model_list if m["model_name"] == model] - return litellm.batch_completion_models( # pyright: ignore[reportReturnType] # batch path returns a list of responses, outside completion()'s single-response return type + return litellm.batch_completion_models( # pyright: ignore[reportReturnType] # batch path returns a list of deployments=deployments, **args ) if litellm.model_alias_map and model in litellm.model_alias_map: @@ -5829,7 +5836,7 @@ def completion( logging.update_environment_variables( model=model, user=user, - optional_params=processed_non_default_params, # [IMPORTANT] - using processed_non_default_params ensures consistent params logged to langfuse for finetuning / eval datasets. + optional_params=processed_non_default_params, # [IMPORTANT] - using processed_non_default_params ensures co litellm_params=litellm_params, custom_llm_provider=custom_llm_provider, ) @@ -5894,7 +5901,7 @@ def completion( else: optional_params["reasoning_effort"] = {"summary": rs_val} - return responses_api_bridge.completion( # pyright: ignore[reportReturnType] # bridge returns a coroutine on the acompletion path; awaited by the async caller + return responses_api_bridge.completion( # pyright: ignore[reportReturnType] # bridge returns a coroutine o model=model, messages=messages, headers=headers, @@ -6047,7 +6054,7 @@ def completion( elif custom_llm_provider == "mistral": response = _complete_mistral(_dispatch_ctx) elif "replicate" in model or custom_llm_provider == "replicate" or model in litellm.replicate_models: - # Setting the relevant API KEY for replicate, replicate defaults to using os.environ.get("REPLICATE_API_TOKEN") + # Setting the relevant API KEY for replicate, replicate defaults to using os.environ.get("REPLICATE_API_TOKE response = _complete_replicate(_dispatch_ctx) elif "clarifai" in model or custom_llm_provider == "clarifai" or model in litellm.clarifai_models: pass # Deprecated - handled in the openai compatible provider section above @@ -6507,7 +6514,11 @@ def embedding( if dynamic_api_key is not None: api_key = dynamic_api_key - allowed_openai_params: Final[list[str] | None] = kwargs.get("allowed_openai_params", None) + allowed_openai_params: Final[list[str] | None] = ( + cast( # cast-ok: allowed_openai_params arrives as a str list in the request kwargs + "list[str] | None", kwargs.get("allowed_openai_params", None) + ) + ) optional_params: Final = get_optional_params_embeddings( model=model, user=user, @@ -6617,7 +6628,7 @@ def embedding( openai.organization = ( litellm.organization or get_secret_str("OPENAI_ORGANIZATION") - or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 + or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088d ) # set API KEY api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY") @@ -7230,14 +7241,14 @@ def embedding( from litellm.llms.azure_ai.common_utils import get_azure_ai_entra_token api_base = ( - api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there + api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in t or litellm.api_base or get_secret_str("AZURE_AI_API_BASE") ) # set API KEY api_key = ( api_key - or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there + or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass or get_secret_str("AZURE_AI_API_KEY") or get_azure_ai_entra_token(litellm_params=litellm_params_dict) ) @@ -7628,7 +7639,7 @@ def text_completion( text_completion_response = TextCompletionResponse() optional_params: Final[dict[str, Any]] = {} - # default values for all optional params are none, litellm only passes them to the llm when they are set to non None values + # default values for all optional params are none, litellm only passes them to the llm when they are set to non None if best_of is not None: optional_params["best_of"] = best_of if echo is not None: @@ -7754,9 +7765,9 @@ def process_prompt(i, individual_prompt): kwargs.pop("prompt", None) - if ( - _model is not None and (custom_llm_provider == "openai") - ): # for openai compatible endpoints - e.g. vllm, call the native /v1/completions endpoint for text completion calls + if _model is not None and ( + custom_llm_provider == "openai" + ): # for openai compatible endpoints - e.g. vllm, call the native /v1/completions endpoint for text completion call if _model not in litellm.open_ai_chat_completion_models: model = "text-completion-openai/" + _model optional_params.pop("custom_llm_provider", None) @@ -7921,7 +7932,7 @@ async def amoderation( api_key=optional_params.api_key, ) except litellm.BadRequestError: - # `model` is optional field for moderation - get_llm_provider will throw BadRequestError if model is not set / not recognized + # `model` is optional field for moderation - get_llm_provider will throw BadRequestError if model is not set / n pass openai_client: Final = kwargs.get("client", None) @@ -8171,7 +8182,7 @@ def transcription( openai.organization = ( litellm.organization or get_secret("OPENAI_ORGANIZATION") - or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 + or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/o ) # set API KEY @@ -8450,7 +8461,7 @@ def speech( llm_provider=custom_llm_provider, ) api_base = ( - api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there + api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the a or litellm.api_base or get_secret("OPENAI_BASE_URL") or get_secret("OPENAI_API_BASE") @@ -8459,7 +8470,7 @@ def speech( # set API KEY api_key = ( api_key - or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there + or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key or litellm.openai_key or get_secret("OPENAI_API_KEY") ) @@ -8468,14 +8479,14 @@ def speech( organization or litellm.organization or get_secret("OPENAI_ORGANIZATION") - or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 + or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/o ) project = ( project or litellm.project or get_secret("OPENAI_PROJECT") - or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 + or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/o ) headers = headers or litellm.headers @@ -8981,7 +8992,7 @@ def stream_chunk_builder_text_completion(chunks: list, messages: Sequence | None completion_tokens: Final = token_counter( model=model, text=combined_content, - count_response_tokens=True, # count_response_tokens is a Flag to tell token counter this is a response, No need to add extra tokens we do for input messages + count_response_tokens=True, # count_response_tokens is a Flag to tell token counter this is a response, No need ) response: Final = { diff --git a/litellm/models/credentials.py b/litellm/models/credentials.py index 025388cbc6ce..b9f4b9a9b6ae 100644 --- a/litellm/models/credentials.py +++ b/litellm/models/credentials.py @@ -22,7 +22,11 @@ class CredentialBase(LiteLLMBaseModel): class CredentialItem(CredentialBase): - credential_values: dict + credential_values: ( + dict[ # mutable-ok: credential values are decrypted and merged in place by the credential endpoints + str, object + ] + ) # PATCH-only instruction naming keys to drop from the stored credential_values. It describes an # edit rather than the credential, so it stays out of dumps: those feed config loading, the DB # write, and the in-memory list, none of which have a place for it. diff --git a/litellm/proxy/common_utils/credential_hydration.py b/litellm/proxy/common_utils/credential_hydration.py index 2a66a3ec2c43..4f22e921022b 100644 --- a/litellm/proxy/common_utils/credential_hydration.py +++ b/litellm/proxy/common_utils/credential_hydration.py @@ -9,7 +9,7 @@ from collections.abc import Mapping from itertools import chain from types import MappingProxyType -from typing import Final +from typing import Final, cast # noqa: TID251 # narrows untyped credential values at the decrypt boundary import litellm from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper @@ -71,11 +71,14 @@ def decrypted_or_stored(key: str, value: str) -> str: def _decrypted(db_credential: CredentialItem) -> CredentialItem: """The stored credential with every value decrypted, leaving already-plaintext values alone.""" decrypted_values: Final = MappingProxyType( - {key: decrypted_or_stored(key, value) for key, value in db_credential.credential_values.items()} + { + key: decrypted_or_stored(key, cast("str", value)) # cast-ok: credential values are str at the decrypt bound + for key, value in db_credential.credential_values.items() + } ) return CredentialItem( credential_name=db_credential.credential_name, - credential_values=decrypted_values, # pyright: ignore[reportArgumentType] # declared dict[str, str], and pydantic copies this mapping into one on validation; LIT002 rules out building that dict here + credential_values=decrypted_values, # pyright: ignore[reportArgumentType] # declared dict[str, str], and pydan credential_info=db_credential.credential_info, ) diff --git a/litellm/proxy/credential_endpoints/endpoints.py b/litellm/proxy/credential_endpoints/endpoints.py index eff48a533b66..4cff0d4eab72 100644 --- a/litellm/proxy/credential_endpoints/endpoints.py +++ b/litellm/proxy/credential_endpoints/endpoints.py @@ -4,6 +4,7 @@ import time from collections.abc import Mapping +from datetime import datetime from typing import ( Annotated, Final, @@ -172,9 +173,12 @@ class CredentialHelperUtils: @staticmethod def encrypt_credential_values(credential: CredentialItem, new_encryption_key: str | None = None) -> CredentialItem: """Encrypt values in credential.credential_values and add to DB""" - encrypted_credential_values: Final = {} + encrypted_credential_values: Final[dict[str, object]] = {} # mutable-ok: built one entry per credential value for key, value in (credential.credential_values or {}).items(): - encrypted_credential_values[key] = encrypt_value_helper(value, new_encryption_key) + encrypted_credential_values[key] = encrypt_value_helper( + cast("str", value), # cast-ok: credential values are str at the encryption boundary + new_encryption_key, + ) # Return a new object to avoid mutating the caller's credential, which # is kept in memory and should remain unencrypted. @@ -390,7 +394,7 @@ async def get_credential_by_name( ) async def get_credential_internal_issuer_jwks( credential_name: str = Path(..., description="The credential name, percent-decoded; may contain slashes"), - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI resolves the dependency from the default + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI resolves the dependency fro ): """ Export the public JWKS for an anthropic ``internal_issuer`` credential, so the operator can @@ -523,7 +527,7 @@ def _per_user_credential_names() -> tuple[str, ...]: async def list_user_connections( request: Request, fastapi_response: Response, - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI resolves the dependency from the default + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI resolves the dependency fro ) -> UserProviderConnectionsResponse: """List the calling user's per-user provider connections.""" from litellm.proxy.proxy_server import prisma_client @@ -545,7 +549,7 @@ async def list_user_connections( if payload is not None: github_logins[row.credential_name] = payload.github_login updated = getattr(row, "updated_at", None) - if updated is not None: + if isinstance(updated, datetime): connected_at[row.credential_name] = updated.isoformat() return UserProviderConnectionsResponse( connections=[ @@ -589,7 +593,7 @@ async def start_user_connection( request: Request, fastapi_response: Response, credential_name: str = Path(..., description="The credential name, percent-decoded"), - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI resolves the dependency from the default + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI resolves the dependency fro ) -> UserConnectionStartResponse: """Begin a GitHub device flow for the calling user's connection to a per-user credential.""" from litellm.llms.github_copilot.per_user_auth import astart_device_flow @@ -636,7 +640,7 @@ async def poll_user_connection( fastapi_response: Response, body: UserConnectionPollRequest, credential_name: str = Path(..., description="The credential name, percent-decoded"), - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI resolves the dependency from the default + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI resolves the dependency fro ) -> UserConnectionPollResponse: """Poll the device flow once and persist the connection on completion.""" from litellm.llms.github_copilot.per_user_auth import ( @@ -711,7 +715,7 @@ async def delete_user_connection( request: Request, fastapi_response: Response, credential_name: str = Path(..., description="The credential name, percent-decoded"), - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI resolves the dependency from the default + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI resolves the dependency fro ) -> UserConnectionDeleteResponse: """Disconnect the calling user's stored GitHub token for a per-user credential. Idempotent.""" from litellm.llms.github_copilot.per_user_auth import evict_copilot_user_session diff --git a/litellm/proxy/credential_endpoints/user_provider_credentials.py b/litellm/proxy/credential_endpoints/user_provider_credentials.py index 47bb19e9b2ec..9eaf1548f97a 100644 --- a/litellm/proxy/credential_endpoints/user_provider_credentials.py +++ b/litellm/proxy/credential_endpoints/user_provider_credentials.py @@ -9,7 +9,12 @@ import json from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Final +from typing import ( + TYPE_CHECKING, + Final, + Protocol, + cast, # noqa: TID251 # narrows the untyped Redis cache to the str-keyed Protocol +) from pydantic import BaseModel, ConfigDict, Field, ValidationError @@ -151,10 +156,35 @@ async def list_user_provider_credentials_for_credential( return await _table(prisma_client).find_many(where={"credential_name": credential_name}) +class _StringKeyCache(Protocol): + async def async_get_cache( + self, + key: str, + **kwargs: object, # kwargs-ok: RedisCache accepts optional cache kwargs + ) -> object: ... + + async def async_set_cache( + self, + key: str, + value: str, + **kwargs: object, # kwargs-ok: RedisCache accepts ttl/nx kwargs + ) -> None: ... + + async def async_delete_cache( + self, + key: str, + **kwargs: object, # kwargs-ok: RedisCache accepts optional cache kwargs + ) -> None: ... + + +def _string_cache(cache: "RedisCache") -> _StringKeyCache: + return cast(_StringKeyCache, cache) # cast-ok: RedisCache exposes the str-keyed cache protocol + + async def _try_cache_get(token_cache: "RedisCache", key: str) -> object: """Redis errors must read as a plain miss: the DB is the source of truth.""" try: - return await token_cache.async_get_cache(key) + return await _string_cache(token_cache).async_get_cache(key) except Exception: # noqa: BLE001 # a Redis outage must fall back to the database, never reject a connected user verbose_proxy_logger.warning("aget_user_provider_tokens: Redis get failed; falling back to the database") return None @@ -162,7 +192,9 @@ async def _try_cache_get(token_cache: "RedisCache", key: str) -> object: async def _try_cache_set(token_cache: "RedisCache", key: str, value: str, nx: bool = False) -> None: try: - await token_cache.async_set_cache(key, value, nx=nx, ttl=GITHUB_COPILOT_USER_CREDENTIAL_CACHE_TTL_SECONDS) + await _string_cache(token_cache).async_set_cache( + key, value, nx=nx, ttl=GITHUB_COPILOT_USER_CREDENTIAL_CACHE_TTL_SECONDS + ) except Exception: # noqa: BLE001 # caching is best-effort; a Redis outage is not worth failing the request verbose_proxy_logger.warning("aget_user_provider_tokens: Redis set failed; skipping the cache write") @@ -181,7 +213,7 @@ async def invalidate_user_provider_credential_cache( if token_cache is None: return True try: - await token_cache.async_set_cache( + await _string_cache(token_cache).async_set_cache( _cache_key(user_id, credential_name), _NOT_CONNECTED, ttl=GITHUB_COPILOT_USER_CREDENTIAL_CACHE_TTL_SECONDS, @@ -211,7 +243,7 @@ async def drop_user_provider_credential_cache( if token_cache is None: return try: - await token_cache.async_delete_cache(_cache_key(user_id, credential_name)) + await _string_cache(token_cache).async_delete_cache(_cache_key(user_id, credential_name)) except Exception: # noqa: BLE001 # a Redis outage must not fail a connect verbose_proxy_logger.warning("drop_user_provider_credential_cache: Redis delete failed") @@ -233,7 +265,7 @@ async def set_user_provider_credential_cache( return True key: Final = _cache_key(user_id, credential_name) try: - await token_cache.async_set_cache( + await _string_cache(token_cache).async_set_cache( key, _encode(payload), ttl=GITHUB_COPILOT_USER_CREDENTIAL_CACHE_TTL_SECONDS, @@ -247,7 +279,7 @@ async def set_user_provider_credential_cache( except Exception: # noqa: BLE001 # a Redis outage must not fail a connect verbose_proxy_logger.warning("set_user_provider_credential_cache: Redis set failed; falling back to delete") try: - await token_cache.async_delete_cache(key) + await _string_cache(token_cache).async_delete_cache(key) except Exception: # noqa: BLE001 # a Redis outage must not fail a connect verbose_proxy_logger.warning("set_user_provider_credential_cache: Redis delete failed") after_delete: object = await _try_cache_get(token_cache, key) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 1ca0c51fcb9a..1c92028219e2 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -75,7 +75,7 @@ strip_callback_config, ) from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # legacy module exports - _safe_get_request_headers, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export + _safe_get_request_headers, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package safe_get_request_headers, ) from litellm.proxy.spend_tracking.carried_budget_state import carried_budget_metadata @@ -1957,7 +1957,9 @@ def add_team_based_callbacks_from_config( return TeamCallbackMetadata( success_callback=team_config.get("success_callback", None), failure_callback=team_config.get("failure_callback", None), - callback_vars=callback_vars_dict, + callback_vars=cast( # cast-ok: callback_vars_dict values are str-coerced above + "dict[str, str]", callback_vars_dict + ), ) @staticmethod @@ -2421,7 +2423,7 @@ async def add_litellm_data_to_request( if key not in data[_metadata_variable_name]: data[_metadata_variable_name][key] = value if _metadata_variable_name == "metadata": - data["metadata"]["tags"] = LiteLLMProxyRequestSetup._merge_tags( # pyright: ignore[reportPrivateUsage] # same-module helper, budget blocks the unsuppressed idiom sibling call sites use + data["metadata"]["tags"] = LiteLLMProxyRequestSetup._merge_tags( # pyright: ignore[reportPrivateUsage] # same request_tags=data["metadata"].get("tags"), tags_to_add=data["litellm_metadata"].get("tags"), ) @@ -2651,24 +2653,32 @@ async def add_litellm_data_to_request( getattr(request.state, "litellm_roi_estimator", False) is True ) - verbose_proxy_logger.debug("[PROXY] returned data from litellm_pre_call_utils: %s", data) + verbose_proxy_logger.debug( + "[PROXY] returned data from litellm_pre_call_utils: %s", + cast("object", data), # cast-ok: data is the untyped request body dict + ) # Team/Project credential overrides from model_config # Placed after the debug log to avoid leaking credential secrets in logs _apply_credential_overrides_from_model_config( data=data, user_api_key_dict=user_api_key_dict, - pre_alias_model_name=_pre_alias_model, + pre_alias_model_name=cast("str | None", _pre_alias_model), # cast-ok: the pre-alias model name arrives as a str llm_router=llm_router, ) await _resolve_user_provider_credentials_for_request( - data=data, + data=cast("dict[str, object]", data), # cast-ok: data is the untyped request body dict authenticated_user_id=authenticated_user_id, team_id=user_api_key_dict.team_id, llm_router=llm_router, router_settings=( - user_api_key_dict.router_settings if isinstance(user_api_key_dict.router_settings, dict) else None + cast("dict[str, object]", rs) # cast-ok: router_settings is a plain config dict + if isinstance( + rs := cast("object", user_api_key_dict.router_settings), # cast-ok: config dict + dict, + ) + else None ), ) @@ -2715,7 +2725,10 @@ def _per_user_oauth_configured() -> bool: return any( isinstance(values := getattr(credential, "credential_values", None), Mapping) - and values.get(GITHUB_COPILOT_AUTH_TYPE_KEY) == GITHUB_COPILOT_PER_USER_AUTH_TYPE + and cast("Mapping[object, object]", values).get( # cast-ok: credential_values is a plain dict at runtime + GITHUB_COPILOT_AUTH_TYPE_KEY + ) + == GITHUB_COPILOT_PER_USER_AUTH_TYPE for credential in (litellm.credential_list or ()) ) @@ -2734,7 +2747,10 @@ def _all_per_user_credential_names() -> tuple[str, ...]: for credential in (litellm.credential_list or ()) if getattr(credential, "credential_name", None) and isinstance(values := getattr(credential, "credential_values", None), Mapping) - and values.get(GITHUB_COPILOT_AUTH_TYPE_KEY) == GITHUB_COPILOT_PER_USER_AUTH_TYPE + and cast("Mapping[object, object]", values).get( # cast-ok: credential_values is a plain dict at runtime + GITHUB_COPILOT_AUTH_TYPE_KEY + ) + == GITHUB_COPILOT_PER_USER_AUTH_TYPE ) @@ -2744,7 +2760,13 @@ def _fallback_entry_target_count(entry: object) -> int: key plus one per target in its value lists, so empty lists and scalar values still cost what the router scan costs.""" if isinstance(entry, dict): - return sum(max(1, len(value)) if isinstance(value, list) else 1 for value in entry.values()) + entry_dict = cast("dict[object, object]", entry) # cast-ok: fallback entries are plain dicts at runtime + return sum( + max(1, len(cast("Sequence[object]", value))) # cast-ok: fallback values are scalar or list entries + if isinstance(value, list) + else 1 + for value in entry_dict.values() + ) return 1 @@ -2761,20 +2783,22 @@ def _fallback_lists( fallbacks in ``_configured_fallbacks``.""" lists: Final[list[Sequence[object]]] = [] # mutable-ok: accumulates each fallback list found for router_list in ( - llm_router.fallbacks, - llm_router.context_window_fallbacks, - llm_router.content_policy_fallbacks, + cast("object", llm_router.fallbacks), # cast-ok: router fallback attrs are untyped + cast("object", llm_router.context_window_fallbacks), # cast-ok: router fallback attrs are untyped + cast("object", llm_router.content_policy_fallbacks), # cast-ok: router fallback attrs are untyped ): if isinstance(router_list, list): - lists.append(router_list) + lists.append(cast("Sequence[object]", router_list)) # cast-ok: fallback lists hold mixed entry shapes for key in _REQUEST_FALLBACK_KEYS: entries: object = data.get(key) if isinstance(entries, list): - lists.append(entries) + lists.append(cast("Sequence[object]", entries)) # cast-ok: request fallback lists hold mixed entry shapes for settings in (router_settings, team_router_settings): key_fallbacks: object = settings.get("fallbacks") if settings is not None else None if isinstance(key_fallbacks, list): - lists.append(key_fallbacks) + lists.append( + cast("Sequence[object]", key_fallbacks) # cast-ok: settings fallback lists hold mixed entry shapes + ) return tuple(lists) @@ -2790,16 +2814,21 @@ def _index_fallback_list(fallback_list: Sequence[object]) -> _FallbackIndex: fuzzy: list[object] = [] # mutable-ok: accumulates matcher-only entries for entry in fallback_list: if isinstance(entry, dict) and entry: - key = next(iter(entry)) - raw = entry[key] - values = raw if isinstance(raw, list) else ([raw] if raw is not None else []) + entry_dict = cast("dict[object, object]", entry) # cast-ok: fallback entries are plain dicts at runtime + key = next(iter(entry_dict)) + raw = entry_dict[key] + values = ( + cast("Sequence[object]", raw) # cast-ok: fallback values are scalar or list entries + if isinstance(raw, list) + else ([raw] if raw is not None else []) + ) targets = tuple(name for value in values if (name := _fallback_target_name(value)) is not None) - if "*" in key or "/" in key: - fuzzy.append(entry) - else: + if isinstance(key, str) and not ("*" in key or "/" in key): exact[key] = exact.get(key, ()) + targets + else: + fuzzy.append(cast("object", entry)) # cast-ok: fuzzy holds mixed entry shapes else: - fuzzy.append(entry) + fuzzy.append(cast("object", entry)) # cast-ok: fuzzy holds mixed entry shapes return exact, tuple(fuzzy) @@ -2807,7 +2836,8 @@ def _fuzzy_entry_matched(entry: object, resolved: object) -> bool: """Whether a fuzzy entry is the one the router matcher returned for a group, so its targets never get expanded a second time.""" if isinstance(entry, dict) and entry: - value: Final = entry[next(iter(entry))] + entry_dict: Final = cast("dict[object, object]", entry) # cast-ok: fallback entries are plain dicts at runtime + value: Final = entry_dict[next(iter(entry_dict))] return value is resolved or value == resolved if isinstance(entry, str): return resolved == [entry] @@ -2848,7 +2878,9 @@ def _fallback_target_name(value: object) -> str | None: if isinstance(value, str): return value if isinstance(value, dict): - model_value: Final = value.get("model") + model_value: Final = cast("dict[str, object]", value).get( # cast-ok: fallback dict entries are str-keyed + "model" + ) return model_value if isinstance(model_value, str) else None return None @@ -2903,22 +2935,38 @@ def _per_user_credential_names_for_groups( *((deployment_id_match,) if deployment_id_match is not None else ()), ) for deployment in deployments: - litellm_params: object = ( - deployment.get("litellm_params") + dep_map = ( + cast("Mapping[str, object]", deployment) # cast-ok: deployments are str-keyed dicts or objects if isinstance(deployment, Mapping) - else getattr(deployment, "litellm_params", None) + else None ) - deployment_model: object = ( - litellm_params.get("model") + litellm_params: object = ( + dep_map.get("litellm_params") if dep_map is not None else getattr(deployment, "litellm_params", None) + ) + lp_map = ( + cast("Mapping[str, object]", litellm_params) # cast-ok: deployment litellm_params is a str-keyed dict if isinstance(litellm_params, Mapping) - else getattr(litellm_params, "model", None) + else None + ) + deployment_model: object = ( + lp_map.get("model") + if lp_map is not None + else getattr( + cast("object", litellm_params), # cast-ok: litellm_params is Mapping or an object + "model", + None, + ) ) if isinstance(deployment_model, str) and classify_strategy_router_model(deployment_model) is not None: return _all_per_user_credential_names() credential_name_obj: object = ( - litellm_params.get("litellm_credential_name") - if isinstance(litellm_params, Mapping) - else getattr(litellm_params, "litellm_credential_name", None) + lp_map.get("litellm_credential_name") + if lp_map is not None + else getattr( + cast("object", litellm_params), # cast-ok: litellm_params is Mapping or an object + "litellm_credential_name", + None, + ) ) if not isinstance(credential_name_obj, str) or not credential_name_obj or credential_name_obj in names: continue @@ -2952,7 +3000,11 @@ async def _team_router_settings(user_api_key_dict_team_id: str | None) -> Mappin proxy_logging_obj=proxy_logging_obj, ) settings: Final = getattr(team_obj, "router_settings", None) - return settings if isinstance(settings, Mapping) else None + return ( + cast("Mapping[str, object]", settings) # cast-ok: team router_settings is a plain config dict + if isinstance(settings, Mapping) + else None + ) except Exception: # noqa: BLE001 # advisory discovery must never break the request return None @@ -2986,7 +3038,10 @@ async def _resolve_user_provider_credentials_for_request( # bound is on aggregate target names, not outer entries, so a single dict # holding thousands of targets is counted honestly. request_fallback_targets: Final = sum( - sum(_fallback_entry_target_count(entry) for entry in entries) + sum( + _fallback_entry_target_count(entry) + for entry in cast("Sequence[object]", entries) # cast-ok: request fallback lists hold mixed entry shapes + ) for key in _REQUEST_FALLBACK_KEYS if isinstance(entries := data.get(key), list) ) @@ -3004,7 +3059,7 @@ async def _resolve_user_provider_credentials_for_request( target for target in ( model, - litellm.model_alias_map.get(model) if isinstance(litellm.model_alias_map, Mapping) else None, + litellm.model_alias_map.get(model), ( resolve_model_group_alias(router_settings.get("model_group_alias"), model) if router_settings is not None @@ -3046,7 +3101,7 @@ async def _resolve_user_provider_credentials_for_request( user_id=user_id, credential_names=credential_names, ) - except Exception: # noqa: BLE001 # credential lookup failures degrade to the shared deployment, never break the request + except Exception: # noqa: BLE001 # credential lookup failures degrade to the shared deployment, never break the re verbose_proxy_logger.exception( "_resolve_user_provider_credentials_for_request: failed to load user provider credentials" ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 67fec38e39eb..0af83e1707b0 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -133,7 +133,7 @@ strip_callback_config, ) from litellm.proxy.common_utils.realtime_utils import ( # noqa: F401, RUF100 # legacy module exports - _realtime_request_body, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export + _realtime_request_body, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package ex realtime_request_body, ) from litellm.proxy.management_helpers.auto_router_availability import AutoRouterCatalogEntry, build_auto_router_catalog @@ -388,7 +388,7 @@ def generate_feedback_box(): ) from litellm.proxy.auth.password_policy import validate_password_not_breached, validate_password_policy from litellm.proxy.auth.user_api_key_auth import ( # noqa: F401, RUF100 # legacy module exports - _fetch_global_spend_with_event_coordination, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export + _fetch_global_spend_with_event_coordination, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards- fetch_global_spend_with_event_coordination, user_api_key_auth, user_api_key_auth_websocket, @@ -401,8 +401,8 @@ def generate_feedback_box(): from litellm.proxy.common_request_processing import ( # noqa: F401, RUF100 # legacy module exports KNOWN_PROXY_ROUTES, ProxyBaseLLMRequestProcessing, - _is_azure_model_router_request, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export - _should_return_raw_model_name, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export + _is_azure_model_router_request, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible pa + _should_return_raw_model_name, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible pac close_guarded_stream, create_response, is_azure_model_router_request, @@ -445,7 +445,7 @@ def generate_feedback_box(): from litellm.proxy.common_utils.html_forms.ui_login import build_ui_login_form from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401, RUF100 # legacy module exports _read_request_body, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export - _safe_get_request_headers, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export + _safe_get_request_headers, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package check_file_size_under_limit, get_form_data, read_request_body, @@ -580,12 +580,12 @@ def generate_feedback_box(): from litellm.proxy.health_endpoints._health_endpoints import router as health_router from litellm.proxy.hooks.model_max_budget_limiter import ( # noqa: F401, RUF100 # legacy module exports PROXY_VirtualKeyModelMaxBudgetLimiter, - _PROXY_VirtualKeyModelMaxBudgetLimiter, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export + _PROXY_VirtualKeyModelMaxBudgetLimiter, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compa ) from litellm.proxy.hooks.parallel_request_limiter_v3 import fail_closed_rate_limit_enforcement_enabled from litellm.proxy.hooks.prompt_injection_detection import ( # noqa: F401, RUF100 # legacy module exports OPTIONAL_PromptInjectionDetection, - _OPTIONAL_PromptInjectionDetection, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export + _OPTIONAL_PromptInjectionDetection, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatibl ) from litellm.proxy.hooks.proxy_track_cost_callback import ( # noqa: F401, RUF100 # legacy module exports ProxyDBLogger, @@ -628,8 +628,8 @@ def generate_feedback_box(): router as callback_management_endpoints_router, ) from litellm.proxy.management_endpoints.common_utils import ( # noqa: F401, RUF100 # legacy module exports - _user_has_admin_privileges, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export - _user_has_admin_view, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export + _user_has_admin_privileges, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible packag + _user_has_admin_view, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package expo admin_can_invite_user, user_api_key_has_admin_view, user_has_admin_privileges, @@ -675,10 +675,10 @@ def generate_feedback_box(): from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( router as model_access_group_management_router, ) -from litellm.proxy.management_endpoints.model_management_endpoints import ( # noqa: F401, RUF100 # legacy module exports +from litellm.proxy.management_endpoints.model_management_endpoints import ( # noqa: F401, RUF100 # legacy module expor _add_model_to_db, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export - _add_team_model_to_db, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export - _deduplicate_litellm_router_models, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export + _add_team_model_to_db, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package exp + _deduplicate_litellm_router_models, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatibl add_model_to_db, add_team_model_to_db, deduplicate_litellm_router_models, @@ -869,10 +869,10 @@ def generate_feedback_box(): _cache_user_row, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _get_docs_url, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _get_openapi_url, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export - _get_projected_spend_over_limit, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export + _get_projected_spend_over_limit, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible p _get_redoc_url, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export - _is_projected_spend_over_limit, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export - _is_valid_team_configs, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export + _is_projected_spend_over_limit, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible pa + _is_valid_team_configs, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package ex cache_user_row, evict_config_param, get_config_param, @@ -1545,7 +1545,7 @@ async def _run_agent_grant_id_migration() -> None: ## when the proxy cache backend is not Redis ## transaction_buffer_redis_cache = redis_usage_cache if transaction_buffer_redis_cache is None: - transaction_buffer_redis_cache = ProxyStartupEvent.get_transaction_buffer_redis_cache( # rebind-ok: pre-existing rebinding on a rename-only line + transaction_buffer_redis_cache = ProxyStartupEvent.get_transaction_buffer_redis_cache( # rebind-ok: pre-existin general_settings=general_settings ) @@ -1790,7 +1790,7 @@ async def _run_agent_grant_id_migration() -> None: await _drain_spend_event_producer_on_shutdown() - # Shutdown event - finish or cancel in-flight scheduled jobs before the shutdown flushes and the DB disconnect + # Shutdown event - finish or cancel in-flight scheduled jobs before the shutdown flushes and the DB disconne if scheduler is not None and scheduler_executor is not None: try: await stop_in_flight_scheduler_jobs(scheduler, scheduler_executor) @@ -6862,7 +6862,7 @@ async def load_config(self, router: litellm.Router | None, config_file_path: str if env_coordination_redis_cache is not None: _set_redis_usage_cache(env_coordination_redis_cache) - ## GENERAL SERVER SETTINGS (e.g. master key,..) # do this after initializing litellm, to ensure sentry logging works for proxylogging + ## GENERAL SERVER SETTINGS (e.g. master key,..) # do this after initializing litellm, to ensure sentry logging w general_settings = config.get("general_settings", {}) if general_settings is None: general_settings = {} @@ -8782,11 +8782,11 @@ async def _init_cyberark_config_override(self, prisma_client: PrismaClient) -> N """ from litellm.proxy.management_endpoints.config_override_endpoints import ( CYBERARK_ENV_VAR_MAPPING, - _clear_cyberark_state, # pyright: ignore[reportPrivateUsage] # module-internal helper shared with the endpoint module - _snapshot_cyberark_boot_env, # pyright: ignore[reportPrivateUsage] # module-internal helper shared with the endpoint module - get_current_env_values, # pyright: ignore[reportPrivateUsage] # module-internal helper shared with the endpoint module - parse_config_value, # pyright: ignore[reportPrivateUsage] # module-internal helper shared with the endpoint module - set_env_vars, # pyright: ignore[reportPrivateUsage] # module-internal helper shared with the endpoint module + _clear_cyberark_state, # pyright: ignore[reportPrivateUsage] # module-internal helper shared with the endp + _snapshot_cyberark_boot_env, # pyright: ignore[reportPrivateUsage] # module-internal helper shared with th + get_current_env_values, # pyright: ignore[reportPrivateUsage] # module-internal helper shared with the end + parse_config_value, # pyright: ignore[reportPrivateUsage] # module-internal helper shared with the endpoin + set_env_vars, # pyright: ignore[reportPrivateUsage] # module-internal helper shared with the endpoint modu ) try: @@ -9010,7 +9010,7 @@ async def _init_prompts_in_db(self, prisma_client: PrismaClient): def parse_row(db_prompt: object) -> PromptSpec | None: try: return self._get_prompt_spec_for_db_prompt(db_prompt=db_prompt) - except Exception as row_error: # noqa: BLE001 # a malformed row must not block syncing the remaining prompts + except Exception as row_error: # noqa: BLE001 # a malformed row must not block syncing the remaining promp verbose_proxy_logger.exception( "litellm.proxy.proxy_server.py::ProxyConfig:_init_prompts_in_db - failed to parse prompt row %s: %s", getattr(db_prompt, "prompt_id", None), @@ -9027,7 +9027,7 @@ def parse_row(db_prompt: object) -> PromptSpec | None: for prompt_spec in parsed_specs: try: IN_MEMORY_PROMPT_REGISTRY.sync_prompt_from_db(prompt=prompt_spec) - except Exception as prompt_sync_error: # noqa: BLE001 # one poisoned row must not block syncing the remaining prompts + except Exception as prompt_sync_error: # noqa: BLE001 # one poisoned row must not block syncing the re verbose_proxy_logger.exception( "litellm.proxy.proxy_server.py::ProxyConfig:_init_prompts_in_db - failed to sync prompt %s: %s", prompt_spec.prompt_id, @@ -9347,7 +9347,10 @@ def decrypt_credentials(self, credential: dict | BaseModel) -> CredentialItem: decrypted_credential_values: Final = {} for k, v in credential_object.credential_values.items(): - decrypted_credential_values[k] = decrypted_or_stored(k, v) + decrypted_credential_values[k] = decrypted_or_stored( + k, + cast("str", v), # cast-ok: credential values are str at the decrypt boundary + ) credential_object.credential_values = decrypted_credential_values return credential_object @@ -10831,7 +10834,7 @@ async def _load_heuristic_v1_tuning_baselines( ) } ) - except Exception as e: # noqa: BLE001 # enforcement is skipped for this boot; refusing every tuned router on a DB blip is the one outcome the gate forbids + except Exception as e: # noqa: BLE001 # enforcement is skipped for this boot; refusing every tuned router on a verbose_proxy_logger.warning("Heuristic-v1 tuning baseline unavailable, gate not enforced this boot: %s", e) return None @@ -11589,7 +11592,7 @@ async def _scheduled_monthly_spend_report() -> None: slack_alerting_args: Final = proxy_logging_obj.slack_alerting_instance.alerting_args user_spend_check_interval: Final = ( slack_alerting_args.user_spend_check_interval - if isinstance(slack_alerting_args, SlackAlertingArgs) # pyright: ignore[reportUnnecessaryIsInstance] # tests inject a mock slack_alerting_instance + if isinstance(slack_alerting_args, SlackAlertingArgs) # pyright: ignore[reportUnnecessaryIsInstance] # test else SlackAlertingArgs().user_spend_check_interval ) @@ -11849,7 +11852,7 @@ async def _deployment_hidden_by_listing_callbacks(deployment: Deployment, user_a "/models", dependencies=[Depends(user_api_key_auth)], tags=["model management"] ) # if project requires model list async def model_list( - request: Request = None, # pyright: ignore[reportArgumentType] # FastAPI always injects the Request; the None default only serves direct in-process callers + request: Request = None, # pyright: ignore[reportArgumentType] # FastAPI always injects the Request; the None defa user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), return_wildcard_routes: bool | None = False, team_id: str | None = None, @@ -13362,7 +13365,7 @@ async def return_body(): code=1011, reason=websocket_close_reason(redacted_error, fallback="Internal server error"), ) - except Exception: # noqa: BLE001 # the lower layer may have closed the socket already; closing twice is not an error + except Exception: # noqa: BLE001 # the lower layer may have closed the socket already; closing twice is not an verbose_proxy_logger.debug("Could not close realtime client websocket; it is already gone") finally: if not litellm_logging_obj.model_call_details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY): @@ -20461,7 +20464,7 @@ async def body_iter(): ) async def proxy_mcp_route(request: Request) -> Response: """Serve the fixed three-tool MCP proxy surface.""" - from litellm.proxy._experimental.mcp_server.mcp_context import ( # pyright: ignore[reportPrivateUsage] # route-owned mode + from litellm.proxy._experimental.mcp_server.mcp_context import ( # pyright: ignore[reportPrivateUsage] # route-own _mcp_proxy_mode, # pyright: ignore[reportPrivateUsage] # route-owned mode ) from litellm.proxy._experimental.mcp_server.server import handle_streamable_http_mcp diff --git a/litellm/responses/main.py b/litellm/responses/main.py index de16406f20a7..00369923d577 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -670,7 +670,9 @@ async def aresponses( _, custom_llm_provider, _, _ = litellm.get_llm_provider( model=model, api_base=local_vars.get("base_url", None), - litellm_params=GenericLiteLLMParams(**kwargs), + litellm_params=GenericLiteLLMParams( + **cast("dict[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict + ), ) # Update local_vars with detected provider (fixes #19782) local_vars["custom_llm_provider"] = custom_llm_provider @@ -794,7 +796,7 @@ async def aresponses( # (mirrors litellm/main.py:1371 for chat completions) response.hidden_params["custom_llm_provider"] = custom_llm_provider - if response is None: # pyright: ignore[reportUnnecessaryComparison] # provider handlers can return None at runtime + if response is None: # pyright: ignore[reportUnnecessaryComparison] # provider handlers can return None at run raise ValueError(f"Got an unexpected None response from the Responses API: {response}") return response @@ -1264,7 +1266,9 @@ def responses( _, custom_llm_provider, _, _ = litellm.get_llm_provider( model=model, api_base=local_vars.get("base_url", None), - litellm_params=GenericLiteLLMParams(**kwargs), + litellm_params=GenericLiteLLMParams( + **cast("dict[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict + ), ) local_vars["custom_llm_provider"] = custom_llm_provider @@ -1732,11 +1736,13 @@ async def aget_responses( response = init_response # Update the responses_api_response_id with the model_id - if not isinstance(response, ResponsesAPIResponse): # pyright: ignore[reportUnnecessaryIsInstance] # handlers can return non-ResponsesAPIResponse objects at runtime + if not isinstance(response, ResponsesAPIResponse): # pyright: ignore[reportUnnecessaryIsInstance] # handlers c return response return ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id( responses_api_response=response, - litellm_metadata=kwargs.get("litellm_metadata", {}), + litellm_metadata=cast( # cast-ok: litellm_metadata is a plain dict when present + "dict[str, object]", kwargs.get("litellm_metadata", {}) + ), custom_llm_provider=custom_llm_provider, ) except Exception as e: @@ -2174,7 +2180,9 @@ async def acompact_responses( _, custom_llm_provider, _, _ = litellm.get_llm_provider( model=model, api_base=local_vars.get("base_url", None), - litellm_params=GenericLiteLLMParams(**kwargs), + litellm_params=GenericLiteLLMParams( + **cast("dict[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict + ), ) # Update local_vars with detected provider (fixes #19782) local_vars["custom_llm_provider"] = custom_llm_provider @@ -2203,11 +2211,13 @@ async def acompact_responses( response = init_response # Update the responses_api_response_id with the model_id - if not isinstance(response, ResponsesAPIResponse): # pyright: ignore[reportUnnecessaryIsInstance] # handlers can return non-ResponsesAPIResponse objects at runtime + if not isinstance(response, ResponsesAPIResponse): # pyright: ignore[reportUnnecessaryIsInstance] # handlers c return response return ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id( responses_api_response=response, - litellm_metadata=kwargs.get("litellm_metadata", {}), + litellm_metadata=cast( # cast-ok: litellm_metadata is a plain dict when present + "dict[str, object]", kwargs.get("litellm_metadata", {}) + ), custom_llm_provider=custom_llm_provider, ) except Exception as e: diff --git a/litellm/router.py b/litellm/router.py index b8c7443095d2..fcf1b982784f 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -204,7 +204,7 @@ from litellm.router_utils.cooldown_cache import CooldownCache from litellm.router_utils.cooldown_handlers import ( DEFAULT_COOLDOWN_TIME_SECONDS, - _first_present, # pyright: ignore[reportPrivateUsage] - shared internal helper across router_utils submodules, matching the other cooldown_handlers imports on this line + _first_present, # pyright: ignore[reportPrivateUsage] - shared internal helper across router_utils submodules, matc async_get_cooldown_deployments, async_get_cooldown_deployments_with_debug_info, get_cooldown_deployments, @@ -791,7 +791,7 @@ def follow_source_attribution(self) -> None: if source_params is None or source_params is self._followed_source_params: return self._followed_source_params = source_params - hidden_params, headers = Router._prepare_fallback_hidden_params(source) # pyright: ignore[reportPrivateUsage] # this wrapper is the Router's own stream type + hidden_params, headers = Router._prepare_fallback_hidden_params(source) # pyright: ignore[reportPrivateUsage] # this self.merge_fallback_hidden_params(hidden_params, headers) def __aiter__(self) -> "FallbackAwareAnthropicMessagesStream": @@ -1065,7 +1065,7 @@ def __init__( from litellm._service_logger import ServiceLogging self.service_logger_obj: ServiceLogging = ServiceLogging() - litellm.suppress_debug_info = True # prevents 'Give Feedback/Get help' message from being emitted on Router - Relevant Issue: https://github.com/BerriAI/litellm/issues/5942 + litellm.suppress_debug_info = True # prevents 'Give Feedback/Get help' message from being emitted on Router - R if self.set_verbose is True: if debug_level == "INFO": verbose_router_logger.setLevel(logging.INFO) @@ -1141,7 +1141,7 @@ def __init__( # Initialize model_group_alias early since it's used in set_model_list self.model_group_alias: dict[str, str | RouterModelGroupAliasItem] = ( model_group_alias or {} - ) # dict to store aliases for router, ex. {"gpt-4": "gpt-3.5-turbo"}, all requests with gpt-4 -> get routed to gpt-3.5-turbo group + ) # dict to store aliases for router, ex. {"gpt-4": "gpt-3.5-turbo"}, all requests with gpt-4 -> get routed to # Initialize model ID to deployment index mapping for O(1) lookups self.model_id_to_deployment_index_map: dict[str, int] = {} @@ -1190,7 +1190,7 @@ def __init__( if "model" in m["litellm_params"]: self.deployment_latency_map[m["litellm_params"]["model"]] = 0 else: - self.model_list: list = [] # initialize an empty list - to allow _add_deployment and delete_deployment to work + self.model_list: list = [] # initialize an empty list - to allow _add_deployment and delete_deployment to w if allowed_fails is not None: self.allowed_fails = allowed_fails @@ -3588,7 +3588,7 @@ async def _aresponses_fallback_attempt( partial_usage: Final = Router._extract_partial_responses_usage(source_iterator) fallback_response = None # rebind-ok: pre-init so finally can close it if a fallback was actually attempted - fallback_yielded = False # rebind-ok: flipped on the first fallback item so a fallback that dies before its first event still replays the primary's held announcement + fallback_yielded = False # rebind-ok: flipped on the first fallback item so a fallback that dies before its fir try: model_group: Final = cast(str, initial_kwargs.get("model")) # cast-ok: model group fallbacks: Final[list | None] = initial_kwargs.get( # mutable-ok: matches the common_utils list|None param @@ -4069,7 +4069,7 @@ def _stamp_retry_skip_deployment_id(exception: Exception, kwargs: Mapping[str, o effective_model_info: Final = kwargs.get("model_info") deployment_id: Final = effective_model_info.get("id") if isinstance(effective_model_info, Mapping) else None if isinstance(deployment_id, str) and deployment_id: - exception.retry_skip_deployment_id = deployment_id # pyright: ignore[reportAttributeAccessIssue] # dynamic stamp, read by _deployment_ids_to_skip_on_retry + exception.retry_skip_deployment_id = deployment_id # pyright: ignore[reportAttributeAccessIssue] # dynamic def _update_kwargs_with_default_litellm_params( self, kwargs: dict, metadata_variable_name: str | None = "metadata" @@ -4229,8 +4229,14 @@ def _update_kwargs_with_deployment( github_copilot_per_user_credential_name, ) - configured_per_user_name: Final = github_copilot_per_user_credential_name(deployment.get("litellm_params")) - caller_credential_name: Final = kwargs.get("litellm_credential_name") + configured_per_user_name: Final = github_copilot_per_user_credential_name( + cast( # cast-ok: deployment litellm_params is a str-keyed dict + "Mapping[str, object]", deployment.get("litellm_params") + ) + ) + caller_credential_name: Final[object] = cast( # cast-ok: kwargs is the untyped request dict + "object", kwargs.get("litellm_credential_name") + ) if ( configured_per_user_name is not None and isinstance(caller_credential_name, str) @@ -5438,7 +5444,7 @@ def _add_deployment_model_to_endpoint_for_llm_passthrough_route( """ if "endpoint" in kwargs and kwargs["endpoint"]: # For provider-specific endpoints, strip the provider prefix from model_name - # e.g., "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0" -> "us.anthropic.claude-3-5-sonnet-20240620-v1:0" + # e.g., "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0" -> "us.anthropic.claude-3-5-sonnet-20240620-v from litellm import get_llm_provider try: @@ -5667,7 +5673,7 @@ async def stream_with_fallbacks() -> AsyncGenerator[bytes, None]: # to take over there is nothing to buffer for, so every frame, # including pings and provider error frames, is forwarded live. model: Final = cast(str, initial_kwargs.get("model")) # cast-ok: kwargs always carries the model group - has_generated_content = not ( # rebind-ok: set once real content is seen, the buffer cap is hit, or neither a retry nor a fallback can take over + has_generated_content = not ( # rebind-ok: set once real content is seen, the buffer cap is hit, or neither self._anthropic_messages_stream_can_retry(initial_kwargs) or self._anthropic_messages_stream_can_fall_back(model, initial_kwargs) ) @@ -5683,8 +5689,8 @@ async def stream_with_fallbacks() -> AsyncGenerator[bytes, None]: # detection parses the accumulated buffer plus the current chunk, never the # chunk alone; the buffer is already capped, which bounds this window too. parse_window = ( - b"".join(c for c in (*buffered_lifecycle_chunks, chunk) if isinstance(c, (bytes, bytearray))) # pyright: ignore[reportUnnecessaryIsInstance] # bridge-path chunks are not always bytes at runtime - if not has_generated_content and isinstance(chunk, (bytes, bytearray)) # pyright: ignore[reportUnnecessaryIsInstance] # bridge-path chunks are not always bytes at runtime + b"".join(c for c in (*buffered_lifecycle_chunks, chunk) if isinstance(c, (bytes, bytearray))) # pyright: ignore[reportUnnecessaryIsInstance] # brid + if not has_generated_content and isinstance(chunk, (bytes, bytearray)) # pyright: ignore[reportUnnecessaryIsInstance] # brid else chunk ) error_event = parse_anthropic_error_event(parse_window) @@ -5956,7 +5962,7 @@ async def _aanthropic_messages_retry_same_group( ) try: response = await self._ageneric_api_call_with_fallbacks_anthropic_messages_attempt(**retry_kwargs) - except Exception as retry_error: # noqa: BLE001 # every failure of a retry before its stream opens is the fallback chain's to judge + except Exception as retry_error: # noqa: BLE001 # every failure of a retry before its stream opens is the wrapped = _anthropic_stream_fallback_error_for_raised(retry_error, model_group, False) if wrapped is None: return _AnthropicStreamRetriesExhausted( @@ -6426,7 +6432,7 @@ async def create_file_for_deployment(deployment: dict) -> OpenAIFileObject: ) if ( "gcs_bucket_name" in data - ): # TODO: Remove this once we have a better way to handle GCS bucket name: Problem is that we need to pass the gcs_bucket_name to the router for the create_file call but it doesn't show up there + ): # TODO: Remove this once we have a better way to handle GCS bucket name: Problem is that we need to kwargs_copy.setdefault("litellm_metadata", {})["gcs_bucket_name"] = data["gcs_bucket_name"] async with self._deployment_slot( deployment=deployment, kwargs=kwargs_copy, parent_otel_span=parent_otel_span @@ -7720,7 +7726,7 @@ async def async_function_with_fallbacks_common_utils( try: verbose_router_logger.info("Trying to fallback b/w models") - # check if client-side fallbacks are used (e.g. fallbacks = ["gpt-3.5-turbo", "claude-3-haiku"] or fallbacks=[{"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Hey, how's it going?"}]}] + # check if client-side fallbacks are used (e.g. fallbacks = ["gpt-3.5-turbo", "claude-3-haiku"] or fallbacks is_non_standard_fallback_format: Final = check_non_standard_fallback_format(fallbacks=fallbacks) if is_non_standard_fallback_format: @@ -7885,7 +7891,7 @@ async def async_function_with_fallbacks(self, *args, **kwargs): ) if compaction_surface is not None: kwargs["_context_compaction_state"] = initialize_compaction_state(kwargs, compaction_surface) - clear_pre_routing_selection(kwargs) # pyright: ignore[reportUnknownArgumentType] # **kwargs is untyped at this boundary + clear_pre_routing_selection(kwargs) # pyright: ignore[reportUnknownArgumentType] # **kwargs is untyped at this if not isinstance(kwargs.get("attempted_targets"), AttemptedFallbackTargets): _fallback_metadata_key: Final = get_router_metadata_variable_name( function_name=getattr(kwargs.get("original_function"), "__name__", None) @@ -8680,8 +8686,11 @@ def deployment_callback_on_failure( original_exception=exception, deployment=deployment_id, time_to_cooldown=_time_to_cooldown, - requested_model_group=(get_litellm_metadata_from_kwargs(kwargs) or {}).get("model_group"), - request_kwargs=kwargs, + requested_model_group=( + get_litellm_metadata_from_kwargs(cast("dict[str, object]", kwargs)) # cast-ok: kwargs is the un + or {} + ).get("model_group"), + request_kwargs=cast("dict[str, object]", kwargs), # cast-ok: kwargs is the untyped request dict ) # setting deployment_id in cooldown deployments return result @@ -10135,10 +10144,21 @@ def _initialize_deployment_for_pass_through(self, deployment: Deployment, custom credential_values: Final = ( CredentialAccessor.get_credential_values(credential_name) if credential_name is not None else {} ) - vertex_project: Final = credential_values.get("vertex_project") or deployment.litellm_params.vertex_project - vertex_location: Final = credential_values.get("vertex_location") or deployment.litellm_params.vertex_location + from litellm.types.llms.vertex_ai import VERTEX_CREDENTIALS_TYPES + + vertex_project: Final = ( + cast("str | None", credential_values.get("vertex_project")) # cast-ok: vertex credential values are str + or deployment.litellm_params.vertex_project + ) + vertex_location: Final = ( + cast("str | None", credential_values.get("vertex_location")) # cast-ok: vertex credential values are str + or deployment.litellm_params.vertex_location + ) vertex_credentials: Final = ( - credential_values.get("vertex_credentials") or deployment.litellm_params.vertex_credentials + cast( # cast-ok: vertex_credentials holds the typed credential union + "VERTEX_CREDENTIALS_TYPES | None", credential_values.get("vertex_credentials") + ) + or deployment.litellm_params.vertex_credentials ) if vertex_project is None or vertex_location is None: @@ -10389,7 +10409,7 @@ def _restore_deployment_after_failed_upsert(self, previous_deployment: Deploymen previous_deployment.model_name, model_id, ) - except Exception as restore_error: # noqa: BLE001 # best-effort restore: a second failure must not abort the reload + except Exception as restore_error: # noqa: BLE001 # best-effort restore: a second failure must not abort the r verbose_router_logger.warning( "Could not restore previously served deployment %s (id=%s) after the failed upsert: %s", previous_deployment.model_name, @@ -12417,7 +12437,7 @@ def _deployment_accepts_param(deployment: DeploymentTypedDict, group: str, param custom_llm_provider=custom_llm_provider, base_model=base_model if isinstance(base_model, str) else None, ) - except Exception as e: # noqa: BLE001 # best-effort filter: an unresolvable provider must not narrow the request + except Exception as e: # noqa: BLE001 # best-effort filter: an unresolvable provider must not narrow the reque verbose_router_logger.debug( "litellm.router.py::_deployment_accepts_param: keeping %s for model=%s. Got - %s", param, group, e ) @@ -13036,7 +13056,7 @@ def _pre_call_checks( model=_dep_model_for_params, litellm_params=LiteLLM_Params.model_validate(_litellm_params), ) - except Exception as e: # noqa: BLE001 # best-effort filter: an unresolvable provider must not fail the request + except Exception as e: # noqa: BLE001 # best-effort filter: an unresolvable provider must not fail the verbose_router_logger.debug( "litellm.router.py::_pre_call_checks: skipping supported-params check for model=%s. Got - %s", _dep_model_for_params, @@ -13483,7 +13503,7 @@ async def async_get_healthy_deployments( request_kwargs=request_kwargs, ) - # IF TEAM ID SPECIFIED ON MODEL, AND REQUEST CONTAINS USER_API_KEY_TEAM_ID, FILTER OUT MODELS THAT ARE NOT IN THE TEAM + # IF TEAM ID SPECIFIED ON MODEL, AND REQUEST CONTAINS USER_API_KEY_TEAM_ID, FILTER OUT MODELS THAT ARE NOT IN TH ## THIS PREVENTS WRITING FILES OF OTHER TEAMS TO MODELS THAT ARE TEAM-ONLY MODELS healthy_deployments = filter_team_based_models( healthy_deployments=healthy_deployments, @@ -13779,7 +13799,7 @@ async def _async_get_available_deployment( and self.routing_strategy != "cost-based-routing" and self.routing_strategy != "latency-based-routing" and self.routing_strategy != "least-busy" - ): # prevent regressions for other routing strategies, that don't have async get available deployments implemented. + ): # prevent regressions for other routing strategies, that don't have async get available deployments implemen return self.get_available_deployment( model=model, messages=messages, @@ -14708,7 +14728,7 @@ def get_available_deployment( "usage-based-routing-v2, cost-based-routing, latency-based-routing, least-busy), " "or remove `plugins` from the Router config." ) - # users need to explicitly call a specific deployment, by setting `specific_deployment = True` as completion()/embedding() kwarg + # users need to explicitly call a specific deployment, by setting `specific_deployment = True` as completion()/e # When this was no explicit we had several issues with fallbacks timing out model, healthy_deployments = self._common_checks_available_deployment( diff --git a/litellm/router_utils/clientside_credential_handler.py b/litellm/router_utils/clientside_credential_handler.py index 05f64f813c2d..150904071749 100644 --- a/litellm/router_utils/clientside_credential_handler.py +++ b/litellm/router_utils/clientside_credential_handler.py @@ -11,7 +11,8 @@ Ensures cooldowns are applied correctly. """ -from typing import Final +from collections.abc import MutableMapping +from typing import Final, cast # noqa: TID251 # narrows the untyped request dict for mutation from litellm.types.utils import server_owned_wif_litellm_params @@ -119,20 +120,22 @@ def get_dynamic_litellm_params(litellm_params: dict, request_kwargs: dict) -> di # shared mode and send the admin's Copilot token to the redirected host. # Per-user mode ignores the caller's api_base anyway (the session's # validated host wins), so keeping the mode is fail-closed. - credential_name_value: Final[object] = litellm_params.get( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # litellm_params is the untyped request dict + credential_name_value: Final[object] = litellm_params.get( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # lite "litellm_credential_name" ) - auth_type: Final[object] = litellm_params.get( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # litellm_params is the untyped request dict + auth_type: Final[object] = litellm_params.get( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # lite "github_copilot_auth_type" ) per_user_credential_locked: Final = github_copilot_auth_mode( - credential_name_value, # pyright: ignore[reportUnknownArgumentType] # value comes from the untyped request dict + credential_name_value, # pyright: ignore[reportUnknownArgumentType] # value comes from the untyped request auth_type, # pyright: ignore[reportUnknownArgumentType] # value comes from the untyped request dict ) for field in _ADMIN_CONFIG_FIELDS_TO_CLEAR_ON_BASE_OVERRIDE: if per_user_credential_locked and field in ("litellm_credential_name", "github_copilot_auth_type"): continue - litellm_params.pop(field, None) + cast( # cast-ok: litellm_params is a mutable request dict at runtime + "MutableMapping[str, object]", litellm_params + ).pop(field, None) if field in request_kwargs: litellm_params[field] = request_kwargs[field] litellm_params[DISABLE_WORKLOAD_IDENTITY_PARAM] = True diff --git a/litellm/utils.py b/litellm/utils.py index 734bb15018da..205288ec0877 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -149,7 +149,7 @@ def _snapshot_exception_for_hook(exception: Exception) -> Exception: # behavior for `raise ... from ...`), so this must be set after, not before. snapshot.__suppress_context__ = exception.__suppress_context__ return snapshot - except Exception: # noqa: BLE001 # any snapshot failure must fall back to the live object, not break the failure path + except Exception: # noqa: BLE001 # any snapshot failure must fall back to the live object, not break the failure p return exception @@ -1413,7 +1413,7 @@ async def _client_async_logging_helper( ): if ( is_completion_with_fallbacks is False - ): # don't log the parent event litellm.completion_with_fallbacks as a 'log_success_event', this will lead to double logging the same call - https://github.com/BerriAI/litellm/issues/7477 + ): # don't log the parent event litellm.completion_with_fallbacks as a 'log_success_event', this will lead to doubl print_verbose( f"Async Wrapper: Completed Call, calling async_success_handler: {logging_obj.async_success_handler}" ) @@ -1630,7 +1630,7 @@ def post_call_processing( and optional_params["response_format"].get("json_schema") is not None ): json_response_format = optional_params["response_format"] - elif _parsing._completions.is_basemodel_type( # pyright: ignore[reportPrivateUsage] # OpenAI parser helper is private + elif _parsing._completions.is_basemodel_type( # pyright: ignore[reportPrivateUsage] # Open optional_params["response_format"] ): json_response_format = type_to_response_format_param( @@ -1742,7 +1742,7 @@ def wrapper(*args, **kwargs): load_credentials_from_list(kwargs) from litellm.llms.github_copilot.per_user_auth import attach_github_copilot_user_session - attach_github_copilot_user_session(kwargs) # pyright: ignore[reportUnknownArgumentType] # kwargs is the untyped request dict + attach_github_copilot_user_session(kwargs) # pyright: ignore[reportUnknownArgumentType] # kwargs is the un kwargs["litellm_logging_obj"] = logging_obj LLMCachingHandler: Final = _get_cached_llm_caching_handler() _llm_caching_handler: Final[LLMCachingHandler] = LLMCachingHandler( @@ -1898,7 +1898,7 @@ def wrapper(*args, **kwargs): end_time=end_time, ) - # LOG SUCCESS - handle streaming success logging in the _next_ object, remove `handle_success` once it's deprecated + # LOG SUCCESS - handle streaming success logging in the _next_ object, remove `handle_success` once it's dep verbose_logger.info("Wrapper: Completed Call, calling success_handler") if not is_internal_call.get(): # Copy the current context to propagate it to the background thread @@ -1978,7 +1978,7 @@ def wrapper(*args, **kwargs): traceback_exception: Final = traceback.format_exc() end_time = datetime.datetime.now() - # LOG FAILURE - handle streaming failure logging in the _next_ object, remove `handle_failure` once it's deprecated + # LOG FAILURE - handle streaming failure logging in the _next_ object, remove `handle_failure` once it's dep if logging_obj: logging_obj.failure_handler( e, traceback_exception, start_time, end_time @@ -2033,7 +2033,7 @@ async def wrapper_async(*args, **kwargs): load_credentials_from_list(kwargs) from litellm.llms.github_copilot.per_user_auth import aattach_github_copilot_user_session - await aattach_github_copilot_user_session(kwargs) # pyright: ignore[reportUnknownArgumentType] # kwargs is the untyped request dict + await aattach_github_copilot_user_session(kwargs) # pyright: ignore[reportUnknownArgumentType] # kwargs is logging_obj.llm_caching_handler = _llm_caching_handler # [OPTIONAL] CHECK BUDGET if litellm.max_budget: @@ -2117,14 +2117,14 @@ async def wrapper_async(*args, **kwargs): try: result = await original_function(*args, **call_kwargs) except Exception as deployment_error: - _deployment_call_end_time = datetime.datetime.now() # noqa: DTZ005 # matches the naive datetimes this whole function already times start_time/end_time with + _deployment_call_end_time = datetime.datetime.now() # noqa: DTZ005 # matches the naive datetimes this try: await async_post_call_failure_deployment_hook( request_data=call_kwargs, exception=deployment_error, call_type=call_type, ) - except BaseException: # noqa: S110, BLE001 # hook dispatch - including cancellation mid-await - must never replace the real deployment failure, so there is nothing to do with what it raises + except BaseException: # noqa: S110, BLE001 # hook dispatch - including cancellation mid-await - must n pass raise end_time = datetime.datetime.now() @@ -2225,7 +2225,7 @@ async def wrapper_async(*args, **kwargs): traceback_exception: Final = traceback.format_exc() # Reuse the timestamp taken right when the deployment call itself failed, before # the failure hook ran, so a slow callback doesn't inflate the reported duration. - end_time = _deployment_call_end_time if _deployment_call_end_time is not None else datetime.datetime.now() # noqa: DTZ005 # matches the naive datetimes this whole function already times start_time/end_time with + end_time = _deployment_call_end_time if _deployment_call_end_time is not None else datetime.datetime.now() # noqa: DTZ005 # matc if logging_obj and not _is_litellm_internal_call: unbind_budget_reservation_from_callbacks(logging_obj.litellm_params) try: @@ -4363,7 +4363,7 @@ def pre_process_non_default_params( non_default_params, list ): # fixes https://github.com/BerriAI/litellm/issues/4933 tools: Final = non_default_params["tools"] - for tool in tools: # clean out 'additionalProperties = False'. Causes vertexai/gemini OpenAI API Schema errors - https://github.com/langchain-ai/langchainjs/issues/5240 + for tool in tools: # clean out 'additionalProperties = False'. Causes vertexai/gemini OpenAI API Schema errors tool_function = tool.get("function", {}) parameters = tool_function.get("parameters", None) if parameters is not None: @@ -4898,7 +4898,7 @@ def _check_valid_arg(supported_params: list[str]): drop_params=bool(drop_params), ) elif custom_llm_provider == "nadir": - optional_params = litellm.NadirConfig().map_openai_params( # rebind-ok: same optional_params rebinding every sibling provider branch does + optional_params = litellm.NadirConfig().map_openai_params( # rebind-ok: same optional_params rebinding every si non_default_params=non_default_params, optional_params=optional_params, model=model, @@ -5105,7 +5105,7 @@ def add_provider_specific_params_to_optional_params( """ if custom_llm_provider in ["openai", "azure", "text-completion-openai"] + litellm.openai_compatible_providers: - # for openai, azure we should pass the extra/passed params within `extra_body` https://github.com/openai/openai-python/blob/ac33853ba10d13ac149b1fa3ca6dba7d613065c9/src/openai/resources/models.py#L46 + # for openai, azure we should pass the extra/passed params within `extra_body` https://github.com/openai/openai- if _should_drop_param(k="extra_body", additional_drop_params=additional_drop_params) is False: extra_body: Final = dict(passed_params.pop("extra_body", None) or {}) for k in passed_params: @@ -7605,7 +7605,7 @@ def attempt_message_addition(final_messages, message, available_tokens, max_toke # if temp_message_tokens > max_tokens, try shortening temp_messages elif "function_call" not in message: verbose_logger.debug("attempting to shorten message to fit limit") - # fit updated_message to be within temp_message_tokens - max_tokens (aka the amount temp_message_tokens is greate than max_tokens) + # fit updated_message to be within temp_message_tokens - max_tokens (aka the amount temp_message_tokens is great updated_message: Final = shorten_message_to_fit_limit(message, available_tokens, model) if can_add_message(updated_message, final_messages, max_tokens, model): verbose_logger.debug("can add message, returning [updated_message] + final_messages") @@ -8311,7 +8311,12 @@ def convert_list_message_to_dict( messages: Sequence[BaseModel | Mapping[str, object]], ) -> list[dict[str, object]]: # mutable-ok: callers mutate the returned message dicts def _as_message_value(message: BaseModel | Mapping[str, object]) -> AllMessageValues: - return cast(AllMessageValues, convert_to_dict(message)) # cast-ok: message dicts satisfy the TypedDict shape + return cast( # cast-ok: message dicts satisfy the TypedDict shape + AllMessageValues, + convert_to_dict( + cast("BaseModel | dict[str, object]", message) # cast-ok: messages are dicts or pydantic models at runt + ), + ) return [dict(cleanup_none_field_in_message(message=_as_message_value(message))) for message in messages] @@ -8330,7 +8335,10 @@ def validate_and_fix_openai_messages(messages: list): message["tool_calls"] = jsonify_tools(tools=message["tool_calls"]) convert_msg_to_dict = cast( # cast-ok: message dicts satisfy the TypedDict shape - AllMessageValues, convert_to_dict(message) + AllMessageValues, + convert_to_dict( + cast("BaseModel | dict[str, object]", message) # cast-ok: dicts or pydantic models at runtime + ), ) cleaned_message = cleanup_none_field_in_message(message=convert_msg_to_dict) new_messages.append(cleaned_message) From a1330a6b5a4b85be5fd4b1fc3fd1eb1b02ef74ab Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:28:55 +0000 Subject: [PATCH 36/38] fix(github_copilot): validate per-user OAuth boundaries and restore truncated comments --- .../get_llm_provider_logic.py | 18 ++-- .../responses/transformation.py | 6 +- litellm/main.py | 96 +++++++++---------- litellm/models/credentials.py | 8 +- .../common_utils/credential_hydration.py | 2 +- .../proxy/credential_endpoints/endpoints.py | 10 +- litellm/proxy/litellm_pre_call_utils.py | 16 ++-- litellm/proxy/proxy_server.py | 60 ++++++------ litellm/responses/main.py | 6 +- litellm/router.py | 42 ++++---- .../clientside_credential_handler.py | 6 +- litellm/utils.py | 28 +++--- 12 files changed, 148 insertions(+), 150 deletions(-) diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 8f5a90f85bb7..fe5d7c66388a 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -240,7 +240,7 @@ def get_llm_provider( model.split("/", 1)[0] in litellm.provider_list and model.split("/", 1)[0] not in litellm.model_list_set and len(model.split("/")) - > 1 # handle edge case where user passes in `litellm --model mistral` https://github.com/BerriAI/litellm/is + > 1 # handle edge case where user passes in `litellm --model mistral` https://github.com/BerriAI/litellm/issues/1351 ): return _get_openai_compatible_provider_info( model=model, @@ -411,7 +411,7 @@ def get_llm_provider( raise Exception(f"dynamic_api_key needs to be a string. dynamic_api_key={dynamic_api_key}") return model, custom_llm_provider, dynamic_api_key, api_base - # check if model in known model provider list -> for huggingface models, raise exception as they don't have a f + # check if model in known model provider list -> for huggingface models, raise exception as they don't have a fixed provider (can be togetherai, anyscale, baseten, runpod, et.) ## openai - chatcompletion + text completion if ( model in litellm.open_ai_chat_completion_models @@ -609,7 +609,7 @@ def _get_openai_compatible_provider_info( return model, custom_llm_provider, dynamic_api_key, api_base if custom_llm_provider == "perplexity": - # perplexity is openai compatible, we just need to set this to custom_openai and have the api_base be https://ap + # perplexity is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.perplexity.ai ( api_base, dynamic_api_key, @@ -617,7 +617,7 @@ def _get_openai_compatible_provider_info( elif custom_llm_provider == "aiohttp_openai": return model, "aiohttp_openai", api_key, api_base elif custom_llm_provider == "anyscale": - # anyscale is openai compatible, we just need to set this to custom_openai and have the api_base be https://api + # anyscale is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.endpoints.anyscale.com/v1 api_base = api_base or get_secret_str("ANYSCALE_API_BASE") or "https://api.endpoints.anyscale.com/v1" dynamic_api_key = api_key or get_secret_str("ANYSCALE_API_KEY") elif custom_llm_provider == "deepinfra": @@ -644,7 +644,7 @@ def _get_openai_compatible_provider_info( ) model = split_mantle_region_prefix(model)[1] # rebind-ok: the prefix is routing only, not a Mantle model id elif custom_llm_provider == "nvidia_nim": - # nvidia_nim is openai compatible, we just need to set this to custom_openai and have the api_base be https://ap + # nvidia_nim is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.endpoints.anyscale.com/v1 api_base = api_base or get_secret("NVIDIA_NIM_API_BASE") or "https://integrate.api.nvidia.com/v1" dynamic_api_key = api_key or get_secret_str("NVIDIA_NIM_API_KEY") elif custom_llm_provider == "nvidia_riva": @@ -692,11 +692,11 @@ def _get_openai_compatible_provider_info( dynamic_api_key = api_key or get_secret_str("AI21_API_KEY") custom_llm_provider = "ai21_chat" elif custom_llm_provider == "volcengine": - # volcengine is openai compatible, we just need to set this to custom_openai and have the api_base be https://ap + # volcengine is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.endpoints.anyscale.com/v1 api_base = api_base or get_secret("VOLCENGINE_API_BASE") or "https://ark.cn-beijing.volces.com/api/v3" dynamic_api_key = api_key or get_secret_str("VOLCENGINE_API_KEY") elif custom_llm_provider == "codestral": - # codestral is openai compatible, we just need to set this to custom_openai and have the api_base be https://cod + # codestral is openai compatible, we just need to set this to custom_openai and have the api_base be https://codestral.mistral.ai/v1 api_base = api_base or get_secret("CODESTRAL_API_BASE") or "https://codestral.mistral.ai/v1" dynamic_api_key = api_key or get_secret_str("CODESTRAL_API_KEY") elif custom_llm_provider == "hosted_vllm": @@ -724,7 +724,7 @@ def _get_openai_compatible_provider_info( dynamic_api_key, ) = litellm.LMStudioChatConfig().get_openai_compatible_provider_info(api_base, api_key) elif custom_llm_provider == "deepseek": - # deepseek is openai compatible, we just need to set this to custom_openai and have the api_base be https://api + # deepseek is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.deepseek.com/v1 api_base = api_base or get_secret("DEEPSEEK_API_BASE") or "https://api.deepseek.com/beta" dynamic_api_key = api_key or get_secret_str("DEEPSEEK_API_KEY") @@ -733,7 +733,7 @@ def _get_openai_compatible_provider_info( dynamic_api_key = api_key or get_secret_str("TENCENT_API_KEY") elif custom_llm_provider == "fireworks_ai": - # fireworks is openai compatible, we just need to set this to custom_openai and have the api_base be https://api + # fireworks is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.fireworks.ai/inference/v1 ( api_base, dynamic_api_key, diff --git a/litellm/llms/github_copilot/responses/transformation.py b/litellm/llms/github_copilot/responses/transformation.py index 471a8b292468..d468889b6263 100644 --- a/litellm/llms/github_copilot/responses/transformation.py +++ b/litellm/llms/github_copilot/responses/transformation.py @@ -358,9 +358,9 @@ def _get_initiator(self, input_param: str | ResponseInputParam) -> str: return "user" # If input is a list, analyze items - if isinstance(input_param, list): # pyright: ignore[reportUnnecessaryIsInstance] # items arrive from untyped r + if isinstance(input_param, list): # pyright: ignore[reportUnnecessaryIsInstance] # items arrive from untyped request params for item in input_param: - if not isinstance(item, dict): # pyright: ignore[reportUnnecessaryIsInstance] # items arrive from unty + if not isinstance(item, dict): # pyright: ignore[reportUnnecessaryIsInstance] # items arrive from untyped request params continue # Check if item has no role (agent-initiated) @@ -369,7 +369,7 @@ def _get_initiator(self, input_param: str | ResponseInputParam) -> str: # Check if role is assistant (agent-initiated) role = item.get("role") - if isinstance(role, str) and role.lower() == "assistant": # pyright: ignore[reportUnnecessaryIsInstance] # role + if isinstance(role, str) and role.lower() == "assistant": # pyright: ignore[reportUnnecessaryIsInstance] # role arrives from untyped request params return "agent" # Default to user-initiated diff --git a/litellm/main.py b/litellm/main.py index 93795f7e0be2..36ce503e525f 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1481,7 +1481,7 @@ def _complete_azure(ctx: CompletionDispatchContext) -> _CompletionDispatchResult }, ) - return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contra + return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract def _complete_azure_text(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: @@ -1862,7 +1862,7 @@ def _complete_text_completion_openai( original_response=_response, additional_args={"headers": headers}, ) - return _response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contr + return _response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract def _complete_fireworks_ai( @@ -2141,7 +2141,7 @@ def _complete_groq(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: timeout: Final = ctx.timeout api_base = ( - api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api b + api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there or litellm.api_base or get_secret("GROQ_API_BASE") or "https://api.groq.com/openai/v1" @@ -2150,7 +2150,7 @@ def _complete_groq(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: # set API KEY api_key = ( api_key - or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the + or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there or litellm.groq_key or get_secret("GROQ_API_KEY") ) @@ -2180,7 +2180,7 @@ def _complete_groq(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: headers=headers, encoding=_get_encoding(), api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alph + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements client=client, ) @@ -2426,7 +2426,7 @@ def _complete_aiohttp_openai( timeout: Final = ctx.timeout api_base = ( - api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api b + api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there or litellm.api_base or get_secret("OPENAI_BASE_URL") or get_secret("OPENAI_API_BASE") @@ -2435,7 +2435,7 @@ def _complete_aiohttp_openai( # set API KEY api_key = ( api_key - or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the + or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there or litellm.openai_key or get_secret("OPENAI_API_KEY") ) @@ -2633,13 +2633,13 @@ def _complete_custom_openai( organization or litellm.organization or get_secret("OPENAI_ORGANIZATION") - or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/opena + or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 ) openai.organization = organization # set API KEY api_key = ( api_key - or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the + or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there or litellm.openai_key or get_secret("OPENAI_API_KEY") ) @@ -2754,7 +2754,7 @@ def _complete_custom_openai( additional_args={"headers": outbound_headers}, ) - return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contra + return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract def _complete_mistral(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: @@ -2904,7 +2904,7 @@ def _complete_anthropic_text( headers=headers, encoding=_get_encoding(), api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alph + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements ) @@ -3021,7 +3021,7 @@ def _complete_nlp_cloud(ctx: CompletionDispatchContext) -> _CompletionDispatchRe original_response=response, ) - return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contra + return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract def _complete_aleph_alpha(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: @@ -3059,7 +3059,7 @@ def _complete_aleph_alpha(ctx: CompletionDispatchContext) -> _CompletionDispatch encoding=_get_encoding(), default_max_tokens_to_sample=litellm.max_tokens, api_key=aleph_alpha_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alph + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements ) if "stream" in optional_params and optional_params["stream"] is True: @@ -3070,7 +3070,7 @@ def _complete_aleph_alpha(ctx: CompletionDispatchContext) -> _CompletionDispatch custom_llm_provider="aleph_alpha", logging_obj=logging, ) - return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch + return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract def _complete_cohere_chat(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: @@ -3134,7 +3134,7 @@ def _complete_cohere_chat(ctx: CompletionDispatchContext) -> _CompletionDispatch encoding=_get_encoding(), api_key=cohere_key, provider_config=provider_config, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alph + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements ) @@ -3358,7 +3358,7 @@ def _complete_oobabooga(ctx: CompletionDispatchContext) -> _CompletionDispatchRe custom_llm_provider="oobabooga", logging_obj=logging, ) - return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch + return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract def _complete_databricks(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: @@ -3408,7 +3408,7 @@ def _complete_databricks(ctx: CompletionDispatchContext) -> _CompletionDispatchR headers=headers, encoding=_get_encoding(), api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements client=client, ) except Exception as e: @@ -3538,7 +3538,7 @@ def _complete_openrouter(ctx: CompletionDispatchContext) -> _CompletionDispatchR headers=headers, encoding=_get_encoding(), api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alph + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements client=client, ) ## LOGGING @@ -3643,7 +3643,7 @@ def _complete_vercel_ai_gateway( headers=headers, encoding=_get_encoding(), api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alph + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements client=client, ) ## LOGGING @@ -3959,7 +3959,7 @@ def _complete_vertex_ai(ctx: CompletionDispatchContext) -> _CompletionDispatchRe custom_llm_provider="vertex_ai", logging_obj=logging, ) - return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch + return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract def _complete_predibase(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: @@ -4067,8 +4067,8 @@ def _complete_text_completion_codestral( ) if "stream" in optional_params and optional_params["stream"] is True and acompletion is False: - return _model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the disp - return _model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch + return _model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + return _model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract def _complete_text_completion_inception( @@ -4132,7 +4132,7 @@ def _complete_text_completion_inception( original_response=_response, additional_args={"headers": headers}, ) - return _response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contr + return _response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract def _complete_sagemaker_chat( @@ -4167,7 +4167,7 @@ def _complete_sagemaker_chat( headers=headers, encoding=_get_encoding(), api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alph + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements client=client, ) @@ -4308,7 +4308,7 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu headers=headers, encoding=_get_encoding(), api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements client=client, ) else: @@ -4442,7 +4442,7 @@ def _complete_watsonx_text( headers=headers, encoding=_get_encoding(), api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alph + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements client=client, ) @@ -4519,7 +4519,7 @@ def _complete_ollama(ctx: CompletionDispatchContext) -> _CompletionDispatchResul headers=headers, encoding=_get_encoding(), api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alph + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements client=client, ) @@ -4561,7 +4561,7 @@ def _complete_ollama_chat(ctx: CompletionDispatchContext) -> _CompletionDispatch headers=headers, encoding=_get_encoding(), api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alph + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements client=client, ) @@ -4637,7 +4637,7 @@ def _complete_cloudflare(ctx: CompletionDispatchContext) -> _CompletionDispatchR headers=headers, encoding=_get_encoding(), api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alph + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements ) @@ -5068,7 +5068,7 @@ def _complete_custom_providers( logging_obj=logging, ) - return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contra + return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract def _complete_langgraph(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: @@ -5354,7 +5354,7 @@ def completion( # Cast tools to Optional[Iterable[ToolParam]] for type checking tools_for_mcp: Final = cast(Iterable[ToolParam] | None, tools) if LiteLLM_Proxy_MCP_Handler.should_use_litellm_mcp_gateway(tools=tools_for_mcp): - return acompletion_with_mcp( # pyright: ignore[reportReturnType] # MCP path returns a coroutine that acomp + return acompletion_with_mcp( # pyright: ignore[reportReturnType] # MCP path returns a coroutine that acompletion() awaits; completion()'s sync return type omits it model=model, messages=messages, functions=functions, @@ -5539,12 +5539,12 @@ def completion( logging: Final[LiteLLMLoggingObj] = cast(LiteLLMLoggingObj, litellm_logging_obj) fallbacks = fallbacks or litellm.model_fallbacks if fallbacks is not None: - return completion_with_fallbacks( # pyright: ignore[reportReturnType] # fallback runner is untyped; resolv + return completion_with_fallbacks( # pyright: ignore[reportReturnType] # fallback runner is untyped; resolves to ModelResponse|CustomStreamWrapper at runtime **args ) if model_list is not None: deployments: Final = [m["litellm_params"] for m in model_list if m["model_name"] == model] - return litellm.batch_completion_models( # pyright: ignore[reportReturnType] # batch path returns a list of + return litellm.batch_completion_models( # pyright: ignore[reportReturnType] # batch path returns a list of responses, outside completion()'s single-response return type deployments=deployments, **args ) if litellm.model_alias_map and model in litellm.model_alias_map: @@ -5836,7 +5836,7 @@ def completion( logging.update_environment_variables( model=model, user=user, - optional_params=processed_non_default_params, # [IMPORTANT] - using processed_non_default_params ensures co + optional_params=processed_non_default_params, # [IMPORTANT] - using processed_non_default_params ensures consistent params logged to langfuse for finetuning / eval datasets. litellm_params=litellm_params, custom_llm_provider=custom_llm_provider, ) @@ -5901,7 +5901,7 @@ def completion( else: optional_params["reasoning_effort"] = {"summary": rs_val} - return responses_api_bridge.completion( # pyright: ignore[reportReturnType] # bridge returns a coroutine o + return responses_api_bridge.completion( # pyright: ignore[reportReturnType] # bridge returns a coroutine on the acompletion path; awaited by the async caller model=model, messages=messages, headers=headers, @@ -6054,7 +6054,7 @@ def completion( elif custom_llm_provider == "mistral": response = _complete_mistral(_dispatch_ctx) elif "replicate" in model or custom_llm_provider == "replicate" or model in litellm.replicate_models: - # Setting the relevant API KEY for replicate, replicate defaults to using os.environ.get("REPLICATE_API_TOKE + # Setting the relevant API KEY for replicate, replicate defaults to using os.environ.get("REPLICATE_API_TOKEN") response = _complete_replicate(_dispatch_ctx) elif "clarifai" in model or custom_llm_provider == "clarifai" or model in litellm.clarifai_models: pass # Deprecated - handled in the openai compatible provider section above @@ -6628,7 +6628,7 @@ def embedding( openai.organization = ( litellm.organization or get_secret_str("OPENAI_ORGANIZATION") - or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088d + or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 ) # set API KEY api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY") @@ -7241,14 +7241,14 @@ def embedding( from litellm.llms.azure_ai.common_utils import get_azure_ai_entra_token api_base = ( - api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in t + api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there or litellm.api_base or get_secret_str("AZURE_AI_API_BASE") ) # set API KEY api_key = ( api_key - or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass + or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there or get_secret_str("AZURE_AI_API_KEY") or get_azure_ai_entra_token(litellm_params=litellm_params_dict) ) @@ -7639,7 +7639,7 @@ def text_completion( text_completion_response = TextCompletionResponse() optional_params: Final[dict[str, Any]] = {} - # default values for all optional params are none, litellm only passes them to the llm when they are set to non None + # default values for all optional params are none, litellm only passes them to the llm when they are set to non None values if best_of is not None: optional_params["best_of"] = best_of if echo is not None: @@ -7767,7 +7767,7 @@ def process_prompt(i, individual_prompt): if _model is not None and ( custom_llm_provider == "openai" - ): # for openai compatible endpoints - e.g. vllm, call the native /v1/completions endpoint for text completion call + ): # for openai compatible endpoints - e.g. vllm, call the native /v1/completions endpoint for text completion calls if _model not in litellm.open_ai_chat_completion_models: model = "text-completion-openai/" + _model optional_params.pop("custom_llm_provider", None) @@ -7932,7 +7932,7 @@ async def amoderation( api_key=optional_params.api_key, ) except litellm.BadRequestError: - # `model` is optional field for moderation - get_llm_provider will throw BadRequestError if model is not set / n + # `model` is optional field for moderation - get_llm_provider will throw BadRequestError if model is not set / not recognized pass openai_client: Final = kwargs.get("client", None) @@ -8182,7 +8182,7 @@ def transcription( openai.organization = ( litellm.organization or get_secret("OPENAI_ORGANIZATION") - or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/o + or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 ) # set API KEY @@ -8461,7 +8461,7 @@ def speech( llm_provider=custom_llm_provider, ) api_base = ( - api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the a + api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there or litellm.api_base or get_secret("OPENAI_BASE_URL") or get_secret("OPENAI_API_BASE") @@ -8470,7 +8470,7 @@ def speech( # set API KEY api_key = ( api_key - or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key + or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there or litellm.openai_key or get_secret("OPENAI_API_KEY") ) @@ -8479,14 +8479,14 @@ def speech( organization or litellm.organization or get_secret("OPENAI_ORGANIZATION") - or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/o + or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 ) project = ( project or litellm.project or get_secret("OPENAI_PROJECT") - or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/o + or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 ) headers = headers or litellm.headers @@ -8992,7 +8992,7 @@ def stream_chunk_builder_text_completion(chunks: list, messages: Sequence | None completion_tokens: Final = token_counter( model=model, text=combined_content, - count_response_tokens=True, # count_response_tokens is a Flag to tell token counter this is a response, No need + count_response_tokens=True, # count_response_tokens is a Flag to tell token counter this is a response, No need to add extra tokens we do for input messages ) response: Final = { diff --git a/litellm/models/credentials.py b/litellm/models/credentials.py index b9f4b9a9b6ae..d569ea126173 100644 --- a/litellm/models/credentials.py +++ b/litellm/models/credentials.py @@ -22,11 +22,9 @@ class CredentialBase(LiteLLMBaseModel): class CredentialItem(CredentialBase): - credential_values: ( - dict[ # mutable-ok: credential values are decrypted and merged in place by the credential endpoints - str, object - ] - ) + credential_values: dict[ # mutable-ok: values are decrypted and merged in place + str, object + ] # PATCH-only instruction naming keys to drop from the stored credential_values. It describes an # edit rather than the credential, so it stays out of dumps: those feed config loading, the DB # write, and the in-memory list, none of which have a place for it. diff --git a/litellm/proxy/common_utils/credential_hydration.py b/litellm/proxy/common_utils/credential_hydration.py index 4f22e921022b..56bf4ef48d09 100644 --- a/litellm/proxy/common_utils/credential_hydration.py +++ b/litellm/proxy/common_utils/credential_hydration.py @@ -78,7 +78,7 @@ def _decrypted(db_credential: CredentialItem) -> CredentialItem: ) return CredentialItem( credential_name=db_credential.credential_name, - credential_values=decrypted_values, # pyright: ignore[reportArgumentType] # declared dict[str, str], and pydan + credential_values=decrypted_values, # pyright: ignore[reportArgumentType] # declared dict[str, str], and pydantic copies this mapping into one on validation; LIT002 rules out building that dict here credential_info=db_credential.credential_info, ) diff --git a/litellm/proxy/credential_endpoints/endpoints.py b/litellm/proxy/credential_endpoints/endpoints.py index 4cff0d4eab72..83af63bf24a6 100644 --- a/litellm/proxy/credential_endpoints/endpoints.py +++ b/litellm/proxy/credential_endpoints/endpoints.py @@ -394,7 +394,7 @@ async def get_credential_by_name( ) async def get_credential_internal_issuer_jwks( credential_name: str = Path(..., description="The credential name, percent-decoded; may contain slashes"), - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI resolves the dependency fro + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI resolves the dependency from the default ): """ Export the public JWKS for an anthropic ``internal_issuer`` credential, so the operator can @@ -527,7 +527,7 @@ def _per_user_credential_names() -> tuple[str, ...]: async def list_user_connections( request: Request, fastapi_response: Response, - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI resolves the dependency fro + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI resolves the dependency from the default ) -> UserProviderConnectionsResponse: """List the calling user's per-user provider connections.""" from litellm.proxy.proxy_server import prisma_client @@ -593,7 +593,7 @@ async def start_user_connection( request: Request, fastapi_response: Response, credential_name: str = Path(..., description="The credential name, percent-decoded"), - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI resolves the dependency fro + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI resolves the dependency from the default ) -> UserConnectionStartResponse: """Begin a GitHub device flow for the calling user's connection to a per-user credential.""" from litellm.llms.github_copilot.per_user_auth import astart_device_flow @@ -640,7 +640,7 @@ async def poll_user_connection( fastapi_response: Response, body: UserConnectionPollRequest, credential_name: str = Path(..., description="The credential name, percent-decoded"), - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI resolves the dependency fro + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI resolves the dependency from the default ) -> UserConnectionPollResponse: """Poll the device flow once and persist the connection on completion.""" from litellm.llms.github_copilot.per_user_auth import ( @@ -715,7 +715,7 @@ async def delete_user_connection( request: Request, fastapi_response: Response, credential_name: str = Path(..., description="The credential name, percent-decoded"), - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI resolves the dependency fro + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI resolves the dependency from the default ) -> UserConnectionDeleteResponse: """Disconnect the calling user's stored GitHub token for a per-user credential. Idempotent.""" from litellm.llms.github_copilot.per_user_auth import evict_copilot_user_session diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 1c92028219e2..eb86c96660bf 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -75,7 +75,7 @@ strip_callback_config, ) from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # legacy module exports - _safe_get_request_headers, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package + _safe_get_request_headers, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export safe_get_request_headers, ) from litellm.proxy.spend_tracking.carried_budget_state import carried_budget_metadata @@ -2423,7 +2423,7 @@ async def add_litellm_data_to_request( if key not in data[_metadata_variable_name]: data[_metadata_variable_name][key] = value if _metadata_variable_name == "metadata": - data["metadata"]["tags"] = LiteLLMProxyRequestSetup._merge_tags( # pyright: ignore[reportPrivateUsage] # same + data["metadata"]["tags"] = LiteLLMProxyRequestSetup._merge_tags( # pyright: ignore[reportPrivateUsage] # same-module helper, budget blocks the unsuppressed idiom sibling call sites use request_tags=data["metadata"].get("tags"), tags_to_add=data["litellm_metadata"].get("tags"), ) @@ -2655,7 +2655,7 @@ async def add_litellm_data_to_request( verbose_proxy_logger.debug( "[PROXY] returned data from litellm_pre_call_utils: %s", - cast("object", data), # cast-ok: data is the untyped request body dict + data, ) # Team/Project credential overrides from model_config @@ -2826,9 +2826,9 @@ def _index_fallback_list(fallback_list: Sequence[object]) -> _FallbackIndex: if isinstance(key, str) and not ("*" in key or "/" in key): exact[key] = exact.get(key, ()) + targets else: - fuzzy.append(cast("object", entry)) # cast-ok: fuzzy holds mixed entry shapes + fuzzy.append(entry) else: - fuzzy.append(cast("object", entry)) # cast-ok: fuzzy holds mixed entry shapes + fuzzy.append(entry) return exact, tuple(fuzzy) @@ -2952,7 +2952,7 @@ def _per_user_credential_names_for_groups( lp_map.get("model") if lp_map is not None else getattr( - cast("object", litellm_params), # cast-ok: litellm_params is Mapping or an object + litellm_params, "model", None, ) @@ -2963,7 +2963,7 @@ def _per_user_credential_names_for_groups( lp_map.get("litellm_credential_name") if lp_map is not None else getattr( - cast("object", litellm_params), # cast-ok: litellm_params is Mapping or an object + litellm_params, "litellm_credential_name", None, ) @@ -3101,7 +3101,7 @@ async def _resolve_user_provider_credentials_for_request( user_id=user_id, credential_names=credential_names, ) - except Exception: # noqa: BLE001 # credential lookup failures degrade to the shared deployment, never break the re + except Exception: # noqa: BLE001 # credential lookup failures degrade to the shared deployment, never break the request verbose_proxy_logger.exception( "_resolve_user_provider_credentials_for_request: failed to load user provider credentials" ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 0af83e1707b0..821669933b74 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -133,7 +133,7 @@ strip_callback_config, ) from litellm.proxy.common_utils.realtime_utils import ( # noqa: F401, RUF100 # legacy module exports - _realtime_request_body, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package ex + _realtime_request_body, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export realtime_request_body, ) from litellm.proxy.management_helpers.auto_router_availability import AutoRouterCatalogEntry, build_auto_router_catalog @@ -388,7 +388,7 @@ def generate_feedback_box(): ) from litellm.proxy.auth.password_policy import validate_password_not_breached, validate_password_policy from litellm.proxy.auth.user_api_key_auth import ( # noqa: F401, RUF100 # legacy module exports - _fetch_global_spend_with_event_coordination, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards- + _fetch_global_spend_with_event_coordination, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export fetch_global_spend_with_event_coordination, user_api_key_auth, user_api_key_auth_websocket, @@ -401,8 +401,8 @@ def generate_feedback_box(): from litellm.proxy.common_request_processing import ( # noqa: F401, RUF100 # legacy module exports KNOWN_PROXY_ROUTES, ProxyBaseLLMRequestProcessing, - _is_azure_model_router_request, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible pa - _should_return_raw_model_name, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible pac + _is_azure_model_router_request, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export + _should_return_raw_model_name, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export close_guarded_stream, create_response, is_azure_model_router_request, @@ -445,7 +445,7 @@ def generate_feedback_box(): from litellm.proxy.common_utils.html_forms.ui_login import build_ui_login_form from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401, RUF100 # legacy module exports _read_request_body, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export - _safe_get_request_headers, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package + _safe_get_request_headers, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export check_file_size_under_limit, get_form_data, read_request_body, @@ -580,12 +580,12 @@ def generate_feedback_box(): from litellm.proxy.health_endpoints._health_endpoints import router as health_router from litellm.proxy.hooks.model_max_budget_limiter import ( # noqa: F401, RUF100 # legacy module exports PROXY_VirtualKeyModelMaxBudgetLimiter, - _PROXY_VirtualKeyModelMaxBudgetLimiter, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compa + _PROXY_VirtualKeyModelMaxBudgetLimiter, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export ) from litellm.proxy.hooks.parallel_request_limiter_v3 import fail_closed_rate_limit_enforcement_enabled from litellm.proxy.hooks.prompt_injection_detection import ( # noqa: F401, RUF100 # legacy module exports OPTIONAL_PromptInjectionDetection, - _OPTIONAL_PromptInjectionDetection, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatibl + _OPTIONAL_PromptInjectionDetection, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export ) from litellm.proxy.hooks.proxy_track_cost_callback import ( # noqa: F401, RUF100 # legacy module exports ProxyDBLogger, @@ -628,8 +628,8 @@ def generate_feedback_box(): router as callback_management_endpoints_router, ) from litellm.proxy.management_endpoints.common_utils import ( # noqa: F401, RUF100 # legacy module exports - _user_has_admin_privileges, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible packag - _user_has_admin_view, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package expo + _user_has_admin_privileges, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export + _user_has_admin_view, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export admin_can_invite_user, user_api_key_has_admin_view, user_has_admin_privileges, @@ -675,10 +675,10 @@ def generate_feedback_box(): from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( router as model_access_group_management_router, ) -from litellm.proxy.management_endpoints.model_management_endpoints import ( # noqa: F401, RUF100 # legacy module expor +from litellm.proxy.management_endpoints.model_management_endpoints import ( # noqa: F401, RUF100 # legacy module exports _add_model_to_db, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export - _add_team_model_to_db, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package exp - _deduplicate_litellm_router_models, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatibl + _add_team_model_to_db, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export + _deduplicate_litellm_router_models, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export add_model_to_db, add_team_model_to_db, deduplicate_litellm_router_models, @@ -869,10 +869,10 @@ def generate_feedback_box(): _cache_user_row, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _get_docs_url, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _get_openapi_url, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export - _get_projected_spend_over_limit, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible p + _get_projected_spend_over_limit, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _get_redoc_url, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export - _is_projected_spend_over_limit, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible pa - _is_valid_team_configs, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package ex + _is_projected_spend_over_limit, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export + _is_valid_team_configs, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export cache_user_row, evict_config_param, get_config_param, @@ -1545,7 +1545,7 @@ async def _run_agent_grant_id_migration() -> None: ## when the proxy cache backend is not Redis ## transaction_buffer_redis_cache = redis_usage_cache if transaction_buffer_redis_cache is None: - transaction_buffer_redis_cache = ProxyStartupEvent.get_transaction_buffer_redis_cache( # rebind-ok: pre-existin + transaction_buffer_redis_cache = ProxyStartupEvent.get_transaction_buffer_redis_cache( # rebind-ok: pre-existing rebinding on a rename-only line general_settings=general_settings ) @@ -1790,7 +1790,7 @@ async def _run_agent_grant_id_migration() -> None: await _drain_spend_event_producer_on_shutdown() - # Shutdown event - finish or cancel in-flight scheduled jobs before the shutdown flushes and the DB disconne + # Shutdown event - finish or cancel in-flight scheduled jobs before the shutdown flushes and the DB disconnect if scheduler is not None and scheduler_executor is not None: try: await stop_in_flight_scheduler_jobs(scheduler, scheduler_executor) @@ -6862,7 +6862,7 @@ async def load_config(self, router: litellm.Router | None, config_file_path: str if env_coordination_redis_cache is not None: _set_redis_usage_cache(env_coordination_redis_cache) - ## GENERAL SERVER SETTINGS (e.g. master key,..) # do this after initializing litellm, to ensure sentry logging w + ## GENERAL SERVER SETTINGS (e.g. master key,..) # do this after initializing litellm, to ensure sentry logging works for proxylogging general_settings = config.get("general_settings", {}) if general_settings is None: general_settings = {} @@ -8782,11 +8782,11 @@ async def _init_cyberark_config_override(self, prisma_client: PrismaClient) -> N """ from litellm.proxy.management_endpoints.config_override_endpoints import ( CYBERARK_ENV_VAR_MAPPING, - _clear_cyberark_state, # pyright: ignore[reportPrivateUsage] # module-internal helper shared with the endp - _snapshot_cyberark_boot_env, # pyright: ignore[reportPrivateUsage] # module-internal helper shared with th - get_current_env_values, # pyright: ignore[reportPrivateUsage] # module-internal helper shared with the end - parse_config_value, # pyright: ignore[reportPrivateUsage] # module-internal helper shared with the endpoin - set_env_vars, # pyright: ignore[reportPrivateUsage] # module-internal helper shared with the endpoint modu + _clear_cyberark_state, # pyright: ignore[reportPrivateUsage] # module-internal helper shared with the endpoint module + _snapshot_cyberark_boot_env, # pyright: ignore[reportPrivateUsage] # module-internal helper shared with the endpoint module + get_current_env_values, # pyright: ignore[reportPrivateUsage] # module-internal helper shared with the endpoint module + parse_config_value, # pyright: ignore[reportPrivateUsage] # module-internal helper shared with the endpoint module + set_env_vars, # pyright: ignore[reportPrivateUsage] # module-internal helper shared with the endpoint module ) try: @@ -9010,7 +9010,7 @@ async def _init_prompts_in_db(self, prisma_client: PrismaClient): def parse_row(db_prompt: object) -> PromptSpec | None: try: return self._get_prompt_spec_for_db_prompt(db_prompt=db_prompt) - except Exception as row_error: # noqa: BLE001 # a malformed row must not block syncing the remaining promp + except Exception as row_error: # noqa: BLE001 # a malformed row must not block syncing the remaining prompts verbose_proxy_logger.exception( "litellm.proxy.proxy_server.py::ProxyConfig:_init_prompts_in_db - failed to parse prompt row %s: %s", getattr(db_prompt, "prompt_id", None), @@ -9027,7 +9027,7 @@ def parse_row(db_prompt: object) -> PromptSpec | None: for prompt_spec in parsed_specs: try: IN_MEMORY_PROMPT_REGISTRY.sync_prompt_from_db(prompt=prompt_spec) - except Exception as prompt_sync_error: # noqa: BLE001 # one poisoned row must not block syncing the re + except Exception as prompt_sync_error: # noqa: BLE001 # one poisoned row must not block syncing the remaining prompts verbose_proxy_logger.exception( "litellm.proxy.proxy_server.py::ProxyConfig:_init_prompts_in_db - failed to sync prompt %s: %s", prompt_spec.prompt_id, @@ -10834,7 +10834,7 @@ async def _load_heuristic_v1_tuning_baselines( ) } ) - except Exception as e: # noqa: BLE001 # enforcement is skipped for this boot; refusing every tuned router on a + except Exception as e: # noqa: BLE001 # enforcement is skipped for this boot; refusing every tuned router on a DB blip is the one outcome the gate forbids verbose_proxy_logger.warning("Heuristic-v1 tuning baseline unavailable, gate not enforced this boot: %s", e) return None @@ -11592,7 +11592,7 @@ async def _scheduled_monthly_spend_report() -> None: slack_alerting_args: Final = proxy_logging_obj.slack_alerting_instance.alerting_args user_spend_check_interval: Final = ( slack_alerting_args.user_spend_check_interval - if isinstance(slack_alerting_args, SlackAlertingArgs) # pyright: ignore[reportUnnecessaryIsInstance] # test + if isinstance(slack_alerting_args, SlackAlertingArgs) # pyright: ignore[reportUnnecessaryIsInstance] # tests inject a mock slack_alerting_instance else SlackAlertingArgs().user_spend_check_interval ) @@ -11852,7 +11852,7 @@ async def _deployment_hidden_by_listing_callbacks(deployment: Deployment, user_a "/models", dependencies=[Depends(user_api_key_auth)], tags=["model management"] ) # if project requires model list async def model_list( - request: Request = None, # pyright: ignore[reportArgumentType] # FastAPI always injects the Request; the None defa + request: Request = None, # pyright: ignore[reportArgumentType] # FastAPI always injects the Request; the None default only serves direct in-process callers user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), return_wildcard_routes: bool | None = False, team_id: str | None = None, @@ -13365,7 +13365,7 @@ async def return_body(): code=1011, reason=websocket_close_reason(redacted_error, fallback="Internal server error"), ) - except Exception: # noqa: BLE001 # the lower layer may have closed the socket already; closing twice is not an + except Exception: # noqa: BLE001 # the lower layer may have closed the socket already; closing twice is not an error verbose_proxy_logger.debug("Could not close realtime client websocket; it is already gone") finally: if not litellm_logging_obj.model_call_details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY): @@ -20464,7 +20464,7 @@ async def body_iter(): ) async def proxy_mcp_route(request: Request) -> Response: """Serve the fixed three-tool MCP proxy surface.""" - from litellm.proxy._experimental.mcp_server.mcp_context import ( # pyright: ignore[reportPrivateUsage] # route-own + from litellm.proxy._experimental.mcp_server.mcp_context import ( # pyright: ignore[reportPrivateUsage] # route-owned mode _mcp_proxy_mode, # pyright: ignore[reportPrivateUsage] # route-owned mode ) from litellm.proxy._experimental.mcp_server.server import handle_streamable_http_mcp diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 00369923d577..f6bd4bfba546 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -796,7 +796,7 @@ async def aresponses( # (mirrors litellm/main.py:1371 for chat completions) response.hidden_params["custom_llm_provider"] = custom_llm_provider - if response is None: # pyright: ignore[reportUnnecessaryComparison] # provider handlers can return None at run + if response is None: # pyright: ignore[reportUnnecessaryComparison] # provider handlers can return None at runtime raise ValueError(f"Got an unexpected None response from the Responses API: {response}") return response @@ -1736,7 +1736,7 @@ async def aget_responses( response = init_response # Update the responses_api_response_id with the model_id - if not isinstance(response, ResponsesAPIResponse): # pyright: ignore[reportUnnecessaryIsInstance] # handlers c + if not isinstance(response, ResponsesAPIResponse): # pyright: ignore[reportUnnecessaryIsInstance] # handlers can return non-ResponsesAPIResponse objects at runtime return response return ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id( responses_api_response=response, @@ -2211,7 +2211,7 @@ async def acompact_responses( response = init_response # Update the responses_api_response_id with the model_id - if not isinstance(response, ResponsesAPIResponse): # pyright: ignore[reportUnnecessaryIsInstance] # handlers c + if not isinstance(response, ResponsesAPIResponse): # pyright: ignore[reportUnnecessaryIsInstance] # handlers can return non-ResponsesAPIResponse objects at runtime return response return ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id( responses_api_response=response, diff --git a/litellm/router.py b/litellm/router.py index fcf1b982784f..788b0ef9236c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -204,7 +204,7 @@ from litellm.router_utils.cooldown_cache import CooldownCache from litellm.router_utils.cooldown_handlers import ( DEFAULT_COOLDOWN_TIME_SECONDS, - _first_present, # pyright: ignore[reportPrivateUsage] - shared internal helper across router_utils submodules, matc + _first_present, # pyright: ignore[reportPrivateUsage] - shared internal helper across router_utils submodules, matching the other cooldown_handlers imports on this line async_get_cooldown_deployments, async_get_cooldown_deployments_with_debug_info, get_cooldown_deployments, @@ -791,7 +791,7 @@ def follow_source_attribution(self) -> None: if source_params is None or source_params is self._followed_source_params: return self._followed_source_params = source_params - hidden_params, headers = Router._prepare_fallback_hidden_params(source) # pyright: ignore[reportPrivateUsage] # this + hidden_params, headers = Router._prepare_fallback_hidden_params(source) # pyright: ignore[reportPrivateUsage] # this wrapper is the Router's own stream type self.merge_fallback_hidden_params(hidden_params, headers) def __aiter__(self) -> "FallbackAwareAnthropicMessagesStream": @@ -1065,7 +1065,7 @@ def __init__( from litellm._service_logger import ServiceLogging self.service_logger_obj: ServiceLogging = ServiceLogging() - litellm.suppress_debug_info = True # prevents 'Give Feedback/Get help' message from being emitted on Router - R + litellm.suppress_debug_info = True # prevents 'Give Feedback/Get help' message from being emitted on Router - Relevant Issue: https://github.com/BerriAI/litellm/issues/5942 if self.set_verbose is True: if debug_level == "INFO": verbose_router_logger.setLevel(logging.INFO) @@ -1141,7 +1141,7 @@ def __init__( # Initialize model_group_alias early since it's used in set_model_list self.model_group_alias: dict[str, str | RouterModelGroupAliasItem] = ( model_group_alias or {} - ) # dict to store aliases for router, ex. {"gpt-4": "gpt-3.5-turbo"}, all requests with gpt-4 -> get routed to + ) # dict to store aliases for router, ex. {"gpt-4": "gpt-3.5-turbo"}, all requests with gpt-4 -> get routed to gpt-3.5-turbo group # Initialize model ID to deployment index mapping for O(1) lookups self.model_id_to_deployment_index_map: dict[str, int] = {} @@ -1190,7 +1190,7 @@ def __init__( if "model" in m["litellm_params"]: self.deployment_latency_map[m["litellm_params"]["model"]] = 0 else: - self.model_list: list = [] # initialize an empty list - to allow _add_deployment and delete_deployment to w + self.model_list: list = [] # initialize an empty list - to allow _add_deployment and delete_deployment to work if allowed_fails is not None: self.allowed_fails = allowed_fails @@ -3588,7 +3588,7 @@ async def _aresponses_fallback_attempt( partial_usage: Final = Router._extract_partial_responses_usage(source_iterator) fallback_response = None # rebind-ok: pre-init so finally can close it if a fallback was actually attempted - fallback_yielded = False # rebind-ok: flipped on the first fallback item so a fallback that dies before its fir + fallback_yielded = False # rebind-ok: flipped on the first fallback item so a fallback that dies before its first event still replays the primary's held announcement try: model_group: Final = cast(str, initial_kwargs.get("model")) # cast-ok: model group fallbacks: Final[list | None] = initial_kwargs.get( # mutable-ok: matches the common_utils list|None param @@ -4069,7 +4069,7 @@ def _stamp_retry_skip_deployment_id(exception: Exception, kwargs: Mapping[str, o effective_model_info: Final = kwargs.get("model_info") deployment_id: Final = effective_model_info.get("id") if isinstance(effective_model_info, Mapping) else None if isinstance(deployment_id, str) and deployment_id: - exception.retry_skip_deployment_id = deployment_id # pyright: ignore[reportAttributeAccessIssue] # dynamic + exception.retry_skip_deployment_id = deployment_id # pyright: ignore[reportAttributeAccessIssue] # dynamic stamp, read by _deployment_ids_to_skip_on_retry def _update_kwargs_with_default_litellm_params( self, kwargs: dict, metadata_variable_name: str | None = "metadata" @@ -5444,7 +5444,7 @@ def _add_deployment_model_to_endpoint_for_llm_passthrough_route( """ if "endpoint" in kwargs and kwargs["endpoint"]: # For provider-specific endpoints, strip the provider prefix from model_name - # e.g., "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0" -> "us.anthropic.claude-3-5-sonnet-20240620-v + # e.g., "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0" -> "us.anthropic.claude-3-5-sonnet-20240620-v1:0" from litellm import get_llm_provider try: @@ -5673,7 +5673,7 @@ async def stream_with_fallbacks() -> AsyncGenerator[bytes, None]: # to take over there is nothing to buffer for, so every frame, # including pings and provider error frames, is forwarded live. model: Final = cast(str, initial_kwargs.get("model")) # cast-ok: kwargs always carries the model group - has_generated_content = not ( # rebind-ok: set once real content is seen, the buffer cap is hit, or neither + has_generated_content = not ( # rebind-ok: set once real content is seen, the buffer cap is hit, or neither a retry nor a fallback can take over self._anthropic_messages_stream_can_retry(initial_kwargs) or self._anthropic_messages_stream_can_fall_back(model, initial_kwargs) ) @@ -5689,8 +5689,8 @@ async def stream_with_fallbacks() -> AsyncGenerator[bytes, None]: # detection parses the accumulated buffer plus the current chunk, never the # chunk alone; the buffer is already capped, which bounds this window too. parse_window = ( - b"".join(c for c in (*buffered_lifecycle_chunks, chunk) if isinstance(c, (bytes, bytearray))) # pyright: ignore[reportUnnecessaryIsInstance] # brid - if not has_generated_content and isinstance(chunk, (bytes, bytearray)) # pyright: ignore[reportUnnecessaryIsInstance] # brid + b"".join(c for c in (*buffered_lifecycle_chunks, chunk) if isinstance(c, (bytes, bytearray))) # pyright: ignore[reportUnnecessaryIsInstance] # bridge-path chunks are not always bytes at runtime + if not has_generated_content and isinstance(chunk, (bytes, bytearray)) # pyright: ignore[reportUnnecessaryIsInstance] # bridge-path chunks are not always bytes at runtime else chunk ) error_event = parse_anthropic_error_event(parse_window) @@ -5962,7 +5962,7 @@ async def _aanthropic_messages_retry_same_group( ) try: response = await self._ageneric_api_call_with_fallbacks_anthropic_messages_attempt(**retry_kwargs) - except Exception as retry_error: # noqa: BLE001 # every failure of a retry before its stream opens is the + except Exception as retry_error: # noqa: BLE001 # every failure of a retry before its stream opens is the fallback chain's to judge wrapped = _anthropic_stream_fallback_error_for_raised(retry_error, model_group, False) if wrapped is None: return _AnthropicStreamRetriesExhausted( @@ -6432,7 +6432,7 @@ async def create_file_for_deployment(deployment: dict) -> OpenAIFileObject: ) if ( "gcs_bucket_name" in data - ): # TODO: Remove this once we have a better way to handle GCS bucket name: Problem is that we need to + ): # TODO: Remove this once we have a better way to handle GCS bucket name: Problem is that we need to pass the gcs_bucket_name to the router for the create_file call but it doesn't show up there kwargs_copy.setdefault("litellm_metadata", {})["gcs_bucket_name"] = data["gcs_bucket_name"] async with self._deployment_slot( deployment=deployment, kwargs=kwargs_copy, parent_otel_span=parent_otel_span @@ -7726,7 +7726,7 @@ async def async_function_with_fallbacks_common_utils( try: verbose_router_logger.info("Trying to fallback b/w models") - # check if client-side fallbacks are used (e.g. fallbacks = ["gpt-3.5-turbo", "claude-3-haiku"] or fallbacks + # check if client-side fallbacks are used (e.g. fallbacks = ["gpt-3.5-turbo", "claude-3-haiku"] or fallbacks=[{"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Hey, how's it going?"}]}] is_non_standard_fallback_format: Final = check_non_standard_fallback_format(fallbacks=fallbacks) if is_non_standard_fallback_format: @@ -7891,7 +7891,7 @@ async def async_function_with_fallbacks(self, *args, **kwargs): ) if compaction_surface is not None: kwargs["_context_compaction_state"] = initialize_compaction_state(kwargs, compaction_surface) - clear_pre_routing_selection(kwargs) # pyright: ignore[reportUnknownArgumentType] # **kwargs is untyped at this + clear_pre_routing_selection(kwargs) # pyright: ignore[reportUnknownArgumentType] # **kwargs is untyped at this boundary if not isinstance(kwargs.get("attempted_targets"), AttemptedFallbackTargets): _fallback_metadata_key: Final = get_router_metadata_variable_name( function_name=getattr(kwargs.get("original_function"), "__name__", None) @@ -10409,7 +10409,7 @@ def _restore_deployment_after_failed_upsert(self, previous_deployment: Deploymen previous_deployment.model_name, model_id, ) - except Exception as restore_error: # noqa: BLE001 # best-effort restore: a second failure must not abort the r + except Exception as restore_error: # noqa: BLE001 # best-effort restore: a second failure must not abort the reload verbose_router_logger.warning( "Could not restore previously served deployment %s (id=%s) after the failed upsert: %s", previous_deployment.model_name, @@ -12437,7 +12437,7 @@ def _deployment_accepts_param(deployment: DeploymentTypedDict, group: str, param custom_llm_provider=custom_llm_provider, base_model=base_model if isinstance(base_model, str) else None, ) - except Exception as e: # noqa: BLE001 # best-effort filter: an unresolvable provider must not narrow the reque + except Exception as e: # noqa: BLE001 # best-effort filter: an unresolvable provider must not narrow the request verbose_router_logger.debug( "litellm.router.py::_deployment_accepts_param: keeping %s for model=%s. Got - %s", param, group, e ) @@ -13056,7 +13056,7 @@ def _pre_call_checks( model=_dep_model_for_params, litellm_params=LiteLLM_Params.model_validate(_litellm_params), ) - except Exception as e: # noqa: BLE001 # best-effort filter: an unresolvable provider must not fail the + except Exception as e: # noqa: BLE001 # best-effort filter: an unresolvable provider must not fail the request verbose_router_logger.debug( "litellm.router.py::_pre_call_checks: skipping supported-params check for model=%s. Got - %s", _dep_model_for_params, @@ -13503,7 +13503,7 @@ async def async_get_healthy_deployments( request_kwargs=request_kwargs, ) - # IF TEAM ID SPECIFIED ON MODEL, AND REQUEST CONTAINS USER_API_KEY_TEAM_ID, FILTER OUT MODELS THAT ARE NOT IN TH + # IF TEAM ID SPECIFIED ON MODEL, AND REQUEST CONTAINS USER_API_KEY_TEAM_ID, FILTER OUT MODELS THAT ARE NOT IN THE TEAM ## THIS PREVENTS WRITING FILES OF OTHER TEAMS TO MODELS THAT ARE TEAM-ONLY MODELS healthy_deployments = filter_team_based_models( healthy_deployments=healthy_deployments, @@ -13799,7 +13799,7 @@ async def _async_get_available_deployment( and self.routing_strategy != "cost-based-routing" and self.routing_strategy != "latency-based-routing" and self.routing_strategy != "least-busy" - ): # prevent regressions for other routing strategies, that don't have async get available deployments implemen + ): # prevent regressions for other routing strategies, that don't have async get available deployments implemented. return self.get_available_deployment( model=model, messages=messages, @@ -14728,7 +14728,7 @@ def get_available_deployment( "usage-based-routing-v2, cost-based-routing, latency-based-routing, least-busy), " "or remove `plugins` from the Router config." ) - # users need to explicitly call a specific deployment, by setting `specific_deployment = True` as completion()/e + # users need to explicitly call a specific deployment, by setting `specific_deployment = True` as completion()/embedding() kwarg # When this was no explicit we had several issues with fallbacks timing out model, healthy_deployments = self._common_checks_available_deployment( diff --git a/litellm/router_utils/clientside_credential_handler.py b/litellm/router_utils/clientside_credential_handler.py index 150904071749..65980449937b 100644 --- a/litellm/router_utils/clientside_credential_handler.py +++ b/litellm/router_utils/clientside_credential_handler.py @@ -120,14 +120,14 @@ def get_dynamic_litellm_params(litellm_params: dict, request_kwargs: dict) -> di # shared mode and send the admin's Copilot token to the redirected host. # Per-user mode ignores the caller's api_base anyway (the session's # validated host wins), so keeping the mode is fail-closed. - credential_name_value: Final[object] = litellm_params.get( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # lite + credential_name_value: Final[object] = litellm_params.get( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # litellm_params is the untyped request dict "litellm_credential_name" ) - auth_type: Final[object] = litellm_params.get( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # lite + auth_type: Final[object] = litellm_params.get( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # litellm_params is the untyped request dict "github_copilot_auth_type" ) per_user_credential_locked: Final = github_copilot_auth_mode( - credential_name_value, # pyright: ignore[reportUnknownArgumentType] # value comes from the untyped request + credential_name_value, # pyright: ignore[reportUnknownArgumentType] # value comes from the untyped request dict auth_type, # pyright: ignore[reportUnknownArgumentType] # value comes from the untyped request dict ) for field in _ADMIN_CONFIG_FIELDS_TO_CLEAR_ON_BASE_OVERRIDE: diff --git a/litellm/utils.py b/litellm/utils.py index 205288ec0877..2f1ee5d896d2 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -149,7 +149,7 @@ def _snapshot_exception_for_hook(exception: Exception) -> Exception: # behavior for `raise ... from ...`), so this must be set after, not before. snapshot.__suppress_context__ = exception.__suppress_context__ return snapshot - except Exception: # noqa: BLE001 # any snapshot failure must fall back to the live object, not break the failure p + except Exception: # noqa: BLE001 # any snapshot failure must fall back to the live object, not break the failure path return exception @@ -1413,7 +1413,7 @@ async def _client_async_logging_helper( ): if ( is_completion_with_fallbacks is False - ): # don't log the parent event litellm.completion_with_fallbacks as a 'log_success_event', this will lead to doubl + ): # don't log the parent event litellm.completion_with_fallbacks as a 'log_success_event', this will lead to double logging the same call - https://github.com/BerriAI/litellm/issues/7477 print_verbose( f"Async Wrapper: Completed Call, calling async_success_handler: {logging_obj.async_success_handler}" ) @@ -1630,7 +1630,7 @@ def post_call_processing( and optional_params["response_format"].get("json_schema") is not None ): json_response_format = optional_params["response_format"] - elif _parsing._completions.is_basemodel_type( # pyright: ignore[reportPrivateUsage] # Open + elif _parsing._completions.is_basemodel_type( # pyright: ignore[reportPrivateUsage] # OpenAI parser helper is private optional_params["response_format"] ): json_response_format = type_to_response_format_param( @@ -1742,7 +1742,7 @@ def wrapper(*args, **kwargs): load_credentials_from_list(kwargs) from litellm.llms.github_copilot.per_user_auth import attach_github_copilot_user_session - attach_github_copilot_user_session(kwargs) # pyright: ignore[reportUnknownArgumentType] # kwargs is the un + attach_github_copilot_user_session(kwargs) # pyright: ignore[reportUnknownArgumentType] # kwargs is the untyped request dict kwargs["litellm_logging_obj"] = logging_obj LLMCachingHandler: Final = _get_cached_llm_caching_handler() _llm_caching_handler: Final[LLMCachingHandler] = LLMCachingHandler( @@ -1898,7 +1898,7 @@ def wrapper(*args, **kwargs): end_time=end_time, ) - # LOG SUCCESS - handle streaming success logging in the _next_ object, remove `handle_success` once it's dep + # LOG SUCCESS - handle streaming success logging in the _next_ object, remove `handle_success` once it's deprecated verbose_logger.info("Wrapper: Completed Call, calling success_handler") if not is_internal_call.get(): # Copy the current context to propagate it to the background thread @@ -1978,7 +1978,7 @@ def wrapper(*args, **kwargs): traceback_exception: Final = traceback.format_exc() end_time = datetime.datetime.now() - # LOG FAILURE - handle streaming failure logging in the _next_ object, remove `handle_failure` once it's dep + # LOG FAILURE - handle streaming failure logging in the _next_ object, remove `handle_failure` once it's deprecated if logging_obj: logging_obj.failure_handler( e, traceback_exception, start_time, end_time @@ -2033,7 +2033,7 @@ async def wrapper_async(*args, **kwargs): load_credentials_from_list(kwargs) from litellm.llms.github_copilot.per_user_auth import aattach_github_copilot_user_session - await aattach_github_copilot_user_session(kwargs) # pyright: ignore[reportUnknownArgumentType] # kwargs is + await aattach_github_copilot_user_session(kwargs) # pyright: ignore[reportUnknownArgumentType] # kwargs is the untyped request dict logging_obj.llm_caching_handler = _llm_caching_handler # [OPTIONAL] CHECK BUDGET if litellm.max_budget: @@ -2117,14 +2117,14 @@ async def wrapper_async(*args, **kwargs): try: result = await original_function(*args, **call_kwargs) except Exception as deployment_error: - _deployment_call_end_time = datetime.datetime.now() # noqa: DTZ005 # matches the naive datetimes this + _deployment_call_end_time = datetime.datetime.now() # noqa: DTZ005 # matches the naive datetimes this whole function already times start_time/end_time with try: await async_post_call_failure_deployment_hook( request_data=call_kwargs, exception=deployment_error, call_type=call_type, ) - except BaseException: # noqa: S110, BLE001 # hook dispatch - including cancellation mid-await - must n + except BaseException: # noqa: S110, BLE001 # hook dispatch - including cancellation mid-await - must never replace the real deployment failure, so there is nothing to do with what it raises pass raise end_time = datetime.datetime.now() @@ -2225,7 +2225,7 @@ async def wrapper_async(*args, **kwargs): traceback_exception: Final = traceback.format_exc() # Reuse the timestamp taken right when the deployment call itself failed, before # the failure hook ran, so a slow callback doesn't inflate the reported duration. - end_time = _deployment_call_end_time if _deployment_call_end_time is not None else datetime.datetime.now() # noqa: DTZ005 # matc + end_time = _deployment_call_end_time if _deployment_call_end_time is not None else datetime.datetime.now() # noqa: DTZ005 # matches the naive datetimes this whole function already times start_time/end_time with if logging_obj and not _is_litellm_internal_call: unbind_budget_reservation_from_callbacks(logging_obj.litellm_params) try: @@ -4363,7 +4363,7 @@ def pre_process_non_default_params( non_default_params, list ): # fixes https://github.com/BerriAI/litellm/issues/4933 tools: Final = non_default_params["tools"] - for tool in tools: # clean out 'additionalProperties = False'. Causes vertexai/gemini OpenAI API Schema errors + for tool in tools: # clean out 'additionalProperties = False'. Causes vertexai/gemini OpenAI API Schema errors - https://github.com/langchain-ai/langchainjs/issues/5240 tool_function = tool.get("function", {}) parameters = tool_function.get("parameters", None) if parameters is not None: @@ -4898,7 +4898,7 @@ def _check_valid_arg(supported_params: list[str]): drop_params=bool(drop_params), ) elif custom_llm_provider == "nadir": - optional_params = litellm.NadirConfig().map_openai_params( # rebind-ok: same optional_params rebinding every si + optional_params = litellm.NadirConfig().map_openai_params( # rebind-ok: same optional_params rebinding every sibling provider branch does non_default_params=non_default_params, optional_params=optional_params, model=model, @@ -5105,7 +5105,7 @@ def add_provider_specific_params_to_optional_params( """ if custom_llm_provider in ["openai", "azure", "text-completion-openai"] + litellm.openai_compatible_providers: - # for openai, azure we should pass the extra/passed params within `extra_body` https://github.com/openai/openai- + # for openai, azure we should pass the extra/passed params within `extra_body` https://github.com/openai/openai-python/blob/ac33853ba10d13ac149b1fa3ca6dba7d613065c9/src/openai/resources/models.py#L46 if _should_drop_param(k="extra_body", additional_drop_params=additional_drop_params) is False: extra_body: Final = dict(passed_params.pop("extra_body", None) or {}) for k in passed_params: @@ -7605,7 +7605,7 @@ def attempt_message_addition(final_messages, message, available_tokens, max_toke # if temp_message_tokens > max_tokens, try shortening temp_messages elif "function_call" not in message: verbose_logger.debug("attempting to shorten message to fit limit") - # fit updated_message to be within temp_message_tokens - max_tokens (aka the amount temp_message_tokens is great + # fit updated_message to be within temp_message_tokens - max_tokens (aka the amount temp_message_tokens is greate than max_tokens) updated_message: Final = shorten_message_to_fit_limit(message, available_tokens, model) if can_add_message(updated_message, final_messages, max_tokens, model): verbose_logger.debug("can add message, returning [updated_message] + final_messages") From 7ab185351d134f051d96c58aac00c8ed04f7e859 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:30:12 +0000 Subject: [PATCH 37/38] style(github_copilot): complete cut-off suppression reasons on new casts --- litellm/litellm_core_utils/get_llm_provider_logic.py | 2 +- litellm/llms/github_copilot/per_user_auth.py | 2 +- litellm/proxy/common_utils/credential_hydration.py | 2 +- litellm/router.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index fe5d7c66388a..c822e3987335 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -808,7 +808,7 @@ def _get_openai_compatible_provider_info( api_base = user_session.api_base # rebind-ok: resolves provider args in place dynamic_api_key = user_session.token # rebind-ok: resolves provider args in place elif ( - github_copilot_per_user_credential_name(cast("dict[str, object]", litellm_params)) # cast-ok: litellm_param + github_copilot_per_user_credential_name(cast("dict[str, object]", litellm_params)) # cast-ok: untyped params dict is not None ): # per-user deployments resolve no token at provider-info time; the diff --git a/litellm/llms/github_copilot/per_user_auth.py b/litellm/llms/github_copilot/per_user_auth.py index 8cd09f6fa8f3..10e5e12dec33 100644 --- a/litellm/llms/github_copilot/per_user_auth.py +++ b/litellm/llms/github_copilot/per_user_auth.py @@ -328,7 +328,7 @@ def _new_async_github_client() -> _AsyncGitHubClient: return cast( # cast-ok: pins the untyped litellm client factory to the local Protocol Callable[[str], _AsyncGitHubClient], - getattr(http_handler, "get_async_httpx_client"), # noqa: B009 # getattr keeps the untyped factory out of membe + getattr(http_handler, "get_async_httpx_client"), # noqa: B009 # untyped factory )(_LLM_PROVIDER) diff --git a/litellm/proxy/common_utils/credential_hydration.py b/litellm/proxy/common_utils/credential_hydration.py index 56bf4ef48d09..a4e858692e64 100644 --- a/litellm/proxy/common_utils/credential_hydration.py +++ b/litellm/proxy/common_utils/credential_hydration.py @@ -72,7 +72,7 @@ def _decrypted(db_credential: CredentialItem) -> CredentialItem: """The stored credential with every value decrypted, leaving already-plaintext values alone.""" decrypted_values: Final = MappingProxyType( { - key: decrypted_or_stored(key, cast("str", value)) # cast-ok: credential values are str at the decrypt bound + key: decrypted_or_stored(key, cast("str", value)) # cast-ok: stored values are str for key, value in db_credential.credential_values.items() } ) diff --git a/litellm/router.py b/litellm/router.py index 788b0ef9236c..8d6287a77ebc 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8687,7 +8687,7 @@ def deployment_callback_on_failure( deployment=deployment_id, time_to_cooldown=_time_to_cooldown, requested_model_group=( - get_litellm_metadata_from_kwargs(cast("dict[str, object]", kwargs)) # cast-ok: kwargs is the un + get_litellm_metadata_from_kwargs(cast("dict[str, object]", kwargs)) # cast-ok: untyped kwargs dict or {} ).get("model_group"), request_kwargs=cast("dict[str, object]", kwargs), # cast-ok: kwargs is the untyped request dict From c2174023865009ea45a57c82b92a8a8e4b8ff84d Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:56:13 +0000 Subject: [PATCH 38/38] refactor(github_copilot): annotate new boundary-typing locals as Final --- litellm/litellm_core_utils/get_llm_provider_logic.py | 4 +++- litellm/main.py | 4 ++-- .../proxy/credential_endpoints/user_provider_credentials.py | 4 ++-- litellm/proxy/litellm_pre_call_utils.py | 6 +++--- litellm/router.py | 4 +++- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index c822e3987335..c9e5167e7a9a 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -808,7 +808,9 @@ def _get_openai_compatible_provider_info( api_base = user_session.api_base # rebind-ok: resolves provider args in place dynamic_api_key = user_session.token # rebind-ok: resolves provider args in place elif ( - github_copilot_per_user_credential_name(cast("dict[str, object]", litellm_params)) # cast-ok: untyped params dict + github_copilot_per_user_credential_name( + cast("dict[str, object]", litellm_params) # cast-ok: untyped params dict + ) is not None ): # per-user deployments resolve no token at provider-info time; the diff --git a/litellm/main.py b/litellm/main.py index 36ce503e525f..b80819354f00 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -7765,8 +7765,8 @@ def process_prompt(i, individual_prompt): kwargs.pop("prompt", None) - if _model is not None and ( - custom_llm_provider == "openai" + if ( + _model is not None and (custom_llm_provider == "openai") ): # for openai compatible endpoints - e.g. vllm, call the native /v1/completions endpoint for text completion calls if _model not in litellm.open_ai_chat_completion_models: model = "text-completion-openai/" + _model diff --git a/litellm/proxy/credential_endpoints/user_provider_credentials.py b/litellm/proxy/credential_endpoints/user_provider_credentials.py index 9eaf1548f97a..4253483a2281 100644 --- a/litellm/proxy/credential_endpoints/user_provider_credentials.py +++ b/litellm/proxy/credential_endpoints/user_provider_credentials.py @@ -270,7 +270,7 @@ async def set_user_provider_credential_cache( _encode(payload), ttl=GITHUB_COPILOT_USER_CREDENTIAL_CACHE_TTL_SECONDS, ) - readback: object = await _try_cache_get(token_cache, key) + readback: Final = await _try_cache_get(token_cache, key) if isinstance(readback, str) and decode_user_provider_credential(readback) == payload: return True verbose_proxy_logger.warning( @@ -282,7 +282,7 @@ async def set_user_provider_credential_cache( await _string_cache(token_cache).async_delete_cache(key) except Exception: # noqa: BLE001 # a Redis outage must not fail a connect verbose_proxy_logger.warning("set_user_provider_credential_cache: Redis delete failed") - after_delete: object = await _try_cache_get(token_cache, key) + after_delete: Final = await _try_cache_get(token_cache, key) if after_delete is None: return True if isinstance(after_delete, str) and decode_user_provider_credential(after_delete) == payload: diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index eb86c96660bf..490b393aa68f 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -2760,7 +2760,7 @@ def _fallback_entry_target_count(entry: object) -> int: key plus one per target in its value lists, so empty lists and scalar values still cost what the router scan costs.""" if isinstance(entry, dict): - entry_dict = cast("dict[object, object]", entry) # cast-ok: fallback entries are plain dicts at runtime + entry_dict: Final = cast("dict[object, object]", entry) # cast-ok: fallback entries are plain dicts at runtime return sum( max(1, len(cast("Sequence[object]", value))) # cast-ok: fallback values are scalar or list entries if isinstance(value, list) @@ -2810,8 +2810,8 @@ def _index_fallback_list(fallback_list: Sequence[object]) -> _FallbackIndex: bare keys (duplicate keys union their targets, a superset of the router's first-wins resolution), plus the entries only the router matcher can score ("*" keys, provider-prefixed keys, bare-string generic targets).""" - exact: dict[str, tuple[str, ...]] = {} # mutable-ok: one entry per bare source key - fuzzy: list[object] = [] # mutable-ok: accumulates matcher-only entries + exact: Final[dict[str, tuple[str, ...]]] = {} # mutable-ok: one entry per bare source key + fuzzy: Final[list[object]] = [] # mutable-ok: accumulates matcher-only entries for entry in fallback_list: if isinstance(entry, dict) and entry: entry_dict = cast("dict[object, object]", entry) # cast-ok: fallback entries are plain dicts at runtime diff --git a/litellm/router.py b/litellm/router.py index 8d6287a77ebc..f173cdb37b42 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8687,7 +8687,9 @@ def deployment_callback_on_failure( deployment=deployment_id, time_to_cooldown=_time_to_cooldown, requested_model_group=( - get_litellm_metadata_from_kwargs(cast("dict[str, object]", kwargs)) # cast-ok: untyped kwargs dict + get_litellm_metadata_from_kwargs( + cast("dict[str, object]", kwargs) # cast-ok: untyped kwargs dict + ) or {} ).get("model_group"), request_kwargs=cast("dict[str, object]", kwargs), # cast-ok: kwargs is the untyped request dict