From 728b73d1b37efe118751752bfeedcf8d6e21faf4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 25 Aug 2026 21:04:27 -0700 Subject: [PATCH 1/6] fix(proxy): strip only the authenticating secret on credential-less Vertex passthrough PR #38114 dropped whichever header user_api_key_auth would read the caller's key from, by precedence. Under custom_auth, JWT auth, or no master key that header is the caller's own Google token, so the bring-your-own-credentials Vertex branch answered 401 to every valid request. A header value is now dropped only when it is the master key or when its hash is the api_key that authenticated the request, so a Google token that auth never consumed keeps flowing while a LiteLLM key still never reaches Google. test_passthrough_post_call_guardrails.py no longer plants a MagicMock proxy_server module in sys.modules at import, which poisoned sibling tests that read module globals at call time. --- bootstrap.log | 271 ++++++++++++++++++ .../llm_passthrough_endpoints.py | 116 ++++---- .../test_llm_pass_through_endpoints.py | 83 +++++- .../test_passthrough_post_call_guardrails.py | 16 -- .../test_vertex_passthrough_load_balancing.py | 3 + 5 files changed, 398 insertions(+), 91 deletions(-) create mode 100644 bootstrap.log diff --git a/bootstrap.log b/bootstrap.log new file mode 100644 index 000000000000..864cbf2d57e9 --- /dev/null +++ b/bootstrap.log @@ -0,0 +1,271 @@ +uv sync --inexact --frozen --extra proxy --group proxy-dev --group e2e-dev +Using CPython 3.12.13 +Creating virtual environment at: .venv + Building litellm @ file:///private/tmp/claude-501/-Users-mateo-Development-litellm/5f605a1a-6687-495a-8f8f-e1f96a73b6da/scratchpad/vertexpt-fix + Building litellm-proxy-extras @ file:///private/tmp/claude-501/-Users-mateo-Development-litellm/5f605a1a-6687-495a-8f8f-e1f96a73b6da/scratchpad/vertexpt-fix/litellm-proxy-extras + Building litellm-enterprise @ file:///private/tmp/claude-501/-Users-mateo-Development-litellm/5f605a1a-6687-495a-8f8f-e1f96a73b6da/scratchpad/vertexpt-fix/enterprise + Built litellm-enterprise @ file:///private/tmp/claude-501/-Users-mateo-Development-litellm/5f605a1a-6687-495a-8f8f-e1f96a73b6da/scratchpad/vertexpt-fix/enterprise + Built litellm-proxy-extras @ file:///private/tmp/claude-501/-Users-mateo-Development-litellm/5f605a1a-6687-495a-8f8f-e1f96a73b6da/scratchpad/vertexpt-fix/litellm-proxy-extras + Built litellm @ file:///private/tmp/claude-501/-Users-mateo-Development-litellm/5f605a1a-6687-495a-8f8f-e1f96a73b6da/scratchpad/vertexpt-fix +Prepared 3 packages in 1m 00s +Installed 229 packages in 637ms + + a2a-sdk==1.1.0 + + aiohappyeyeballs==2.6.1 + + aiohttp==3.14.3 + + aiologic==0.17.0 + + aiosignal==1.4.0 + + annotated-doc==0.0.4 + + annotated-types==0.7.0 + + anyio==4.13.0 + + apscheduler==3.11.2 + + asgiref==3.11.1 + + attrs==26.1.0 + + azure-core==1.41.0 + + azure-identity==1.25.2 + + azure-storage-blob==12.28.0 + + backoff==2.2.1 + + basedpyright==1.39.7 + + bidict==0.23.1 + + blinker==1.9.0 + + boto3==1.43.1 + + botocore==1.43.6 + + botocore-stubs==1.43.14 + + brotli==1.2.0 + + certifi==2026.4.22 + + cffi==2.0.0 + + chardet==7.4.3 + + charset-normalizer==3.4.7 + + click==8.3.3 + + configargparse==1.7.5 + + coverage==7.14.0 + + croniter==6.2.2 + + cryptography==50.0.0 + + culsans==0.11.0 + + deprecated==1.3.1 + + diff-cover==9.7.2 + + distro==1.9.0 + + dnspython==2.8.0 + + email-validator==2.3.0 + + execnet==2.1.2 + + expression==5.6.0 + + fakeredis==2.34.1 + + fastapi==0.136.3 + + fastapi-offline==1.7.6 + + fastapi-sso==0.19.0 + + fastuuid==0.14.0 + + filelock==3.29.0 + + flask==3.1.3 + + flask-cors==6.0.2 + + flask-login==0.6.3 + + frozenlist==1.8.0 + + fsspec==2026.4.0 + + gevent==25.9.1 + + geventhttpclient==2.3.9 + + google-api-core==2.30.3 + + google-auth==2.52.0 + + googleapis-common-protos==1.75.0 + + granian==2.7.4 + + greenlet==3.5.0 + + grpcio==1.78.0 + + gunicorn==23.0.0 + + h11==0.16.0 + + h2==4.4.1 + + hf-xet==1.5.0 + + hiredis==3.4.0 + + hpack==4.2.0 + + httpcore==1.0.9 + + httpx==0.28.1 + + httpx-sse==0.4.3 + + huggingface-hub==1.14.0 + + hypercorn==0.17.3 + + hyperframe==6.1.0 + + idna==3.15 + + importlib-metadata==8.5.0 + + iniconfig==2.3.0 + + inquirerpy==0.3.4 + + isodate==0.7.2 + + itsdangerous==2.2.0 + + jaraco-classes==3.4.0 + + jaraco-context==6.1.2 + + jaraco-functools==4.6.0 + + jinja2==3.1.6 + + jiter==0.14.0 + + jmespath==1.1.0 + + json-rpc==1.15.0 + + jsonschema==4.26.0 + + jsonschema-path==0.3.4 + + jsonschema-specifications==2025.9.1 + + keyring==25.7.0 + + langfuse==2.59.7 + + lazy-object-proxy==1.12.0 + + litellm==1.100.0 (from file:///private/tmp/claude-501/-Users-mateo-Development-litellm/5f605a1a-6687-495a-8f8f-e1f96a73b6da/scratchpad/vertexpt-fix) + + litellm-enterprise==0.1.60 (from file:///private/tmp/claude-501/-Users-mateo-Development-litellm/5f605a1a-6687-495a-8f8f-e1f96a73b6da/scratchpad/vertexpt-fix/enterprise) + + litellm-proxy-extras==0.4.89 (from file:///private/tmp/claude-501/-Users-mateo-Development-litellm/5f605a1a-6687-495a-8f8f-e1f96a73b6da/scratchpad/vertexpt-fix/litellm-proxy-extras) + + locust==2.45.0 + + markdown-it-py==4.2.0 + + markupsafe==3.0.3 + + mcp==1.28.1 + + mdurl==0.1.2 + + mirakuru==3.0.2 + + more-itertools==11.0.2 + + msal==1.37.0 + + msal-extensions==1.3.1 + + msgpack==1.2.1 + + multidict==6.7.1 + + nodeenv==1.10.0 + + nodejs-wheel-binaries==24.16.0 + + oauthlib==3.3.1 + + openai==2.33.0 + + openapi-core==0.22.0 + + openapi-schema-validator==0.6.3 + + openapi-spec-validator==0.7.2 + + opentelemetry-api==1.28.0 + + opentelemetry-exporter-otlp==1.28.0 + + opentelemetry-exporter-otlp-proto-common==1.28.0 + + opentelemetry-exporter-otlp-proto-grpc==1.28.0 + + opentelemetry-exporter-otlp-proto-http==1.28.0 + + opentelemetry-instrumentation==0.49b0 + + opentelemetry-instrumentation-asgi==0.49b0 + + opentelemetry-instrumentation-fastapi==0.49b0 + + opentelemetry-proto==1.28.0 + + opentelemetry-sdk==1.28.0 + + opentelemetry-semantic-conventions==0.49b0 + + opentelemetry-util-http==0.49b0 + + orjson==3.11.6 + + packaging==26.2 + + parameterized==0.9.0 + + pathable==0.4.4 + + pfzy==0.3.4 + + playwright==1.61.0 + + pluggy==1.6.0 + + polars==1.38.1 + + polars-runtime-32==1.38.1 + + port-for==1.0.0 + + priority==2.0.0 + + prisma==0.11.0 + + prometheus-client==0.20.0 + + prompt-toolkit==3.0.52 + + propcache==0.5.2 + + proto-plus==1.28.0 + + protobuf==5.29.6 + + psutil==7.2.2 + + psycopg==3.3.3 + + psycopg-binary==3.3.3 + + pyasn1==0.6.4 + + pyasn1-modules==0.4.2 + + pycparser==3.0 + + pydantic==2.13.4 + + pydantic-core==2.46.4 + + pydantic-settings==2.14.2 + + pyee==13.0.1 + + pygments==2.20.0 + + pyjwt==2.13.0 + + pynacl==1.6.2 + + pyroscope-io==0.8.16 + + pytest==9.0.3 + + pytest-asyncio==1.3.0 + + pytest-cov==5.0.0 + + pytest-mock==3.15.1 + + pytest-postgresql==7.0.2 + + pytest-recording==0.13.4 + + pytest-rerunfailures==15.1 + + pytest-timeout==2.4.0 + + pytest-xdist==3.8.0 + + python-dateutil==2.9.0.post0 + + python-dotenv==1.2.2 + + python-engineio==4.13.3 + + python-multipart==0.0.32 + + python-socketio==5.16.3 + + pyyaml==6.0.3 + + pyzmq==27.1.0 + + redis==5.3.1 + + referencing==0.36.2 + + regex==2026.5.9 + + requests==2.34.0 + + requests-mock==1.12.1 + + responses==0.26.0 + + respx==0.22.0 + + restrictedpython==8.1 + + rfc3339-validator==0.1.4 + + rich==13.9.4 + + rpds-py==0.30.0 + + rq==2.7.0 + + ruff==0.15.3 + + s3transfer==0.17.0 + + shellingham==1.5.4 + + simple-websocket==1.1.0 + + six==1.17.0 + + sniffio==1.3.1 + + sortedcontainers==2.4.0 + + soundfile==0.12.1 + + sse-starlette==3.4.4 + + starlette==1.3.1 + + tiktoken==0.12.0 + + tokenizers==0.23.1 + + tomlkit==0.15.0 + + tqdm==4.67.3 + + typer==0.25.1 + + types-awscrt==0.34.1 + + types-boto3==1.43.30 + + types-boto3-bedrock==1.43.26 + + types-boto3-bedrock-agent==1.43.0 + + types-boto3-bedrock-runtime==1.43.30 + + types-boto3-kms==1.43.12 + + types-boto3-s3==1.43.14 + + types-boto3-sagemaker-runtime==1.43.29 + + types-boto3-sts==1.43.0 + + types-cffi==2.0.0.20260508 + + types-pyopenssl==24.1.0.20240722 + + types-pyyaml==6.0.12.20250915 + + types-redis==4.6.0.20241004 + + types-requests==2.32.4.20260107 + + types-s3transfer==0.16.0 + + types-setuptools==75.8.0.20250225 + + typing-extensions==4.15.0 + + typing-inspection==0.4.2 + + tzlocal==5.3.1 + + urllib3==2.7.0 + + uvicorn==0.51.0 + + uvloop==0.21.0 + + vcrpy==8.2.1 + + wcwidth==0.7.0 + + websocket-client==1.9.0 + + websockets==15.0.1 + + werkzeug==3.1.8 + + wrapt==1.17.3 + + wsproto==1.3.2 + + yarl==1.23.0 + + zipp==3.23.1 + + zope-event==6.2 + + zope-interface==8.5 +uv run --no-sync python scripts/prisma_generate_if_needed.py +Prisma schema loaded from litellm/proxy/schema.prisma + +Some types are disabled by default due to being incompatible with Mypy, it is highly recommended +to use Pyright instead and configure Prisma Python to use recursive types. To re-enable certain types: + +generator client { + provider = "prisma-client-py" + recursive_type_depth = -1 +} + +If you need to use Mypy, you can also disable this message by explicitly setting the default value: + +generator client { + provider = "prisma-client-py" + recursive_type_depth = 5 +} + +For more information see: https://prisma-client-py.readthedocs.io/en/stable/reference/limitations/#default-type-limitations + + +Warning: The binaryTargets option is not officially supported by Prisma Client Python. + +✔ Generated Prisma Client Python (v0.11.0) to ./.venv/lib/python3.12/site-packages/prisma in 401ms + +cd ui/litellm-dashboard && ../../scripts/with_dashboard_node.sh npm install --no-audit --no-fund +npm warn deprecated node-domexception@1.0.0: Use your platform's native DOMException instead + +added 738 packages in 9s +bootstrap: copied .env from /Users/mateo/Development/litellm +bootstrap: done diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 850b50227f4e..0234e38c9210 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -6,6 +6,7 @@ Use litellm with Anthropic SDK, Vertex AI SDK, Cohere SDK, etc. """ +import hmac import json import os import re @@ -48,7 +49,7 @@ create_websocket_passthrough_route, websocket_passthrough_request, ) -from litellm.proxy.utils import is_known_model +from litellm.proxy.utils import hash_token, is_known_model from litellm.proxy.vector_store_endpoints.utils import ( assert_proxy_admin_for_vector_store_index_management, assert_user_can_access_vector_store, @@ -1791,28 +1792,17 @@ def _normalize_credential_value(value: str) -> str: ) -_VERTEX_CALLER_KEY_HEADER_PRECEDENCE: Final = ( - SpecialHeaders.custom_litellm_api_key.value.lower(), - SpecialHeaders.openai_authorization.value.lower(), - SpecialHeaders.azure_authorization.value.lower(), - SpecialHeaders.anthropic_authorization.value.lower(), - SpecialHeaders.google_ai_studio_authorization.value.lower(), - SpecialHeaders.azure_apim_authorization.value.lower(), -) - _MAPPED_ROUTE_CALLER_KEY_HEADER: Final = "litellm_user_api_key" -def _operator_configured_caller_key_header_names() -> tuple[tuple[str, ...], tuple[str, ...]]: - """Operator-configured caller-key header names, as (override, pass_through). +def _operator_configured_caller_key_header_names() -> tuple[str, ...]: + """Operator-configured caller-key header names. - ``user_api_key_auth`` accepts the caller's key from two runtime-configured - header sources beyond the built-in ones, at opposite ends of its precedence. - ``general_settings.litellm_key_header_name`` overrides every built-in source - (it replaces the resolved key after ``get_api_key`` runs), so it is highest - precedence. Each ``general_settings.pass_through_endpoints`` entry's - ``headers.litellm_user_api_key`` is checked last inside ``get_api_key``, so it - is lowest. Google never consumes either, so both are also dropped by name. + ``user_api_key_auth`` also reads the caller's key from + ``general_settings.litellm_key_header_name`` and from each + ``general_settings.pass_through_endpoints`` entry's + ``headers.litellm_user_api_key``. Google never consumes either, so both are + dropped by name. """ from litellm.proxy.proxy_server import general_settings @@ -1829,42 +1819,34 @@ def _operator_configured_caller_key_header_names() -> tuple[tuple[str, ...], tup if isinstance(headers, dict) and isinstance(headers.get("litellm_user_api_key"), str) ) ) - return override, pass_through + return override + pass_through -def _authenticated_caller_key_values(request: Request) -> frozenset[str]: - """The value ``user_api_key_auth`` would accept as this caller's LiteLLM key. +def _is_authenticated_caller_secret(value: str, user_api_key_dict: UserAPIKeyAuth) -> bool: + """Whether a header value is a LiteLLM secret that authenticated this caller. - The Vertex route authenticates through ``Depends(user_api_key_auth)``, which - resolves the key by precedence, matched here exactly. The ``/vertex_ai`` route - is a mapped pass-through route, so a header literally named - ``litellm_user_api_key`` overrides every other source (``user_api_key_auth`` - applies it last), making it highest precedence. Then an operator - ``litellm_key_header_name``, then the built-in headers in ``get_api_key`` order, - then a ``pass_through_endpoints`` ``litellm_user_api_key`` header which - ``get_api_key`` checks last. Some of those headers (``Authorization``, - ``x-goog-api-key``) are also kept as genuine bring-your-own Google credentials, - so returning only the value that actually authenticated lets the filter strip - that value wherever it appears while leaving a real Google credential in place. - An empty set means no caller key was found, so nothing is value-stripped. + That is the proxy master key, or a LiteLLM-issued key whose ``hash_token`` is + the ``api_key`` ``user_api_key_auth`` resolved (a DB virtual key, or a custom + auth returning the key it validated). A value auth only echoed unhashed (a + custom auth's own identifier, the no-master-key path returning the raw + header) or never consumed (JWT auth resolves ``api_key=None``) is not a + LiteLLM secret, so a bring-your-own Google credential in that position is + still forwarded. """ - incoming: Final = _safe_get_request_headers(request) - override_headers, pass_through_headers = _operator_configured_caller_key_header_names() - ordered_names: Final = ( - (_MAPPED_ROUTE_CALLER_KEY_HEADER,) - + override_headers - + _VERTEX_CALLER_KEY_HEADER_PRECEDENCE - + pass_through_headers - ) - present_values: Final = (incoming[name] for name in ordered_names if incoming.get(name)) - authenticated_key: Final = next( - (stripped for value in present_values if (stripped := _normalize_credential_value(value))), - "", - ) - return frozenset({authenticated_key}) if authenticated_key else frozenset() + from litellm.proxy.proxy_server import master_key + + normalized: Final = _normalize_credential_value(value) + if master_key is not None and hmac.compare_digest(normalized.encode(), master_key.encode()): + return True + authenticated_key: Final = user_api_key_dict.api_key + if authenticated_key is None: + return False + return hmac.compare_digest(hash_token(normalized).encode(), authenticated_key.encode()) -def _forwarded_headers_for_credentialless_vertex_passthrough(request: Request) -> Mapping[str, str]: +def _forwarded_headers_for_credentialless_vertex_passthrough( + request: Request, user_api_key_dict: UserAPIKeyAuth +) -> Mapping[str, str]: """ Header set to forward on the bring-your-own-credentials Vertex passthrough branch, used when the proxy has no Vertex credential configured. @@ -1878,31 +1860,25 @@ def _forwarded_headers_for_credentialless_vertex_passthrough(request: Request) - ``api-key`` / ``x-api-key`` / ``Ocp-Apim-Subscription-Key``, plus the mapped pass-through ``litellm_user_api_key`` header and any operator-configured ``litellm_key_header_name`` / ``pass_through_endpoints`` key header) are dropped - by name. ``Authorization`` and ``x-goog-api-key`` may - instead carry a genuine bring-your-own Google credential, so they are kept - unless their value is the caller's authenticated LiteLLM key, which is dropped - by value (normalizing any ``Bearer`` / ``Basic`` / ``AWS4`` auth-scheme prefix - the same way authentication does). Because the value that authenticated is - resolved by the same precedence ``user_api_key_auth`` uses, a virtual key sent - only in ``x-goog-api-key`` (or in an operator-configured key header) is dropped - too, while a real Google key in ``x-goog-api-key`` alongside a virtual key in a - higher-precedence header is preserved. When neither a surviving - ``Authorization`` nor ``x-goog-api-key`` remains the request is rejected so the - virtual key cannot leak upstream. + by name. ``Authorization`` and ``x-goog-api-key`` may instead carry a genuine + bring-your-own Google credential, so they are kept unless their value is the + LiteLLM secret that authenticated this caller (``_is_authenticated_caller_secret``), + which is dropped by value wherever it appears. Matching on what actually + authenticated, rather than on header precedence, keeps a caller's own Google + token in ``Authorization`` flowing when a custom auth or JWT authenticated + them without consuming it. When neither a surviving ``Authorization`` nor + ``x-goog-api-key`` remains the request is rejected so the virtual key cannot + leak upstream. """ incoming: Final = _safe_get_request_headers(request) - caller_key_values: Final = _authenticated_caller_key_values(request) - override_headers, pass_through_headers = _operator_configured_caller_key_header_names() - never_forwarded: Final = ( - _HEADERS_NEVER_FORWARDED_TO_VERTEX.union((_MAPPED_ROUTE_CALLER_KEY_HEADER,)) - .union(override_headers) - .union(pass_through_headers) + never_forwarded: Final = _HEADERS_NEVER_FORWARDED_TO_VERTEX.union( + (_MAPPED_ROUTE_CALLER_KEY_HEADER, *_operator_configured_caller_key_header_names()) ) forwarded: Final = MappingProxyType( { name: value for name, value in incoming.items() - if name not in never_forwarded and _normalize_credential_value(value) not in caller_key_values + if name not in never_forwarded and not _is_authenticated_caller_secret(value, user_api_key_dict) } ) if "authorization" not in forwarded and "x-goog-api-key" not in forwarded: @@ -1918,6 +1894,7 @@ async def _prepare_vertex_auth_headers( vertex_location: str | None, base_target_url: str | None, get_vertex_pass_through_handler: BaseVertexAIPassThroughHandler, + user_api_key_dict: UserAPIKeyAuth, ) -> tuple[Mapping[str, str], str | None, bool, str | None, str | None]: """ Prepare authentication headers for Vertex AI pass-through requests. @@ -1930,6 +1907,8 @@ async def _prepare_vertex_auth_headers( vertex_location: Vertex location base_target_url: Base URL for the Vertex AI service get_vertex_pass_through_handler: Handler for the specific Vertex AI service + user_api_key_dict: The caller's resolved authentication, so only the secret that + authenticated them is stripped on the credential-less branch Returns: Tuple containing: @@ -1944,7 +1923,7 @@ async def _prepare_vertex_auth_headers( # Use headers from the incoming request if no vertex credentials are found if (vertex_credentials is None or vertex_credentials.vertex_project is None) and router_credentials is None: - headers = _forwarded_headers_for_credentialless_vertex_passthrough(request) + headers = _forwarded_headers_for_credentialless_vertex_passthrough(request, user_api_key_dict) headers_passed_through = True verbose_proxy_logger.debug( "default_vertex_config not set, forwarding caller-provided headers %s", tuple(headers.keys()) @@ -2104,6 +2083,7 @@ async def _base_vertex_proxy_route( vertex_location=vertex_location, base_target_url=base_target_url, get_vertex_pass_through_handler=get_vertex_pass_through_handler, + user_api_key_dict=user_api_key_dict, ) if base_target_url is None: diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index a1725005dbfe..27f2521702a8 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -16,6 +16,7 @@ import litellm +from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( BaseOpenAIPassThroughHandler, RouteChecks, @@ -592,7 +593,7 @@ async def test_vertex_passthrough_with_no_default_credentials(self, monkeypatch) "method": "POST", "path": endpoint, "headers": [ - (b"authorization", b"Bearer test-creds"), + (b"authorization", b"Bearer sk-test-creds"), ], } ) @@ -617,7 +618,7 @@ async def test_vertex_passthrough_with_no_default_credentials(self, monkeypatch) ): mock_ensure_token.return_value = ("test-auth-header", test_project) mock_get_token.return_value = (test_token, "") - mock_auth.return_value = MagicMock() + mock_auth.return_value = UserAPIKeyAuth(api_key="sk-test-creds") with pytest.raises(HTTPException) as exc_info: await vertex_proxy_route( @@ -3342,14 +3343,14 @@ def fake_create_pass_through_route(**kwargs): ), mock.patch(f"{module}.create_pass_through_route", side_effect=fake_create_pass_through_route), mock.patch(f"{module}.get_litellm_virtual_key", return_value="Bearer test-key"), - mock.patch(f"{module}.user_api_key_auth", new=AsyncMock(return_value={"api_key": "test-key"})), + mock.patch(f"{module}.user_api_key_auth", new=AsyncMock(return_value=UserAPIKeyAuth(api_key="test-key"))), mock.patch(f"{module}.get_vertex_pass_through_handler", return_value=mock_handler), ): await vertex_proxy_route( endpoint=endpoint, request=request, fastapi_response=Response(), - user_api_key_dict=UserAPIKeyAuth(token="test-key"), + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), ) assert captured, "create_pass_through_route was never called" @@ -3466,6 +3467,12 @@ class TestVertexCredentiallessPassthroughVirtualKeyLeak: genuine bring-your-own Google credential that must still pass through. The by-value strip also covers a virtual key sent in the operator-configured ``general_settings.litellm_key_header_name``, whatever that header is named. + + The by-value strip keys off what actually authenticated the caller (the + master key, or the LiteLLM key whose hash ``user_api_key_auth`` resolved as + ``api_key``), never off header precedence: a custom auth or JWT that + authenticated the caller without consuming ``Authorization`` leaves the + caller's own Google token there, and it must keep flowing. """ VKEY = "sk-litellm-victim-key" @@ -3475,8 +3482,12 @@ class TestVertexCredentiallessPassthroughVirtualKeyLeak: ) async def _run( - self, monkeypatch, headers: list[tuple[bytes, bytes]] + self, + monkeypatch, + headers: list[tuple[bytes, bytes]], + authenticated: UserAPIKeyAuth | None = None, ) -> tuple[HTTPException | None, dict | None]: + caller: Final = authenticated if authenticated is not None else UserAPIKeyAuth(api_key=self.VKEY) from litellm.proxy.pass_through_endpoints.passthrough_endpoint_router import ( PassthroughEndpointRouter, ) @@ -3509,7 +3520,7 @@ def fake_create_pass_through_route(**kwargs): raised: HTTPException | None = None with ( mock.patch(f"{module}.create_pass_through_route", side_effect=fake_create_pass_through_route), - mock.patch(f"{module}.user_api_key_auth", new=AsyncMock(return_value=UserAPIKeyAuth(token="hashed"))), + mock.patch(f"{module}.user_api_key_auth", new=AsyncMock(return_value=caller)), mock.patch(f"{module}.get_vertex_pass_through_handler", return_value=mock_handler), ): try: @@ -3517,7 +3528,7 @@ def fake_create_pass_through_route(**kwargs): endpoint=self.ENDPOINT, request=request, fastapi_response=Response(), - user_api_key_dict=UserAPIKeyAuth(token="hashed"), + user_api_key_dict=caller, ) except HTTPException as exc: raised = exc @@ -3767,6 +3778,64 @@ async def test_virtual_key_in_mapped_route_litellm_user_api_key_header_alone_is_ assert forwarded is None, "a virtual key in the mapped-route litellm_user_api_key header must be dropped, not forwarded" assert raised is not None and raised.status_code == 401 + GOOGLE_OAUTH_TOKEN = "ya29.byo-google-oauth-token" + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "authenticated", + [ + pytest.param( + UserAPIKeyAuth(api_key="best-api-key-ever", user_role=LitellmUserRoles.PROXY_ADMIN), + id="custom-auth-returning-its-own-identifier", + ), + pytest.param(UserAPIKeyAuth(api_key=None, user_id="jwt-subject"), id="jwt-auth"), + pytest.param(UserAPIKeyAuth(api_key=GOOGLE_OAUTH_TOKEN), id="no-master-key-echoes-raw-header"), + ], + ) + async def test_google_token_in_authorization_is_forwarded_when_auth_did_not_consume_it( + self, monkeypatch, authenticated: UserAPIKeyAuth + ): + raised, forwarded = await self._run( + monkeypatch, + [ + (b"authorization", f"Bearer {self.GOOGLE_OAUTH_TOKEN}".encode()), + (b"content-type", b"application/json"), + ], + authenticated=authenticated, + ) + assert raised is None, f"the caller's own Google token must not be mistaken for a LiteLLM key: {raised}" + assert forwarded is not None + assert forwarded.get("authorization") == f"Bearer {self.GOOGLE_OAUTH_TOKEN}" + + @pytest.mark.asyncio + async def test_master_key_in_authorization_alone_is_rejected(self, monkeypatch): + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "sk-master-1234") + raised, forwarded = await self._run( + monkeypatch, + [(b"authorization", b"Bearer sk-master-1234"), (b"content-type", b"application/json")], + authenticated=UserAPIKeyAuth(api_key=LITELLM_PROXY_MASTER_KEY_ALIAS, user_role=LitellmUserRoles.PROXY_ADMIN), + ) + assert forwarded is None, "the master key must never reach the upstream forwarder" + assert raised is not None and raised.status_code == 401 + + @pytest.mark.asyncio + async def test_master_key_is_stripped_and_byo_x_goog_api_key_forwards(self, monkeypatch): + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "sk-master-1234") + raised, forwarded = await self._run( + monkeypatch, + [ + (b"authorization", b"Bearer sk-master-1234"), + (b"x-goog-api-key", b"AIza-real-google-api-key"), + (b"content-type", b"application/json"), + ], + authenticated=UserAPIKeyAuth(api_key=LITELLM_PROXY_MASTER_KEY_ALIAS, user_role=LitellmUserRoles.PROXY_ADMIN), + ) + assert raised is None + assert forwarded is not None + assert forwarded.get("x-goog-api-key") == "AIza-real-google-api-key" + assert "authorization" not in forwarded + assert "sk-master-1234" not in " ".join(f"{name}:{value}" for name, value in forwarded.items()) + class TestGetAzureAISearchIndexFromEndpoint: """The operable index is only the segment right after ``indexes``. diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_passthrough_post_call_guardrails.py b/tests/test_litellm/proxy/pass_through_endpoints/test_passthrough_post_call_guardrails.py index a48e9e9e17f4..9d1975513a15 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_passthrough_post_call_guardrails.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_passthrough_post_call_guardrails.py @@ -6,7 +6,6 @@ """ import json -import sys from contextlib import ExitStack from unittest.mock import AsyncMock, MagicMock, patch @@ -66,21 +65,6 @@ def _make_mock_request(): return mock_request -def _ensure_proxy_server_mock(): - """Insert a mock proxy_server module if the real one can't import.""" - key = "litellm.proxy.proxy_server" - if key not in sys.modules: - mock_mod = MagicMock() - mock_mod.proxy_logging_obj = MagicMock() - sys.modules[key] = mock_mod - import litellm.proxy - - if not hasattr(litellm.proxy, "proxy_server"): - litellm.proxy.proxy_server = sys.modules[key] - - -_ensure_proxy_server_mock() - from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( pass_through_request, ) diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py b/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py index 8e973fc37719..961479c03931 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py @@ -2,6 +2,7 @@ import pytest +from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( _base_vertex_proxy_route, ) @@ -323,6 +324,7 @@ async def test_vertex_passthrough_forwards_anthropic_beta_header(): vertex_location="us-central1", base_target_url="https://us-central1-aiplatform.googleapis.com", get_vertex_pass_through_handler=mock_handler, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-litellm-secret-key"), ) # Verify that allowlisted headers are preserved @@ -417,6 +419,7 @@ async def test_vertex_passthrough_does_not_forward_litellm_auth_token(): vertex_location="us-central1", base_target_url="https://us-central1-aiplatform.googleapis.com", get_vertex_pass_through_handler=mock_handler, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-litellm-secret-key"), ) # The ONLY Authorization header should be the Vertex token From 54aebc5696be343d953c8a50262fc024225dee94 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 25 Aug 2026 21:05:10 -0700 Subject: [PATCH 2/6] chore: drop stray bootstrap.log --- bootstrap.log | 271 -------------------------------------------------- 1 file changed, 271 deletions(-) delete mode 100644 bootstrap.log diff --git a/bootstrap.log b/bootstrap.log deleted file mode 100644 index 864cbf2d57e9..000000000000 --- a/bootstrap.log +++ /dev/null @@ -1,271 +0,0 @@ -uv sync --inexact --frozen --extra proxy --group proxy-dev --group e2e-dev -Using CPython 3.12.13 -Creating virtual environment at: .venv - Building litellm @ file:///private/tmp/claude-501/-Users-mateo-Development-litellm/5f605a1a-6687-495a-8f8f-e1f96a73b6da/scratchpad/vertexpt-fix - Building litellm-proxy-extras @ file:///private/tmp/claude-501/-Users-mateo-Development-litellm/5f605a1a-6687-495a-8f8f-e1f96a73b6da/scratchpad/vertexpt-fix/litellm-proxy-extras - Building litellm-enterprise @ file:///private/tmp/claude-501/-Users-mateo-Development-litellm/5f605a1a-6687-495a-8f8f-e1f96a73b6da/scratchpad/vertexpt-fix/enterprise - Built litellm-enterprise @ file:///private/tmp/claude-501/-Users-mateo-Development-litellm/5f605a1a-6687-495a-8f8f-e1f96a73b6da/scratchpad/vertexpt-fix/enterprise - Built litellm-proxy-extras @ file:///private/tmp/claude-501/-Users-mateo-Development-litellm/5f605a1a-6687-495a-8f8f-e1f96a73b6da/scratchpad/vertexpt-fix/litellm-proxy-extras - Built litellm @ file:///private/tmp/claude-501/-Users-mateo-Development-litellm/5f605a1a-6687-495a-8f8f-e1f96a73b6da/scratchpad/vertexpt-fix -Prepared 3 packages in 1m 00s -Installed 229 packages in 637ms - + a2a-sdk==1.1.0 - + aiohappyeyeballs==2.6.1 - + aiohttp==3.14.3 - + aiologic==0.17.0 - + aiosignal==1.4.0 - + annotated-doc==0.0.4 - + annotated-types==0.7.0 - + anyio==4.13.0 - + apscheduler==3.11.2 - + asgiref==3.11.1 - + attrs==26.1.0 - + azure-core==1.41.0 - + azure-identity==1.25.2 - + azure-storage-blob==12.28.0 - + backoff==2.2.1 - + basedpyright==1.39.7 - + bidict==0.23.1 - + blinker==1.9.0 - + boto3==1.43.1 - + botocore==1.43.6 - + botocore-stubs==1.43.14 - + brotli==1.2.0 - + certifi==2026.4.22 - + cffi==2.0.0 - + chardet==7.4.3 - + charset-normalizer==3.4.7 - + click==8.3.3 - + configargparse==1.7.5 - + coverage==7.14.0 - + croniter==6.2.2 - + cryptography==50.0.0 - + culsans==0.11.0 - + deprecated==1.3.1 - + diff-cover==9.7.2 - + distro==1.9.0 - + dnspython==2.8.0 - + email-validator==2.3.0 - + execnet==2.1.2 - + expression==5.6.0 - + fakeredis==2.34.1 - + fastapi==0.136.3 - + fastapi-offline==1.7.6 - + fastapi-sso==0.19.0 - + fastuuid==0.14.0 - + filelock==3.29.0 - + flask==3.1.3 - + flask-cors==6.0.2 - + flask-login==0.6.3 - + frozenlist==1.8.0 - + fsspec==2026.4.0 - + gevent==25.9.1 - + geventhttpclient==2.3.9 - + google-api-core==2.30.3 - + google-auth==2.52.0 - + googleapis-common-protos==1.75.0 - + granian==2.7.4 - + greenlet==3.5.0 - + grpcio==1.78.0 - + gunicorn==23.0.0 - + h11==0.16.0 - + h2==4.4.1 - + hf-xet==1.5.0 - + hiredis==3.4.0 - + hpack==4.2.0 - + httpcore==1.0.9 - + httpx==0.28.1 - + httpx-sse==0.4.3 - + huggingface-hub==1.14.0 - + hypercorn==0.17.3 - + hyperframe==6.1.0 - + idna==3.15 - + importlib-metadata==8.5.0 - + iniconfig==2.3.0 - + inquirerpy==0.3.4 - + isodate==0.7.2 - + itsdangerous==2.2.0 - + jaraco-classes==3.4.0 - + jaraco-context==6.1.2 - + jaraco-functools==4.6.0 - + jinja2==3.1.6 - + jiter==0.14.0 - + jmespath==1.1.0 - + json-rpc==1.15.0 - + jsonschema==4.26.0 - + jsonschema-path==0.3.4 - + jsonschema-specifications==2025.9.1 - + keyring==25.7.0 - + langfuse==2.59.7 - + lazy-object-proxy==1.12.0 - + litellm==1.100.0 (from file:///private/tmp/claude-501/-Users-mateo-Development-litellm/5f605a1a-6687-495a-8f8f-e1f96a73b6da/scratchpad/vertexpt-fix) - + litellm-enterprise==0.1.60 (from file:///private/tmp/claude-501/-Users-mateo-Development-litellm/5f605a1a-6687-495a-8f8f-e1f96a73b6da/scratchpad/vertexpt-fix/enterprise) - + litellm-proxy-extras==0.4.89 (from file:///private/tmp/claude-501/-Users-mateo-Development-litellm/5f605a1a-6687-495a-8f8f-e1f96a73b6da/scratchpad/vertexpt-fix/litellm-proxy-extras) - + locust==2.45.0 - + markdown-it-py==4.2.0 - + markupsafe==3.0.3 - + mcp==1.28.1 - + mdurl==0.1.2 - + mirakuru==3.0.2 - + more-itertools==11.0.2 - + msal==1.37.0 - + msal-extensions==1.3.1 - + msgpack==1.2.1 - + multidict==6.7.1 - + nodeenv==1.10.0 - + nodejs-wheel-binaries==24.16.0 - + oauthlib==3.3.1 - + openai==2.33.0 - + openapi-core==0.22.0 - + openapi-schema-validator==0.6.3 - + openapi-spec-validator==0.7.2 - + opentelemetry-api==1.28.0 - + opentelemetry-exporter-otlp==1.28.0 - + opentelemetry-exporter-otlp-proto-common==1.28.0 - + opentelemetry-exporter-otlp-proto-grpc==1.28.0 - + opentelemetry-exporter-otlp-proto-http==1.28.0 - + opentelemetry-instrumentation==0.49b0 - + opentelemetry-instrumentation-asgi==0.49b0 - + opentelemetry-instrumentation-fastapi==0.49b0 - + opentelemetry-proto==1.28.0 - + opentelemetry-sdk==1.28.0 - + opentelemetry-semantic-conventions==0.49b0 - + opentelemetry-util-http==0.49b0 - + orjson==3.11.6 - + packaging==26.2 - + parameterized==0.9.0 - + pathable==0.4.4 - + pfzy==0.3.4 - + playwright==1.61.0 - + pluggy==1.6.0 - + polars==1.38.1 - + polars-runtime-32==1.38.1 - + port-for==1.0.0 - + priority==2.0.0 - + prisma==0.11.0 - + prometheus-client==0.20.0 - + prompt-toolkit==3.0.52 - + propcache==0.5.2 - + proto-plus==1.28.0 - + protobuf==5.29.6 - + psutil==7.2.2 - + psycopg==3.3.3 - + psycopg-binary==3.3.3 - + pyasn1==0.6.4 - + pyasn1-modules==0.4.2 - + pycparser==3.0 - + pydantic==2.13.4 - + pydantic-core==2.46.4 - + pydantic-settings==2.14.2 - + pyee==13.0.1 - + pygments==2.20.0 - + pyjwt==2.13.0 - + pynacl==1.6.2 - + pyroscope-io==0.8.16 - + pytest==9.0.3 - + pytest-asyncio==1.3.0 - + pytest-cov==5.0.0 - + pytest-mock==3.15.1 - + pytest-postgresql==7.0.2 - + pytest-recording==0.13.4 - + pytest-rerunfailures==15.1 - + pytest-timeout==2.4.0 - + pytest-xdist==3.8.0 - + python-dateutil==2.9.0.post0 - + python-dotenv==1.2.2 - + python-engineio==4.13.3 - + python-multipart==0.0.32 - + python-socketio==5.16.3 - + pyyaml==6.0.3 - + pyzmq==27.1.0 - + redis==5.3.1 - + referencing==0.36.2 - + regex==2026.5.9 - + requests==2.34.0 - + requests-mock==1.12.1 - + responses==0.26.0 - + respx==0.22.0 - + restrictedpython==8.1 - + rfc3339-validator==0.1.4 - + rich==13.9.4 - + rpds-py==0.30.0 - + rq==2.7.0 - + ruff==0.15.3 - + s3transfer==0.17.0 - + shellingham==1.5.4 - + simple-websocket==1.1.0 - + six==1.17.0 - + sniffio==1.3.1 - + sortedcontainers==2.4.0 - + soundfile==0.12.1 - + sse-starlette==3.4.4 - + starlette==1.3.1 - + tiktoken==0.12.0 - + tokenizers==0.23.1 - + tomlkit==0.15.0 - + tqdm==4.67.3 - + typer==0.25.1 - + types-awscrt==0.34.1 - + types-boto3==1.43.30 - + types-boto3-bedrock==1.43.26 - + types-boto3-bedrock-agent==1.43.0 - + types-boto3-bedrock-runtime==1.43.30 - + types-boto3-kms==1.43.12 - + types-boto3-s3==1.43.14 - + types-boto3-sagemaker-runtime==1.43.29 - + types-boto3-sts==1.43.0 - + types-cffi==2.0.0.20260508 - + types-pyopenssl==24.1.0.20240722 - + types-pyyaml==6.0.12.20250915 - + types-redis==4.6.0.20241004 - + types-requests==2.32.4.20260107 - + types-s3transfer==0.16.0 - + types-setuptools==75.8.0.20250225 - + typing-extensions==4.15.0 - + typing-inspection==0.4.2 - + tzlocal==5.3.1 - + urllib3==2.7.0 - + uvicorn==0.51.0 - + uvloop==0.21.0 - + vcrpy==8.2.1 - + wcwidth==0.7.0 - + websocket-client==1.9.0 - + websockets==15.0.1 - + werkzeug==3.1.8 - + wrapt==1.17.3 - + wsproto==1.3.2 - + yarl==1.23.0 - + zipp==3.23.1 - + zope-event==6.2 - + zope-interface==8.5 -uv run --no-sync python scripts/prisma_generate_if_needed.py -Prisma schema loaded from litellm/proxy/schema.prisma - -Some types are disabled by default due to being incompatible with Mypy, it is highly recommended -to use Pyright instead and configure Prisma Python to use recursive types. To re-enable certain types: - -generator client { - provider = "prisma-client-py" - recursive_type_depth = -1 -} - -If you need to use Mypy, you can also disable this message by explicitly setting the default value: - -generator client { - provider = "prisma-client-py" - recursive_type_depth = 5 -} - -For more information see: https://prisma-client-py.readthedocs.io/en/stable/reference/limitations/#default-type-limitations - - -Warning: The binaryTargets option is not officially supported by Prisma Client Python. - -✔ Generated Prisma Client Python (v0.11.0) to ./.venv/lib/python3.12/site-packages/prisma in 401ms - -cd ui/litellm-dashboard && ../../scripts/with_dashboard_node.sh npm install --no-audit --no-fund -npm warn deprecated node-domexception@1.0.0: Use your platform's native DOMException instead - -added 738 packages in 9s -bootstrap: copied .env from /Users/mateo/Development/litellm -bootstrap: done From 8ec8a2f16c916f42d1b8700a4c90abcce3332c42 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 25 Aug 2026 23:18:28 -0700 Subject: [PATCH 3/6] docs(passthrough): trim credential-filter docstrings to the why --- .../llm_passthrough_endpoints.py | 53 ++++--------------- 1 file changed, 9 insertions(+), 44 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 0234e38c9210..088f4e7fe117 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -49,7 +49,7 @@ create_websocket_passthrough_route, websocket_passthrough_request, ) -from litellm.proxy.utils import hash_token, is_known_model +from litellm.proxy.utils import is_known_model from litellm.proxy.vector_store_endpoints.utils import ( assert_proxy_admin_for_vector_store_index_management, assert_user_can_access_vector_store, @@ -1796,14 +1796,7 @@ def _normalize_credential_value(value: str) -> str: def _operator_configured_caller_key_header_names() -> tuple[str, ...]: - """Operator-configured caller-key header names. - - ``user_api_key_auth`` also reads the caller's key from - ``general_settings.litellm_key_header_name`` and from each - ``general_settings.pass_through_endpoints`` entry's - ``headers.litellm_user_api_key``. Google never consumes either, so both are - dropped by name. - """ + """Operator-configured header names ``user_api_key_auth`` reads the caller's key from.""" from litellm.proxy.proxy_server import general_settings custom_key_header: Final = general_settings.get("litellm_key_header_name") @@ -1823,53 +1816,25 @@ def _operator_configured_caller_key_header_names() -> tuple[str, ...]: def _is_authenticated_caller_secret(value: str, user_api_key_dict: UserAPIKeyAuth) -> bool: - """Whether a header value is a LiteLLM secret that authenticated this caller. - - That is the proxy master key, or a LiteLLM-issued key whose ``hash_token`` is - the ``api_key`` ``user_api_key_auth`` resolved (a DB virtual key, or a custom - auth returning the key it validated). A value auth only echoed unhashed (a - custom auth's own identifier, the no-master-key path returning the raw - header) or never consumed (JWT auth resolves ``api_key=None``) is not a - LiteLLM secret, so a bring-your-own Google credential in that position is - still forwarded. - """ + """Whether a header value is the master key or the key ``user_api_key_auth`` stored as ``api_key``.""" from litellm.proxy.proxy_server import master_key + if master_key is None: + return False normalized: Final = _normalize_credential_value(value) - if master_key is not None and hmac.compare_digest(normalized.encode(), master_key.encode()): + if hmac.compare_digest(normalized.encode(), master_key.encode()): return True authenticated_key: Final = user_api_key_dict.api_key if authenticated_key is None: return False - return hmac.compare_digest(hash_token(normalized).encode(), authenticated_key.encode()) + stored_representation: Final = UserAPIKeyAuth._safe_hash_litellm_api_key(normalized) # pyright: ignore[reportPrivateUsage] # the exact transform auth applied when it stored api_key + return hmac.compare_digest(stored_representation.encode(), authenticated_key.encode()) def _forwarded_headers_for_credentialless_vertex_passthrough( request: Request, user_api_key_dict: UserAPIKeyAuth ) -> Mapping[str, str]: - """ - Header set to forward on the bring-your-own-credentials Vertex passthrough - branch, used when the proxy has no Vertex credential configured. - - No credential the proxy accepts for caller authentication is forwarded to - Google. ``user_api_key_auth`` reads the caller's key from every header in - ``SpecialHeaders.litellm_credential_header_names()``, and Vertex only ever - authenticates with an OAuth token in ``Authorization`` or an API key in - ``x-goog-api-key``. So the proxy-only auth headers Google never consumes - (everything in that set except those two, e.g. ``x-litellm-api-key`` / - ``api-key`` / ``x-api-key`` / ``Ocp-Apim-Subscription-Key``, plus the mapped - pass-through ``litellm_user_api_key`` header and any operator-configured - ``litellm_key_header_name`` / ``pass_through_endpoints`` key header) are dropped - by name. ``Authorization`` and ``x-goog-api-key`` may instead carry a genuine - bring-your-own Google credential, so they are kept unless their value is the - LiteLLM secret that authenticated this caller (``_is_authenticated_caller_secret``), - which is dropped by value wherever it appears. Matching on what actually - authenticated, rather than on header precedence, keeps a caller's own Google - token in ``Authorization`` flowing when a custom auth or JWT authenticated - them without consuming it. When neither a surviving ``Authorization`` nor - ``x-goog-api-key`` remains the request is rejected so the virtual key cannot - leak upstream. - """ + """Caller headers to forward on the bring-your-own-credentials Vertex branch, minus LiteLLM secrets.""" incoming: Final = _safe_get_request_headers(request) never_forwarded: Final = _HEADERS_NEVER_FORWARDED_TO_VERTEX.union( (_MAPPED_ROUTE_CALLER_KEY_HEADER, *_operator_configured_caller_key_header_names()) From 7abd82f5e82aaa11c55f4f3e8ba727b14ce48ad2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 26 Aug 2026 00:19:24 -0700 Subject: [PATCH 4/6] test(passthrough): cover opaque and jwt credential stripping under custom auth --- .../test_llm_pass_through_endpoints.py | 39 ++++++++++++++++--- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 27f2521702a8..3844ff603a48 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -3486,7 +3486,9 @@ async def _run( monkeypatch, headers: list[tuple[bytes, bytes]], authenticated: UserAPIKeyAuth | None = None, + master_key: str | None = "sk-master-1234", ) -> tuple[HTTPException | None, dict | None]: + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", master_key) caller: Final = authenticated if authenticated is not None else UserAPIKeyAuth(api_key=self.VKEY) from litellm.proxy.pass_through_endpoints.passthrough_endpoint_router import ( PassthroughEndpointRouter, @@ -3780,20 +3782,27 @@ async def test_virtual_key_in_mapped_route_litellm_user_api_key_header_alone_is_ GOOGLE_OAUTH_TOKEN = "ya29.byo-google-oauth-token" + LITELLM_JWT = ( + "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJzdWIiOiJqd3Qtc3ViamVjdCIsImlzcyI6Imh0dHBzOi8vaWRwLmV4YW1wbGUuY29tIn0." + "c2lnbmF0dXJl" + ) + @pytest.mark.asyncio @pytest.mark.parametrize( - "authenticated", + ("master_key", "authenticated"), [ pytest.param( + "sk-master-1234", UserAPIKeyAuth(api_key="best-api-key-ever", user_role=LitellmUserRoles.PROXY_ADMIN), id="custom-auth-returning-its-own-identifier", ), - pytest.param(UserAPIKeyAuth(api_key=None, user_id="jwt-subject"), id="jwt-auth"), - pytest.param(UserAPIKeyAuth(api_key=GOOGLE_OAUTH_TOKEN), id="no-master-key-echoes-raw-header"), + pytest.param("sk-master-1234", UserAPIKeyAuth(api_key=None, user_id="jwt-subject"), id="jwt-auth"), + pytest.param(None, UserAPIKeyAuth(api_key=GOOGLE_OAUTH_TOKEN), id="no-master-key-echoes-raw-header"), ], ) async def test_google_token_in_authorization_is_forwarded_when_auth_did_not_consume_it( - self, monkeypatch, authenticated: UserAPIKeyAuth + self, monkeypatch, master_key: str | None, authenticated: UserAPIKeyAuth ): raised, forwarded = await self._run( monkeypatch, @@ -3802,14 +3811,33 @@ async def test_google_token_in_authorization_is_forwarded_when_auth_did_not_cons (b"content-type", b"application/json"), ], authenticated=authenticated, + master_key=master_key, ) assert raised is None, f"the caller's own Google token must not be mistaken for a LiteLLM key: {raised}" assert forwarded is not None assert forwarded.get("authorization") == f"Bearer {self.GOOGLE_OAUTH_TOKEN}" + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("credential", "authenticated"), + [ + pytest.param("modified_key", UserAPIKeyAuth(api_key="modified_key"), id="custom-auth-echoing-opaque-credential"), + pytest.param(LITELLM_JWT, UserAPIKeyAuth(api_key=LITELLM_JWT, user_id="jwt-subject"), id="jwt-auth-consuming-header"), + ], + ) + async def test_non_sk_litellm_credential_that_authenticated_is_rejected_not_forwarded( + self, monkeypatch, credential: str, authenticated: UserAPIKeyAuth + ): + raised, forwarded = await self._run( + monkeypatch, + [(b"authorization", f"Bearer {credential}".encode()), (b"content-type", b"application/json")], + authenticated=authenticated, + ) + assert forwarded is None, "the credential that authenticated the caller must never reach the upstream forwarder" + assert raised is not None and raised.status_code == 401 + @pytest.mark.asyncio async def test_master_key_in_authorization_alone_is_rejected(self, monkeypatch): - monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "sk-master-1234") raised, forwarded = await self._run( monkeypatch, [(b"authorization", b"Bearer sk-master-1234"), (b"content-type", b"application/json")], @@ -3820,7 +3848,6 @@ async def test_master_key_in_authorization_alone_is_rejected(self, monkeypatch): @pytest.mark.asyncio async def test_master_key_is_stripped_and_byo_x_goog_api_key_forwards(self, monkeypatch): - monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "sk-master-1234") raised, forwarded = await self._run( monkeypatch, [ From be0cac1ca7d00d326cd5d28050df8f17be7a5bfd Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 26 Aug 2026 00:37:17 -0700 Subject: [PATCH 5/6] fix(proxy): keep sk-shaped secrets stripped on no-master-key proxies --- .../pass_through_endpoints/llm_passthrough_endpoints.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 088f4e7fe117..6116c719be7f 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -1819,14 +1819,14 @@ def _is_authenticated_caller_secret(value: str, user_api_key_dict: UserAPIKeyAut """Whether a header value is the master key or the key ``user_api_key_auth`` stored as ``api_key``.""" from litellm.proxy.proxy_server import master_key - if master_key is None: - return False normalized: Final = _normalize_credential_value(value) - if hmac.compare_digest(normalized.encode(), master_key.encode()): + if master_key is not None and hmac.compare_digest(normalized.encode(), master_key.encode()): return True authenticated_key: Final = user_api_key_dict.api_key if authenticated_key is None: return False + if master_key is None and not normalized.startswith("sk-"): + return False stored_representation: Final = UserAPIKeyAuth._safe_hash_litellm_api_key(normalized) # pyright: ignore[reportPrivateUsage] # the exact transform auth applied when it stored api_key return hmac.compare_digest(stored_representation.encode(), authenticated_key.encode()) From a3f654719e7467ad22b0235303673960855aca26 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 26 Aug 2026 10:43:04 -0700 Subject: [PATCH 6/6] fix(proxy): strip the JWT that authenticated on credential-less Vertex passthrough --- .../llm_passthrough_endpoints.py | 18 ++++- .../test_llm_pass_through_endpoints.py | 67 +++++++++++++++++-- 2 files changed, 78 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 6116c719be7f..9a3bc82c6fa4 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -29,6 +29,7 @@ from litellm.llms.anthropic.common_utils import AnthropicModelInfo from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.proxy._types import * +from litellm.proxy.auth.handle_jwt import JWTHandler from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.auth.user_api_key_auth import ( _get_bearer_token, @@ -1815,13 +1816,28 @@ def _operator_configured_caller_key_header_names() -> tuple[str, ...]: return override + pass_through +def _is_authenticated_caller_jwt(value: str, jwt_claims: Mapping[str, object]) -> bool: + """Whether a header value is the JWT whose claims ``user_api_key_auth`` stored as ``jwt_claims``.""" + presented_claims: Final = JWTHandler.get_unverified_claims(value) + if presented_claims is None: + return False + return all( + presented_claims.get(name) == claim + for name, claim in jwt_claims.items() + if name not in JWTHandler.LITELLM_INTERNAL_CLAIMS + ) + + def _is_authenticated_caller_secret(value: str, user_api_key_dict: UserAPIKeyAuth) -> bool: - """Whether a header value is the master key or the key ``user_api_key_auth`` stored as ``api_key``.""" + """Whether a header value is the master key, the JWT that authenticated, or the key stored as ``api_key``.""" from litellm.proxy.proxy_server import master_key normalized: Final = _normalize_credential_value(value) if master_key is not None and hmac.compare_digest(normalized.encode(), master_key.encode()): return True + jwt_claims: Final = user_api_key_dict.jwt_claims + if jwt_claims and _is_authenticated_caller_jwt(normalized, jwt_claims): + return True authenticated_key: Final = user_api_key_dict.api_key if authenticated_key is None: return False diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 3844ff603a48..3b506324ad73 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -1,3 +1,4 @@ +import base64 import contextlib import json import os @@ -38,6 +39,7 @@ vllm_proxy_route, ) from litellm.proxy._types import LitellmUserRoles, SpecialHeaders, UserAPIKeyAuth +from litellm.proxy.auth.handle_jwt import JWTHandler from litellm.types.passthrough_endpoints.vertex_ai import VertexPassThroughCredentials @@ -3448,6 +3450,13 @@ def test_is_passthrough_request_streaming_tolerates_non_object_bodies(request_bo assert is_passthrough_request_streaming(request_body) is expected +def _unsigned_jwt(claims: Mapping[str, str]) -> str: + def segment(payload: Mapping[str, str]) -> str: + return base64.urlsafe_b64encode(json.dumps(dict(payload)).encode()).rstrip(b"=").decode() + + return ".".join((segment({"alg": "RS256", "typ": "JWT"}), segment(claims), "c2lnbmF0dXJl")) + + class TestVertexCredentiallessPassthroughVirtualKeyLeak: """Regression coverage for LIT-5997. @@ -3782,10 +3791,14 @@ async def test_virtual_key_in_mapped_route_litellm_user_api_key_header_alone_is_ GOOGLE_OAUTH_TOKEN = "ya29.byo-google-oauth-token" - LITELLM_JWT = ( - "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9." - "eyJzdWIiOiJqd3Qtc3ViamVjdCIsImlzcyI6Imh0dHBzOi8vaWRwLmV4YW1wbGUuY29tIn0." - "c2lnbmF0dXJl" + LITELLM_JWT_CLAIMS = MappingProxyType({"sub": "jwt-subject", "iss": "https://idp.example.com"}) + LITELLM_JWT = _unsigned_jwt(LITELLM_JWT_CLAIMS) + GOOGLE_SERVICE_ACCOUNT_JWT = _unsigned_jwt( + { + "sub": "vertex-caller@my-proj.iam.gserviceaccount.com", + "iss": "vertex-caller@my-proj.iam.gserviceaccount.com", + "aud": "https://aiplatform.googleapis.com/", + } ) @pytest.mark.asyncio @@ -3797,7 +3810,11 @@ async def test_virtual_key_in_mapped_route_litellm_user_api_key_header_alone_is_ UserAPIKeyAuth(api_key="best-api-key-ever", user_role=LitellmUserRoles.PROXY_ADMIN), id="custom-auth-returning-its-own-identifier", ), - pytest.param("sk-master-1234", UserAPIKeyAuth(api_key=None, user_id="jwt-subject"), id="jwt-auth"), + pytest.param( + "sk-master-1234", + UserAPIKeyAuth(api_key=None, user_id="jwt-subject", jwt_claims=dict(LITELLM_JWT_CLAIMS)), + id="jwt-auth", + ), pytest.param(None, UserAPIKeyAuth(api_key=GOOGLE_OAUTH_TOKEN), id="no-master-key-echoes-raw-header"), ], ) @@ -3822,7 +3839,29 @@ async def test_google_token_in_authorization_is_forwarded_when_auth_did_not_cons ("credential", "authenticated"), [ pytest.param("modified_key", UserAPIKeyAuth(api_key="modified_key"), id="custom-auth-echoing-opaque-credential"), - pytest.param(LITELLM_JWT, UserAPIKeyAuth(api_key=LITELLM_JWT, user_id="jwt-subject"), id="jwt-auth-consuming-header"), + pytest.param( + LITELLM_JWT, + UserAPIKeyAuth(api_key=LITELLM_JWT, user_id="jwt-subject"), + id="custom-auth-echoing-jwt", + ), + pytest.param( + LITELLM_JWT, + UserAPIKeyAuth(api_key=None, user_id="jwt-subject", jwt_claims=dict(LITELLM_JWT_CLAIMS)), + id="jwt-auth", + ), + pytest.param( + LITELLM_JWT, + UserAPIKeyAuth( + api_key=None, + user_id="jwt-subject", + jwt_claims={ + **LITELLM_JWT_CLAIMS, + JWTHandler.LITELLM_JWT_ISSUER_CLAIM: "https://idp.example.com", + JWTHandler.LITELLM_USER_ID_CLAIM: "jwt-subject", + }, + ), + id="multi-issuer-jwt-auth-normalized-claims", + ), ], ) async def test_non_sk_litellm_credential_that_authenticated_is_rejected_not_forwarded( @@ -3836,6 +3875,22 @@ async def test_non_sk_litellm_credential_that_authenticated_is_rejected_not_forw assert forwarded is None, "the credential that authenticated the caller must never reach the upstream forwarder" assert raised is not None and raised.status_code == 401 + @pytest.mark.asyncio + async def test_jwt_authenticated_caller_keeps_a_different_byo_google_jwt(self, monkeypatch): + raised, forwarded = await self._run( + monkeypatch, + [ + (b"x-litellm-api-key", self.LITELLM_JWT.encode()), + (b"authorization", f"Bearer {self.GOOGLE_SERVICE_ACCOUNT_JWT}".encode()), + (b"content-type", b"application/json"), + ], + authenticated=UserAPIKeyAuth(api_key=None, user_id="jwt-subject", jwt_claims=dict(self.LITELLM_JWT_CLAIMS)), + ) + assert raised is None, f"a Google JWT that is not the one that authenticated must keep flowing: {raised}" + assert forwarded is not None + assert forwarded.get("authorization") == f"Bearer {self.GOOGLE_SERVICE_ACCOUNT_JWT}" + assert "x-litellm-api-key" not in forwarded + @pytest.mark.asyncio async def test_master_key_in_authorization_alone_is_rejected(self, monkeypatch): raised, forwarded = await self._run(