Skip to content
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 117 additions & 8 deletions litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
import json
import os
import re
from collections.abc import Callable
from collections.abc import Callable, Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Annotated, Any, Final, cast

Expand All @@ -29,7 +29,11 @@
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
from litellm.proxy._types import *
from litellm.proxy.auth.route_checks import RouteChecks
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth, user_api_key_auth_websocket
from litellm.proxy.auth.user_api_key_auth import (
_get_bearer_token,
user_api_key_auth,
user_api_key_auth_websocket,
)
from litellm.proxy.common_utils.http_parsing_utils import (
_read_request_body,
_safe_get_request_headers,
Expand Down Expand Up @@ -1726,6 +1730,111 @@ def _override_vertex_params_from_router_credentials(
return vertex_project, vertex_location


_CREDENTIALLESS_VERTEX_MISSING_CREDENTIAL_DETAIL: Final = (
"No Vertex AI credential is configured on this proxy and the request carried no upstream "
"Google credential. The LiteLLM virtual key is not forwarded to Google. Configure a Vertex "
"credential (DEFAULT_VERTEXAI_PROJECT / DEFAULT_VERTEXAI_LOCATION / DEFAULT_VERTEXAI_CREDENTIALS, "
"or a model with use_in_pass_through: true), or send your own Google OAuth token in the "
"Authorization header."
)


def _normalize_credential_value(value: str) -> str:
"""Reduce a header value to the bare token, matching how ``user_api_key_auth``
reads a caller's key.

Reuses the auth module's ``_get_bearer_token`` so the caller-key comparison
strips exactly the schemes authentication accepts (``Bearer`` / ``bearer`` /
``Basic`` / ``AWS4-HMAC-SHA256`` credential), rather than re-deriving a
narrower normalization here. ``_get_bearer_token`` returns ``""`` for a value
with no recognized scheme prefix, so a bare token (or a real Google
credential that carries no scheme) falls back to its own value.
"""
return _get_bearer_token(value) or value


_VERTEX_UPSTREAM_CREDENTIAL_HEADERS: Final = frozenset({"authorization", "x-goog-api-key"})
_HEADERS_NEVER_FORWARDED_TO_VERTEX: Final = frozenset({"content-length", "host"}) | (
SpecialHeaders.litellm_credential_header_names() - _VERTEX_UPSTREAM_CREDENTIAL_HEADERS
)


_VERTEX_CALLER_KEY_HEADER_PRECEDENCE: Final = (
SpecialHeaders.custom_litellm_api_key.value.lower(),
SpecialHeaders.openai_authorization.value.lower(),
SpecialHeaders.azure_authorization.value.lower(),
SpecialHeaders.anthropic_authorization.value.lower(),
SpecialHeaders.google_ai_studio_authorization.value.lower(),
SpecialHeaders.azure_apim_authorization.value.lower(),
)
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved


def _authenticated_caller_key_values(request: Request) -> frozenset[str]:
"""The value ``user_api_key_auth`` would accept as this caller's LiteLLM key.

The Vertex route authenticates through ``Depends(user_api_key_auth)``, which
resolves the key from the first present of the credential headers in
``get_api_key``'s precedence order, with the operator-configured
``general_settings.litellm_key_header_name`` overriding all of them. Some of
those headers (``Authorization``, ``x-goog-api-key``) are also kept as genuine
bring-your-own Google credentials, so returning only the value that actually
authenticated lets the filter strip that value wherever it appears while
leaving a real Google credential in place. An empty set means no caller key
was found, so nothing is value-stripped.
"""
from litellm.proxy.proxy_server import general_settings

incoming: Final = _safe_get_request_headers(request)
custom_key_header_name: Final = (general_settings.get("litellm_key_header_name") or "").lower()
ordered_names: Final = (
(custom_key_header_name,) if custom_key_header_name else ()
) + _VERTEX_CALLER_KEY_HEADER_PRECEDENCE
present_values: Final = (incoming[name] for name in ordered_names if incoming.get(name))
authenticated_key: Final = next(
(stripped for value in present_values if (stripped := _normalize_credential_value(value))),
"",
)
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
return frozenset({authenticated_key}) if authenticated_key else frozenset()
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved


def _forwarded_headers_for_credentialless_vertex_passthrough(request: Request) -> Mapping[str, str]:
"""
Header set to forward on the bring-your-own-credentials Vertex passthrough
branch, used when the proxy has no Vertex credential configured.

No credential the proxy accepts for caller authentication is forwarded to
Google. ``user_api_key_auth`` reads the caller's key from every header in
``SpecialHeaders.litellm_credential_header_names()``, and Vertex only ever
authenticates with an OAuth token in ``Authorization`` or an API key in
``x-goog-api-key``. So the proxy-only auth headers Google never consumes
(everything in that set except those two, e.g. ``x-litellm-api-key`` /
``api-key`` / ``x-api-key`` / ``Ocp-Apim-Subscription-Key``) are dropped by
name. ``Authorization`` and ``x-goog-api-key`` may instead carry a genuine
bring-your-own Google credential, so they are kept unless their value is the
caller's authenticated LiteLLM key, which is dropped by value (normalizing any
``Bearer`` / ``Basic`` / ``AWS4`` auth-scheme prefix the same way authentication does). Because the value that authenticated is resolved by the
same precedence ``user_api_key_auth`` uses, a virtual key sent only in
``x-goog-api-key`` (or in the operator-configured ``litellm_key_header_name``)
is dropped too, while a real Google key in ``x-goog-api-key`` alongside a
virtual key in a higher-precedence header is preserved. When neither a
surviving ``Authorization`` nor ``x-goog-api-key`` remains the request is
rejected so the virtual key cannot leak upstream.
"""
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
incoming: Final = _safe_get_request_headers(request)
caller_key_values: Final = _authenticated_caller_key_values(request)
forwarded: Final = MappingProxyType(
{
name: value
for name, value in incoming.items()
if name not in _HEADERS_NEVER_FORWARDED_TO_VERTEX
and _normalize_credential_value(value) not in caller_key_values
}
)
if "authorization" not in forwarded and "x-goog-api-key" not in forwarded:
raise HTTPException(status_code=401, detail=_CREDENTIALLESS_VERTEX_MISSING_CREDENTIAL_DETAIL)
return forwarded


async def _prepare_vertex_auth_headers(
request: Request,
vertex_credentials: Any | None,
Expand All @@ -1734,7 +1843,7 @@ async def _prepare_vertex_auth_headers(
vertex_location: str | None,
base_target_url: str | None,
get_vertex_pass_through_handler: BaseVertexAIPassThroughHandler,
) -> tuple[dict, str | None, bool, str | None, str | None]:
) -> tuple[Mapping[str, str], str | None, bool, str | None, str | None]:
"""
Prepare authentication headers for Vertex AI pass-through requests.

Expand All @@ -1760,11 +1869,11 @@ async def _prepare_vertex_auth_headers(

# Use headers from the incoming request if no vertex credentials are found
if (vertex_credentials is None or vertex_credentials.vertex_project is None) and router_credentials is None:
headers = _safe_get_request_headers(request).copy()
headers = _forwarded_headers_for_credentialless_vertex_passthrough(request)
headers_passed_through = True
verbose_proxy_logger.debug("default_vertex_config not set, incoming request headers %s", headers)
headers.pop("content-length", None)
headers.pop("host", None)
verbose_proxy_logger.debug(
"default_vertex_config not set, forwarding caller-provided headers %s", tuple(headers.keys())
)
else:
if router_credentials is not None:
vertex_credentials_str = None
Expand Down Expand Up @@ -1850,7 +1959,7 @@ async def _base_vertex_proxy_route(

encoded_endpoint = httpx.URL(endpoint).path
verbose_proxy_logger.debug("requested endpoint %s", endpoint)
headers: dict = {}
headers: Mapping[str, str] = {}
api_key_to_use = get_litellm_virtual_key(request=request)
user_api_key_dict = await user_api_key_auth(
request=request,
Expand Down
Loading
Loading