Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 14 additions & 1 deletion litellm/llms/bedrock/common_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
from litellm.types.llms.bedrock import AWS_AUTH_PARAM_KEYS, AwsAuthParams

if TYPE_CHECKING:
from litellm.llms.custom_httpx.http_handler import HTTPHandler
from litellm.types.llms.openai import AllMessageValues


Expand Down Expand Up @@ -1327,6 +1328,10 @@ class BedrockModelInfo(BaseLLMModelInfo):
global_config = AmazonBedrockGlobalConfig()
all_global_regions = global_config.get_all_regions()

def __init__(self, client: HTTPHandler | None = None) -> None:
super().__init__()
self._client: Final = client

@staticmethod
def get_api_base(api_base: str | None = None) -> str | None:
"""
Expand Down Expand Up @@ -1354,7 +1359,15 @@ def validate_environment(
return headers

def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]:
return []
return self.discover_models({"api_key": api_key})

def discover_models(
self, litellm_params: Mapping[str, object] | None = None
) -> list[str]: # mutable-ok: matches get_models' list[str] contract shared by every provider override
from litellm.llms.bedrock.model_listing import BedrockModelLister

client: Final = self._client if self._client is not None else litellm.module_level_client
return sorted(BedrockModelLister(deployment=litellm_params or {}, client=client).invocable_model_ids())

# def get_provider_info(self, model: str) -> Optional[ProviderSpecificModelInfo]:
# """
Expand Down
123 changes: 123 additions & 0 deletions litellm/llms/bedrock/model_listing.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
from collections.abc import Iterator, Mapping
from types import MappingProxyType
from typing import Final, TypeAlias
from urllib.parse import quote, urlencode

import httpx
from pydantic import BaseModel, ConfigDict, Field

from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.llms.custom_httpx.http_handler import HTTPHandler
from litellm.types.llms.bedrock import AwsAuthParams

MODEL_PREFIX: Final = "bedrock/"
LIST_MODELS_TIMEOUT: Final = 10.0
INFERENCE_PROFILES_PAGE_SIZE: Final = 1000
INFERENCE_PROFILES_PAGE_CAP: Final = 20

QueryPairs: TypeAlias = tuple[tuple[str, str], ...]


class FoundationModelSummary(BaseModel):
model_config = ConfigDict(extra="ignore")

id: str = Field(alias="modelId")


class ListFoundationModelsResponse(BaseModel):
model_config = ConfigDict(extra="ignore")

summaries: tuple[FoundationModelSummary, ...] = Field(default=(), alias="modelSummaries")


class InferenceProfileSummary(BaseModel):
model_config = ConfigDict(extra="ignore")

id: str = Field(alias="inferenceProfileId")
status: str


class ListInferenceProfilesResponse(BaseModel):
model_config = ConfigDict(extra="ignore")

summaries: tuple[InferenceProfileSummary, ...] = Field(default=(), alias="inferenceProfileSummaries")
next_token: str | None = Field(default=None, alias="nextToken")


class BedrockModelLister(BaseAWSLLM):
def __init__(self, deployment: Mapping[str, object], client: HTTPHandler) -> None:
super().__init__()
self._aws_params: Final = dict(deployment)
self._aws_region_name: Final = self._get_aws_region_name(optional_params=self._aws_params)
api_key: Final = deployment.get("api_key")
self._bearer_token: Final = bedrock_bearer_token(api_key if isinstance(api_key, str) else None)
self._client: Final = client

def invocable_model_ids(self) -> frozenset[str]:
model_ids: Final = frozenset(self._active_inference_profile_ids()) | frozenset(
self._on_demand_foundation_model_ids()
)
return frozenset(MODEL_PREFIX + model_id for model_id in model_ids)

def _on_demand_foundation_model_ids(self) -> Iterator[str]:
response: Final = ListFoundationModelsResponse.model_validate(
self._get_json("/foundation-models", (("byInferenceType", "ON_DEMAND"),))
)
return (summary.id for summary in response.summaries)

def _active_inference_profile_ids(self) -> tuple[str, ...]:
collected: tuple[str, ...] = () # rebind-ok: accumulates one page of ids per iteration
next_token: str | None = None # rebind-ok: advances to each page's nextToken
for _ in range(INFERENCE_PROFILES_PAGE_CAP):
page = self._inference_profiles_page(next_token)
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
collected += tuple(summary.id for summary in page.summaries if summary.status == "ACTIVE")
if page.next_token is None:
return collected
next_token = page.next_token
raise BedrockError(
status_code=500,
message=(
f"Bedrock inference profile listing in {self._aws_region_name} did not end within "
f"{INFERENCE_PROFILES_PAGE_CAP} pages."
),
)

def _inference_profiles_page(self, next_token: str | None) -> ListInferenceProfilesResponse:
continuation: Final[QueryPairs] = (("nextToken", next_token),) if next_token is not None else ()
return ListInferenceProfilesResponse.model_validate(
self._get_json(
"/inference-profiles",
(("maxResults", str(INFERENCE_PROFILES_PAGE_SIZE)), ("typeEquals", "SYSTEM_DEFINED"), *continuation),
)
)

def _get_json(self, path: str, query: QueryPairs) -> object:
host: Final = f"bedrock.{self._aws_region_name}.{get_aws_dns_suffix(self._aws_region_name)}"
url: Final = f"https://{host}{path}?{urlencode(query, quote_via=quote)}"
response: Final = self._client.get(url=url, headers=self._auth_headers(url), timeout=LIST_MODELS_TIMEOUT)
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
try:
response.raise_for_status()
except httpx.HTTPStatusError:
raise BedrockError(
status_code=response.status_code,
message=(
f"Failed to list Bedrock models in {self._aws_region_name}. "
f"Status code: {response.status_code}, Response: {response.text}"
),
)
return response.json()

def _auth_headers(self, url: str) -> Mapping[str, str]:
if self._bearer_token is not None:
return MappingProxyType({"Authorization": f"Bearer {self._bearer_token}"})
from botocore.auth import SigV4Auth
from botocore.awsrequest import AWSRequest

credentials: Final = self.resolve_credentials(
AwsAuthParams.model_validate(self._aws_params), self._aws_region_name
)
request: Final = AWSRequest(method="GET", url=url, data="")
SigV4Auth(credentials, "bedrock", self._aws_region_name).add_auth(request)
return MappingProxyType({name: str(value) for name, value in request.prepare().headers.items()})
166 changes: 166 additions & 0 deletions tests/unit/llms/bedrock/test_bedrock_model_listing.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
import hashlib
import hmac
import re
from collections.abc import Callable, Mapping
from functools import reduce
from typing import Final
from urllib.parse import parse_qsl, quote

import httpx
import pytest

from litellm.llms.bedrock.common_utils import BedrockError, BedrockModelInfo
from litellm.llms.custom_httpx.http_handler import HTTPHandler
from litellm.types.router import LiteLLM_Params

ACCESS_KEY: Final = "AKIAIOSFODNN7EXAMPLE"
SECRET_KEY: Final = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
REGION: Final = "eu-central-1"
BEARER_TOKEN: Final = "bedrock-api-key-for-tests"
SECOND_PAGE_TOKEN: Final = "page2+token/with=reserved chars"

ON_DEMAND_MODELS: Final = (
{"modelId": "anthropic.claude-3-haiku-20240307-v1:0", "inferenceTypesSupported": ["ON_DEMAND"]},
{"modelId": "amazon.nova-micro-v1:0", "inferenceTypesSupported": ["ON_DEMAND"]},
)
PROFILE_ONLY_MODEL: Final = {
"modelId": "anthropic.claude-opus-4-5-20251101-v1:0",
"inferenceTypesSupported": ["INFERENCE_PROFILE"],
}
PROFILE_PAGES: Final = {
None: {
"inferenceProfileSummaries": [
{"inferenceProfileId": "global.anthropic.claude-opus-4-5-20251101-v1:0", "status": "ACTIVE"},
{"inferenceProfileId": "eu.anthropic.claude-3-haiku-20240307-v1:0", "status": "INACTIVE"},
],
"nextToken": SECOND_PAGE_TOKEN,
},
SECOND_PAGE_TOKEN: {
"inferenceProfileSummaries": [{"inferenceProfileId": "eu.amazon.nova-micro-v1:0", "status": "ACTIVE"}],
},
}
EXPECTED_MODELS: Final = [
"bedrock/amazon.nova-micro-v1:0",
"bedrock/anthropic.claude-3-haiku-20240307-v1:0",
"bedrock/eu.amazon.nova-micro-v1:0",
"bedrock/global.anthropic.claude-opus-4-5-20251101-v1:0",
]
SIGV4_AUTHORIZATION: Final = re.compile(
rf"^AWS4-HMAC-SHA256 Credential={ACCESS_KEY}/\d{{8}}/{REGION}/bedrock/aws4_request, "
r"SignedHeaders=([^,]+), Signature=([0-9a-f]{64})$"
)


@pytest.fixture(autouse=True)
def no_ambient_bearer_token(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)


def _sigv4_signature_as_aws_computes_it(request: httpx.Request, signed_headers: tuple[str, ...]) -> str:
amz_date: Final = request.headers["x-amz-date"]
canonical_query: Final = "&".join(
f"{quote(name, safe='-_.~')}={quote(value, safe='-_.~')}"
for name, value in sorted(parse_qsl(request.url.query.decode(), keep_blank_values=True))
)
canonical_headers: Final = "".join(f"{name}:{request.headers[name].strip()}\n" for name in signed_headers)
canonical_request: Final = "\n".join(
(
request.method,
request.url.path,
canonical_query,
canonical_headers,
";".join(signed_headers),
hashlib.sha256(request.content).hexdigest(),
)
)
scope: Final = f"{amz_date[:8]}/{REGION}/bedrock/aws4_request"
string_to_sign: Final = "\n".join(
("AWS4-HMAC-SHA256", amz_date, scope, hashlib.sha256(canonical_request.encode()).hexdigest())
)
signing_key: Final = reduce(
lambda key, part: hmac.new(key, part.encode(), hashlib.sha256).digest(),
(amz_date[:8], REGION, "bedrock", "aws4_request"),
f"AWS4{SECRET_KEY}".encode(),
)
return hmac.new(signing_key, string_to_sign.encode(), hashlib.sha256).hexdigest()


def _authorized(request: httpx.Request, bearer_token: str | None) -> bool:
authorization: Final = request.headers.get("authorization", "")
if bearer_token is not None:
return authorization == f"Bearer {bearer_token}"
signed: Final = SIGV4_AUTHORIZATION.match(authorization)
if signed is None or "x-amz-date" not in request.headers:
return False
signed_headers: Final = tuple(signed.group(1).split(";"))
if "host" not in signed_headers or "x-amz-date" not in signed_headers:
return False
return signed.group(2) == _sigv4_signature_as_aws_computes_it(request, signed_headers)


def _control_plane(bearer_token: str | None = None) -> Callable[[httpx.Request], httpx.Response]:
def handler(request: httpx.Request) -> httpx.Response:
if request.url.host != f"bedrock.{REGION}.amazonaws.com":
return httpx.Response(404, json={"message": f"no Bedrock at {request.url.host}"})
if not _authorized(request, bearer_token):
return httpx.Response(403, json={"message": "not signed for this account, region and service"})
if request.url.path == "/foundation-models":
on_demand_only: Final = request.url.params.get("byInferenceType") == "ON_DEMAND"
summaries: Final = ON_DEMAND_MODELS if on_demand_only else (*ON_DEMAND_MODELS, PROFILE_ONLY_MODEL)
return httpx.Response(200, json={"modelSummaries": list(summaries)})
if request.url.path == "/inference-profiles":
if request.url.params.get("typeEquals") != "SYSTEM_DEFINED":
return httpx.Response(400, json={"message": "application profiles are not listable here"})
return httpx.Response(200, json=PROFILE_PAGES[request.url.params.get("nextToken")])
return httpx.Response(404, json={"message": f"unknown path {request.url.path}"})

return handler


def _bedrock(handler: Callable[[httpx.Request], httpx.Response]) -> BedrockModelInfo:
return BedrockModelInfo(client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handler))))


def _deployment(region: str = REGION, api_key: str | None = None) -> Mapping[str, object]:
return LiteLLM_Params(
model="bedrock/*",
aws_access_key_id=ACCESS_KEY,
aws_secret_access_key=SECRET_KEY,
aws_region_name=region,
api_key=api_key,
).model_dump(exclude_none=True)


def test_lists_active_profiles_and_on_demand_models_signed_for_the_deployment() -> None:
assert _bedrock(_control_plane()).discover_models(_deployment()) == EXPECTED_MODELS


def test_lists_in_the_deployment_region_not_the_ambient_one(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("AWS_REGION_NAME", REGION)

with pytest.raises(BedrockError, match=r"us-west-2.*404"):
_bedrock(_control_plane()).discover_models(_deployment(region="us-west-2"))


def test_bearer_token_api_key_replaces_sigv4() -> None:
bedrock: Final = _bedrock(_control_plane(bearer_token=BEARER_TOKEN))

assert bedrock.discover_models(_deployment(api_key=BEARER_TOKEN)) == EXPECTED_MODELS
with pytest.raises(BedrockError, match="403"):
bedrock.discover_models(_deployment())


def test_get_models_without_a_deployment_uses_ambient_credentials(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("AWS_ACCESS_KEY_ID", ACCESS_KEY)
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", SECRET_KEY)
monkeypatch.setenv("AWS_REGION_NAME", REGION)

assert _bedrock(_control_plane()).get_models() == EXPECTED_MODELS


def test_listing_failure_names_the_region_and_response() -> None:
def throttled(request: httpx.Request) -> httpx.Response:
return httpx.Response(429, json={"message": "Too many requests"})

with pytest.raises(BedrockError, match=rf"{REGION}.*429.*Too many requests"):
_bedrock(throttled).discover_models(_deployment())
50 changes: 49 additions & 1 deletion tests/unit/test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
from litellm.types.caching import CachingSupportedCallTypes
from litellm.types.integrations.custom_logger import HEADROOM_CONVERTED_STREAM_KEY
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.router import CredentialLiteLLMParams, GenericLiteLLMParams
from litellm.types.router import CredentialLiteLLMParams, GenericLiteLLMParams, LiteLLM_Params
from litellm.types.utils import (
ADDRESSED_RESPONSE_ID_FIELD,
CallTypes,
Expand Down Expand Up @@ -2526,6 +2526,54 @@ def test_get_valid_models_with_cli_pattern(self):
assert headers.get("Authorization") == "Bearer sk-test-cli-key-123"


@respx.mock
def test_get_valid_models_bedrock_lists_what_the_deployment_credentials_can_invoke(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
region: Final = "eu-west-3"
deployment: Final = LiteLLM_Params(
model="bedrock/*",
aws_access_key_id="AKIAIOSFODNN7EXAMPLE",
aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
aws_region_name=region,
)
expected_scope: Final = f"/{region}/bedrock/aws4_request"

def signed_by_the_deployment(request: httpx.Request, body: dict[str, object]) -> httpx.Response:
authorization: Final = request.headers.get("authorization", "")
if not authorization.startswith("AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/"):
return httpx.Response(403, json={"message": "not signed with the deployment's key"})
if expected_scope not in authorization:
return httpx.Response(403, json={"message": "not signed for the deployment's region"})
return httpx.Response(200, json=body)

respx.get(
f"https://bedrock.{region}.amazonaws.com/foundation-models", params={"byInferenceType": "ON_DEMAND"}
).mock(
side_effect=lambda request: signed_by_the_deployment(
request, {"modelSummaries": [{"modelId": "amazon.nova-micro-v1:0"}]}
)
)
respx.get(
f"https://bedrock.{region}.amazonaws.com/inference-profiles",
params={"typeEquals": "SYSTEM_DEFINED"},
).mock(
side_effect=lambda request: signed_by_the_deployment(
request,
{
"inferenceProfileSummaries": [
{"inferenceProfileId": "eu.anthropic.claude-sonnet-4-5-20250929-v1:0", "status": "ACTIVE"}
]
},
)
)

assert litellm.get_valid_models(
check_provider_endpoint=True, custom_llm_provider="bedrock", litellm_params=deployment
) == ["bedrock/amazon.nova-micro-v1:0", "bedrock/eu.anthropic.claude-sonnet-4-5-20250929-v1:0"]


class TestIsCachedMessage:
"""Test is_cached_message function for context caching detection.

Expand Down
Loading