fix(bedrock): list the account's invocable models behind bedrock/* when check_provider_endpoint is on

litellm/llms/base_llm/base_utils.py

@@ -5,7 +5,7 @@
 import copy
 import json
 from abc import ABC, abstractmethod
-from typing import Any, Final
+from typing import TYPE_CHECKING, Any, Final
 
 from openai.lib import _parsing, _pydantic
 from pydantic import BaseModel
@@ -14,6 +14,9 @@
 from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk
 from litellm.types.utils import Message, ProviderSpecificModelInfo, TokenCountResponse
 
+if TYPE_CHECKING:
+    from litellm.types.router import LiteLLM_Params
+
 
 class BaseTokenCounter(ABC):
     @abstractmethod
@@ -65,6 +68,17 @@ def get_models(self, api_key: str | None = None, api_base: str | None = None) ->
         """
         return []
 
+    def get_models_for_deployment(self, litellm_params: "LiteLLM_Params | None") -> list[str]:
+        """
+        Returns the models the deployment described by `litellm_params` can invoke. The default only reads its
+        api_key and api_base; a provider whose listing needs more of the deployment (cloud credentials, a region)
+        overrides this.
+        """
+        return self.get_models(
+            api_key=litellm_params.api_key if litellm_params is not None else None,
+            api_base=litellm_params.api_base if litellm_params is not None else None,
+        )
+
     @staticmethod
     @abstractmethod
     def get_api_key(api_key: str | None = None) -> str | None:

litellm/llms/bedrock/common_utils.py

@@ -32,7 +32,9 @@
 from litellm.types.llms.bedrock import AWS_AUTH_PARAM_KEYS, AwsAuthParams
 
 if TYPE_CHECKING:
+    from litellm.llms.custom_httpx.http_handler import HTTPHandler
     from litellm.types.llms.openai import AllMessageValues
+    from litellm.types.router import LiteLLM_Params
 
 
 _ERROR_REQUEST_URL: Final = "https://docs.litellm.ai/docs"
@@ -1073,6 +1075,9 @@ class BedrockModelInfo(BaseLLMModelInfo):
     global_config = AmazonBedrockGlobalConfig()
     all_global_regions = global_config.get_all_regions()
 
+    def __init__(self, client: HTTPHandler | None = None) -> None:
+        self._client: Final = client
+
     @staticmethod
     def get_api_base(api_base: str | None = None) -> str | None:
         """
@@ -1100,7 +1105,17 @@ def validate_environment(
         return headers
 
     def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]:
-        return []
+        from litellm.types.router import LiteLLM_Params
+
+        return self.get_models_for_deployment(LiteLLM_Params(model="bedrock/*", api_key=api_key))
+
+    def get_models_for_deployment(self, litellm_params: LiteLLM_Params | None) -> list[str]:
+        from litellm.llms.bedrock.model_listing import BedrockModelLister
+
+        if litellm_params is None:
+            return self.get_models()
+        client: Final = self._client if self._client is not None else litellm.module_level_client
+        return sorted(BedrockModelLister(deployment=litellm_params, client=client).invocable_model_ids())
 
     # def get_provider_info(self, model: str) -> Optional[ProviderSpecificModelInfo]:
     #     """

litellm/llms/bedrock/model_listing.py

@@ -0,0 +1,103 @@
+from collections.abc import Iterator, Mapping
+from types import MappingProxyType
+from typing import Final
+from urllib.parse import urlencode
+
+import httpx
+from pydantic import BaseModel, ConfigDict, Field
+
+from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
+from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token
+from litellm.llms.custom_httpx.http_handler import HTTPHandler
+from litellm.types.llms.bedrock import AwsAuthParams
+from litellm.types.router import LiteLLM_Params
+
+MODEL_PREFIX: Final = "bedrock/"
+LIST_MODELS_TIMEOUT: Final = 10.0
+INFERENCE_PROFILES_PAGE_SIZE: Final = 1000
+
+QueryPairs = tuple[tuple[str, str], ...]
+
+
+class FoundationModelSummary(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    id: str = Field(alias="modelId")
+
+
+class ListFoundationModelsResponse(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    summaries: tuple[FoundationModelSummary, ...] = Field(default=(), alias="modelSummaries")
+
+
+class InferenceProfileSummary(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    id: str = Field(alias="inferenceProfileId")
+    status: str
+
+
+class ListInferenceProfilesResponse(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    summaries: tuple[InferenceProfileSummary, ...] = Field(default=(), alias="inferenceProfileSummaries")
+    next_token: str | None = Field(default=None, alias="nextToken")
+
+
+class BedrockModelLister(BaseAWSLLM):
+    def __init__(self, deployment: LiteLLM_Params, client: HTTPHandler) -> None:
+        super().__init__()
+        self._aws_params: Final = deployment.model_dump(exclude_none=True)
+        self._aws_region_name: Final = self._get_aws_region_name(optional_params=self._aws_params)
+        self._bearer_token: Final = bedrock_bearer_token(deployment.api_key)
+        self._client: Final = client
+
+    def invocable_model_ids(self) -> frozenset[str]:
+        model_ids: Final = frozenset(self._active_inference_profile_ids(None)) | frozenset(
+            self._on_demand_foundation_model_ids()
+        )
+        return frozenset(MODEL_PREFIX + model_id for model_id in model_ids)
+
+    def _on_demand_foundation_model_ids(self) -> Iterator[str]:
+        response: Final = ListFoundationModelsResponse.model_validate(
+            self._get_json("/foundation-models", (("byInferenceType", "ON_DEMAND"),))
+        )
+        return (summary.id for summary in response.summaries)
+
+    def _active_inference_profile_ids(self, next_token: str | None) -> Iterator[str]:
+        continuation: Final[QueryPairs] = (("nextToken", next_token),) if next_token is not None else ()
+        page: Final = ListInferenceProfilesResponse.model_validate(
+            self._get_json(
+                "/inference-profiles",
+                (("maxResults", str(INFERENCE_PROFILES_PAGE_SIZE)), ("typeEquals", "SYSTEM_DEFINED"), *continuation),
+            )
+        )
+        yield from (summary.id for summary in page.summaries if summary.status == "ACTIVE")
+        if page.next_token is not None:
+            yield from self._active_inference_profile_ids(page.next_token)
+
+    def _get_json(self, path: str, query: QueryPairs) -> object:
+        url: Final = f"https://bedrock.{self._aws_region_name}.{get_aws_dns_suffix(self._aws_region_name)}{path}?{urlencode(query)}"
+        response: Final = self._client.get(url=url, headers=self._auth_headers(url), timeout=LIST_MODELS_TIMEOUT)
+        try:
+            response.raise_for_status()
+        except httpx.HTTPStatusError:
+            raise Exception(
+                f"Failed to list Bedrock models in {self._aws_region_name}. "
+                f"Status code: {response.status_code}, Response: {response.text}"
+            )
+        return response.json()
+
+    def _auth_headers(self, url: str) -> Mapping[str, str]:
+        if self._bearer_token is not None:
+            return MappingProxyType({"Authorization": f"Bearer {self._bearer_token}"})
+        from botocore.auth import SigV4Auth
+        from botocore.awsrequest import AWSRequest
+
+        credentials: Final = self.resolve_credentials(
+            AwsAuthParams.model_validate(self._aws_params), self._aws_region_name
+        )
+        request: Final = AWSRequest(method="GET", url=url, data="")
+        SigV4Auth(credentials, "bedrock", self._aws_region_name).add_auth(request)
+        return MappingProxyType({name: str(value) for name, value in request.prepare().headers.items()})

litellm/llms/custom_httpx/http_handler.py

@@ -1436,7 +1436,7 @@ def get(
         self,
         url: str,
         params: dict | None = None,
-        headers: dict | None = None,
+        headers: Mapping[str, str] | None = None,
         follow_redirects: bool | None = None,
         timeout: float | httpx.Timeout | None = None,
     ):

litellm/utils.py

@@ -7790,10 +7790,7 @@ def _get_valid_models_from_provider_api(
 
         if cached_result is not None:
             return cached_result
-        models: Final = provider_config.get_models(
-            api_key=litellm_params.api_key if litellm_params is not None else None,
-            api_base=litellm_params.api_base if litellm_params is not None else None,
-        )
+        models: Final = provider_config.get_models_for_deployment(litellm_params)
 
         _model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models)
         return models

tests/unit/llms/bedrock/test_bedrock_model_listing.py

@@ -0,0 +1,130 @@
+import re
+from collections.abc import Callable
+from typing import Final
+
+import httpx
+import pytest
+
+from litellm.llms.bedrock.common_utils import BedrockModelInfo
+from litellm.llms.custom_httpx.http_handler import HTTPHandler
+from litellm.types.router import LiteLLM_Params
+
+ACCESS_KEY: Final = "AKIAIOSFODNN7EXAMPLE"
+SECRET_KEY: Final = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+REGION: Final = "eu-central-1"
+BEARER_TOKEN: Final = "bedrock-api-key-for-tests"
+SECOND_PAGE_TOKEN: Final = "page2+token/with=reserved chars"
+
+ON_DEMAND_MODELS: Final = (
+    {"modelId": "anthropic.claude-3-haiku-20240307-v1:0", "inferenceTypesSupported": ["ON_DEMAND"]},
+    {"modelId": "amazon.nova-micro-v1:0", "inferenceTypesSupported": ["ON_DEMAND"]},
+)
+PROFILE_ONLY_MODEL: Final = {
+    "modelId": "anthropic.claude-opus-4-5-20251101-v1:0",
+    "inferenceTypesSupported": ["INFERENCE_PROFILE"],
+}
+PROFILE_PAGES: Final = {
+    None: {
+        "inferenceProfileSummaries": [
+            {"inferenceProfileId": "global.anthropic.claude-opus-4-5-20251101-v1:0", "status": "ACTIVE"},
+            {"inferenceProfileId": "eu.anthropic.claude-3-haiku-20240307-v1:0", "status": "INACTIVE"},
+        ],
+        "nextToken": SECOND_PAGE_TOKEN,
+    },
+    SECOND_PAGE_TOKEN: {
+        "inferenceProfileSummaries": [{"inferenceProfileId": "eu.amazon.nova-micro-v1:0", "status": "ACTIVE"}],
+    },
+}
+EXPECTED_MODELS: Final = [
+    "bedrock/amazon.nova-micro-v1:0",
+    "bedrock/anthropic.claude-3-haiku-20240307-v1:0",
+    "bedrock/eu.amazon.nova-micro-v1:0",
+    "bedrock/global.anthropic.claude-opus-4-5-20251101-v1:0",
+]
+SIGV4_CREDENTIAL: Final = re.compile(
+    rf"^AWS4-HMAC-SHA256 Credential={ACCESS_KEY}/\d{{8}}/{REGION}/bedrock/aws4_request, SignedHeaders=([^,]+), Signature="
+)
+
+
+@pytest.fixture(autouse=True)
+def no_ambient_bearer_token(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
+
+
+def _authorized(request: httpx.Request, bearer_token: str | None) -> bool:
+    authorization: Final = request.headers.get("authorization", "")
+    if bearer_token is not None:
+        return authorization == f"Bearer {bearer_token}"
+    signed: Final = SIGV4_CREDENTIAL.match(authorization)
+    if signed is None or "x-amz-date" not in request.headers:
+        return False
+    signed_headers: Final = signed.group(1).split(";")
+    return "host" in signed_headers and "x-amz-date" in signed_headers
+
+
+def _control_plane(bearer_token: str | None = None) -> Callable[[httpx.Request], httpx.Response]:
+    def handler(request: httpx.Request) -> httpx.Response:
+        if request.url.host != f"bedrock.{REGION}.amazonaws.com":
+            return httpx.Response(404, json={"message": f"no Bedrock at {request.url.host}"})
+        if not _authorized(request, bearer_token):
+            return httpx.Response(403, json={"message": "not signed for this account, region and service"})
+        if request.url.path == "/foundation-models":
+            on_demand_only: Final = request.url.params.get("byInferenceType") == "ON_DEMAND"
+            summaries: Final = ON_DEMAND_MODELS if on_demand_only else (*ON_DEMAND_MODELS, PROFILE_ONLY_MODEL)
+            return httpx.Response(200, json={"modelSummaries": list(summaries)})
+        if request.url.path == "/inference-profiles":
+            if request.url.params.get("typeEquals") != "SYSTEM_DEFINED":
+                return httpx.Response(400, json={"message": "application profiles are not listable here"})
+            return httpx.Response(200, json=PROFILE_PAGES[request.url.params.get("nextToken")])
+        return httpx.Response(404, json={"message": f"unknown path {request.url.path}"})
+
+    return handler
+
+
+def _bedrock(handler: Callable[[httpx.Request], httpx.Response]) -> BedrockModelInfo:
+    return BedrockModelInfo(client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handler))))
+
+
+def _deployment(region: str = REGION, api_key: str | None = None) -> LiteLLM_Params:
+    return LiteLLM_Params(
+        model="bedrock/*",
+        aws_access_key_id=ACCESS_KEY,
+        aws_secret_access_key=SECRET_KEY,
+        aws_region_name=region,
+        api_key=api_key,
+    )
+
+
+def test_lists_active_profiles_and_on_demand_models_signed_for_the_deployment() -> None:
+    assert _bedrock(_control_plane()).get_models_for_deployment(_deployment()) == EXPECTED_MODELS
+
+
+def test_lists_in_the_deployment_region_not_the_ambient_one(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("AWS_REGION_NAME", REGION)
+
+    with pytest.raises(Exception, match=r"us-west-2.*404"):
+        _bedrock(_control_plane()).get_models_for_deployment(_deployment(region="us-west-2"))
+
+
+def test_bearer_token_api_key_replaces_sigv4() -> None:
+    bedrock: Final = _bedrock(_control_plane(bearer_token=BEARER_TOKEN))
+
+    assert bedrock.get_models_for_deployment(_deployment(api_key=BEARER_TOKEN)) == EXPECTED_MODELS
+    with pytest.raises(Exception, match="403"):
+        bedrock.get_models_for_deployment(_deployment())
+
+
+def test_get_models_without_a_deployment_uses_ambient_credentials(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("AWS_ACCESS_KEY_ID", ACCESS_KEY)
+    monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", SECRET_KEY)
+    monkeypatch.setenv("AWS_REGION_NAME", REGION)
+
+    assert _bedrock(_control_plane()).get_models() == EXPECTED_MODELS
+
+
+def test_listing_failure_names_the_region_and_response() -> None:
+    def throttled(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(429, json={"message": "Too many requests"})
+
+    with pytest.raises(Exception, match=rf"{REGION}.*429.*Too many requests"):
+        _bedrock(throttled).get_models_for_deployment(_deployment())

tests/unit/test_utils.py

@@ -42,7 +42,7 @@
 from litellm.types.caching import CachingSupportedCallTypes
 from litellm.types.integrations.custom_logger import HEADROOM_CONVERTED_STREAM_KEY
 from litellm.types.llms.openai import ResponsesAPIResponse
-from litellm.types.router import CredentialLiteLLMParams, GenericLiteLLMParams
+from litellm.types.router import CredentialLiteLLMParams, GenericLiteLLMParams, LiteLLM_Params
 from litellm.types.utils import (
     ADDRESSED_RESPONSE_ID_FIELD,
     CallTypes,
@@ -2477,6 +2477,50 @@ def test_get_valid_models_with_cli_pattern(self):
             assert headers.get("Authorization") == "Bearer sk-test-cli-key-123"
 
 
+@respx.mock
+def test_get_valid_models_bedrock_lists_what_the_deployment_credentials_can_invoke(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
+    region: Final = "eu-west-3"
+    deployment: Final = LiteLLM_Params(
+        model="bedrock/*",
+        aws_access_key_id="AKIAIOSFODNN7EXAMPLE",
+        aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+        aws_region_name=region,
+    )
+    expected_scope: Final = f"/{region}/bedrock/aws4_request"
+
+    def signed_by_the_deployment(request: httpx.Request, body: dict[str, object]) -> httpx.Response:
+        authorization: Final = request.headers.get("authorization", "")
+        if not authorization.startswith("AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/"):
+            return httpx.Response(403, json={"message": "not signed with the deployment's key"})
+        if expected_scope not in authorization:
+            return httpx.Response(403, json={"message": "not signed for the deployment's region"})
+        return httpx.Response(200, json=body)
+
+    respx.get(
+        f"https://bedrock.{region}.amazonaws.com/foundation-models", params={"byInferenceType": "ON_DEMAND"}
+    ).mock(side_effect=lambda request: signed_by_the_deployment(request, {"modelSummaries": [{"modelId": "amazon.nova-micro-v1:0"}]}))
+    respx.get(
+        f"https://bedrock.{region}.amazonaws.com/inference-profiles",
+        params={"typeEquals": "SYSTEM_DEFINED"},
+    ).mock(
+        side_effect=lambda request: signed_by_the_deployment(
+            request,
+            {
+                "inferenceProfileSummaries": [
+                    {"inferenceProfileId": "eu.anthropic.claude-sonnet-4-5-20250929-v1:0", "status": "ACTIVE"}
+                ]
+            },
+        )
+    )
+
+    assert litellm.get_valid_models(
+        check_provider_endpoint=True, custom_llm_provider="bedrock", litellm_params=deployment
+    ) == ["bedrock/amazon.nova-micro-v1:0", "bedrock/eu.anthropic.claude-sonnet-4-5-20250929-v1:0"]
+
+
 class TestIsCachedMessage:
     """Test is_cached_message function for context caching detection.