From f45909cb81e698ea9172d3622ed90da42af3345d Mon Sep 17 00:00:00 2001 From: milan-berri Date: Mon, 25 May 2026 16:51:55 +0300 Subject: [PATCH 01/13] fix(proxy): Bedrock Knowledge Base pass-through: preserve SigV4 headers and signed request body (#27526) * Fix Bedrock KB pass-through SigV4 headers and signed body Coerce botocore HeadersDict to a dict for pass-through routes. When forward_headers is true, drop request headers that collide case-insensitively with signed headers so client Bearer auth does not shadow AWS SigV4. Send prepped.body as raw content so the outbound payload matches the signature after logging hooks mutate the parsed dict. Co-authored-by: Cursor * Simplify pass-through raw body handling Read the SigV4-signed bytes directly from request.state inside pass_through_request instead of threading a custom_raw_body argument through three functions. Helper methods are restored to their original signatures, and the new branch lives in one place at each httpx call site. Co-authored-by: Cursor * Harden pass-through raw body read from request.state Guard missing request.state (test fixtures) and ignore non-bytes/str values so MagicMock does not trigger the SigV4 raw-body path. Co-authored-by: Cursor * Test pass_through_request state_raw_body uses httpx content= Cover non-streaming (async_client.request) and streaming (build_request) paths so SigV4 bytes on request.state are not replaced by json= of a hook-mutated dict. Co-authored-by: Cursor --------- Co-authored-by: Cursor --- litellm/passthrough/utils.py | 5 + .../llm_passthrough_endpoints.py | 4 + .../pass_through_endpoints.py | 66 ++++++-- .../pass_through_endpoints.py | 4 + .../test_pass_through_endpoints.py | 151 +++++++++++++++++- .../test_vertex_passthrough_load_balancing.py | 30 ++++ 6 files changed, 246 insertions(+), 14 deletions(-) diff --git a/litellm/passthrough/utils.py b/litellm/passthrough/utils.py index d39a0dda1520..9484922833ad 100644 --- a/litellm/passthrough/utils.py +++ b/litellm/passthrough/utils.py @@ -71,6 +71,11 @@ def forward_headers_from_request( request_headers.pop("content-length", None) request_headers.pop("host", None) + custom_header_names = {header_name.lower() for header_name in headers} + for header_name in list(request_headers.keys()): + if header_name.lower() in custom_header_names: + request_headers.pop(header_name, None) + # Combine request headers with custom headers headers = {**request_headers, **headers} diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index ce103f806e19..7ca28a5d4ac4 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -44,6 +44,7 @@ ) from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY, + LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, ) from litellm.proxy.utils import is_known_model from litellm.proxy.vector_store_endpoints.utils import ( @@ -1123,6 +1124,9 @@ async def bedrock_proxy_route( _forward_headers=True, ) # dynamically construct pass-through endpoint based on incoming path setattr(request.state, LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY, data) + # SigV4 signs an exact payload; pass-through must send prepped.body, not json.dumps + # of a dict that hooks may mutate (logging_obj, metadata, etc.). + setattr(request.state, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, prepped.body) received_value = await endpoint_func( request, fastapi_response, diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index df52c0fe2049..51dbf5ce890b 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -6,7 +6,7 @@ import traceback from base64 import b64encode from datetime import datetime -from typing import Any, Dict, List, Optional, Tuple, Union, cast +from typing import Any, Dict, List, Mapping, Optional, Tuple, Union, cast from urllib.parse import urlencode, urlparse import httpx @@ -62,6 +62,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( EndpointType, LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY, + LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, PassthroughStandardLoggingPayload, ) @@ -735,6 +736,22 @@ async def pass_through_request( # noqa: PLR0915 str(url) ) + # SigV4-signed callers (e.g. Bedrock) attach the exact bytes that were + # signed via request.state; we must send those instead of re-encoding the + # parsed dict (hooks mutate it, breaking the signature / Content-Length). + # Tolerate request objects without `state` (test fixtures) and only honor + # values httpx accepts for `content=`. + _request_state = getattr(request, "state", None) + state_raw_body: Optional[Union[str, bytes]] = ( + getattr(_request_state, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, None) + if _request_state is not None + else None + ) + if state_raw_body is not None and not isinstance( + state_raw_body, (str, bytes, bytearray) + ): + state_raw_body = None + # Skip body parsing for multipart requests - make_multipart_http_request will handle it # But if custom_body is provided (e.g., JSON parsed despite multipart content-type), use it is_multipart = ( @@ -883,12 +900,19 @@ async def pass_through_request( # noqa: PLR0915 ) ) else: + # SigV4-signed callers (Bedrock) supply the exact pre-signed bytes; + # otherwise httpx encodes the parsed JSON dict as before. + body_kwargs: Dict[str, Any] = ( + {"content": state_raw_body} + if state_raw_body is not None + else {"json": _parsed_body} + ) req = async_client.build_request( "POST", url, - json=_parsed_body, params=requested_query_params, headers=headers, + **body_kwargs, ) response = await async_client.send(req, stream=stream) @@ -917,17 +941,28 @@ async def pass_through_request( # noqa: PLR0915 status_code=response.status_code, ) - response = ( - await HttpPassThroughEndpointHelpers.non_streaming_http_request_handler( - request=request, - async_client=async_client, + if state_raw_body is not None: + # SigV4-signed callers (Bedrock) require the exact pre-signed bytes + # to be forwarded so the signature/Content-Length stay valid. + response = await async_client.request( + method=request.method, url=url, headers=headers, - requested_query_params=requested_query_params, - _parsed_body=_parsed_body, - forward_multipart=is_multipart, + params=requested_query_params, + content=state_raw_body, + ) + else: + response = ( + await HttpPassThroughEndpointHelpers.non_streaming_http_request_handler( + request=request, + async_client=async_client, + url=url, + headers=headers, + requested_query_params=requested_query_params, + _parsed_body=_parsed_body, + forward_multipart=is_multipart, + ) ) - ) verbose_proxy_logger.debug("response.headers= %s", response.headers) if _is_streaming_response(response) is True: @@ -1225,7 +1260,7 @@ async def _parse_request_data_by_content_type( def create_pass_through_route( endpoint, target: str, - custom_headers: Optional[dict] = None, + custom_headers: Optional[Mapping[str, Any]] = None, _forward_headers: Optional[bool] = False, _merge_query_params: Optional[bool] = False, dependencies: Optional[List] = None, @@ -1335,9 +1370,12 @@ async def endpoint_func( # type: ignore ) ) - # Ensure custom_headers is a dict + # Ensure custom_headers is a dict. Botocore returns a HeadersDict + # for SigV4-prepared requests, which is a Mapping but not a dict. headers_dict = ( - param_custom_headers if isinstance(param_custom_headers, dict) else {} + dict(param_custom_headers) + if isinstance(param_custom_headers, Mapping) + else {} ) # Ensure query_params and custom_body are dicts or None @@ -1380,6 +1418,8 @@ async def endpoint_func( # type: ignore finally: if hasattr(request.state, LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY): delattr(request.state, LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY) + if hasattr(request.state, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY): + delattr(request.state, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY) return endpoint_func diff --git a/litellm/types/passthrough_endpoints/pass_through_endpoints.py b/litellm/types/passthrough_endpoints/pass_through_endpoints.py index 4a07fa5e8492..3524a7eb7f72 100644 --- a/litellm/types/passthrough_endpoints/pass_through_endpoints.py +++ b/litellm/types/passthrough_endpoints/pass_through_endpoints.py @@ -7,6 +7,10 @@ # JSON without a FastAPI `custom_body` parameter (which would consume the HTTP body). LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY = "litellm_pass_through_custom_body" +# Request.state key for programmatic pass-through callers that must preserve an +# exact byte/string body, such as AWS SigV4-signed requests. +LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY = "litellm_pass_through_raw_body" + class EndpointType(str, Enum): VERTEX_AI = "vertex-ai" diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 97a211361989..344742ffe894 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -20,6 +20,9 @@ LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY, pass_through_request, ) +from litellm.types.passthrough_endpoints.pass_through_endpoints import ( + LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, +) from litellm.proxy.pass_through_endpoints.success_handler import ( PassThroughEndpointLogging, ) @@ -2153,7 +2156,12 @@ async def test_create_pass_through_route_custom_body_url_target(): endpoint_func = create_pass_through_route( endpoint=unique_path, target="https://bedrock-agent-runtime.us-east-1.amazonaws.com", - custom_headers={"Content-Type": "application/json"}, + custom_headers=Headers( + { + "Authorization": "AWS4-HMAC-SHA256 signed", + "Content-Type": "application/json", + } + ), _forward_headers=True, ) @@ -2213,6 +2221,147 @@ async def test_create_pass_through_route_custom_body_url_target(): # The critical assertion: custom_body takes precedence over # the body parsed from the raw request assert call_kwargs["custom_body"] == bedrock_body + # HeadersDict-like custom_headers (e.g. botocore SigV4) must be coerced + # to a plain dict so signed headers actually reach the upstream. + assert call_kwargs["custom_headers"] == { + "authorization": "AWS4-HMAC-SHA256 signed", + "content-type": "application/json", + } + + +@pytest.mark.asyncio +async def test_pass_through_request_non_streaming_uses_content_for_state_raw_body(): + """ + Bedrock SigV4 path: exact signed bytes live on request.state; upstream must receive + content=... even if pre_call_hook mutates the parsed dict (would change json=). + """ + # Bytes that were signed (simulated); parsed body + hook will diverge on purpose. + raw_signed = b'{"retrievalQuery":{"text":"signed"},"sig":"intact"}' + parsed_from_wire = {"retrievalQuery": {"text": "signed"}, "sig": "intact"} + + mock_request = MagicMock(spec=Request) + mock_request.method = "POST" + mock_request.query_params = QueryParams({}) + mock_request.headers = Headers({"Content-Type": "application/json"}) + mock_request.state = SimpleNamespace() + setattr(mock_request.state, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, raw_signed) + mock_request.body = AsyncMock( + return_value=json.dumps(parsed_from_wire).encode("utf-8") + ) + + mock_user = MagicMock() + mock_user.api_key = "sk-test" + + upstream = httpx.Response( + status_code=200, + headers={"content-type": "application/json"}, + content=b'{"ok": true}', + request=httpx.Request( + "POST", + "https://bedrock-agent-runtime.us-east-1.amazonaws.com/knowledgebases/KB/retrieve", + ), + ) + + mock_async_client = AsyncMock() + mock_async_client.request = AsyncMock(return_value=upstream) + mock_client_obj = MagicMock() + mock_client_obj.client = mock_async_client + + async def _hook_mutates_body(**kwargs): + data = kwargs["data"] + if isinstance(data, dict): + data["hook_mutated"] = True + return data + + with ( + patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_async_httpx_client", + return_value=mock_client_obj, + ), + patch( + "litellm.proxy.proxy_server.proxy_logging_obj.pre_call_hook", + new=AsyncMock(side_effect=_hook_mutates_body), + ), + patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.pass_through_endpoint_logging.pass_through_async_success_handler", + new=AsyncMock(), + ), + ): + await pass_through_request( + request=mock_request, + target="https://bedrock-agent-runtime.us-east-1.amazonaws.com/knowledgebases/KB/retrieve", + custom_headers={"content-type": "application/json"}, + user_api_key_dict=mock_user, + stream=False, + ) + + mock_async_client.request.assert_called_once() + req_kw = mock_async_client.request.call_args[1] + assert req_kw.get("content") == raw_signed + assert "json" not in req_kw + + +@pytest.mark.asyncio +async def test_pass_through_request_streaming_uses_content_for_state_raw_body(): + """Streaming pass-through with state raw body must use build_request(..., content=...).""" + raw_signed = b'{"model":"m","stream":true}' + parsed_from_wire = {"model": "m", "stream": True} + + mock_request = MagicMock(spec=Request) + mock_request.method = "POST" + mock_request.query_params = QueryParams({}) + mock_request.headers = Headers({"Content-Type": "application/json"}) + mock_request.state = SimpleNamespace() + setattr(mock_request.state, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, raw_signed) + mock_request.body = AsyncMock( + return_value=json.dumps(parsed_from_wire).encode("utf-8") + ) + + mock_user = MagicMock() + mock_user.api_key = "sk-test" + + mock_built = MagicMock() + mock_async_client = AsyncMock() + mock_async_client.build_request = MagicMock(return_value=mock_built) + stream_resp = httpx.Response( + status_code=200, + headers={"content-type": "text/event-stream"}, + content=b"data: {}\n\n", + request=httpx.Request("POST", "https://example.com/v1/messages"), + ) + mock_async_client.send = AsyncMock(return_value=stream_resp) + mock_client_obj = MagicMock() + mock_client_obj.client = mock_async_client + + with ( + patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_async_httpx_client", + return_value=mock_client_obj, + ), + patch( + "litellm.proxy.proxy_server.proxy_logging_obj.pre_call_hook", + new=AsyncMock(side_effect=lambda **kw: kw["data"]), + ), + patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.pass_through_endpoint_logging.pass_through_async_success_handler", + new=AsyncMock(), + ), + ): + response = await pass_through_request( + request=mock_request, + target="https://example.com/v1/messages", + custom_headers={"Authorization": "Bearer x"}, + user_api_key_dict=mock_user, + stream=None, + ) + + from fastapi.responses import StreamingResponse + + assert isinstance(response, StreamingResponse) + mock_async_client.build_request.assert_called_once() + br_kw = mock_async_client.build_request.call_args[1] + assert br_kw.get("content") == raw_signed + assert "json" not in br_kw @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py b/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py index 7176cf455c8b..aaf1dad49104 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py @@ -538,6 +538,36 @@ def test_forward_headers_from_request_protected_headers_not_overwritten(): assert "Anthropic-Beta" not in result +def test_forward_headers_custom_wins_case_insensitive_over_request_authorization(): + """ + When forwarding request headers, provider-signed/custom headers must win + even if the incoming request uses a different case for the same header name. + """ + from litellm.passthrough.utils import BasePassthroughUtils + + request_headers = { + "authorization": "Bearer sk-litellm-key", + "content-type": "application/json", + "x-request-id": "req-123", + } + signed_headers = { + "Authorization": "AWS4-HMAC-SHA256 signed", + "Content-Type": "application/json", + } + + result = BasePassthroughUtils.forward_headers_from_request( + request_headers=request_headers, + headers=signed_headers.copy(), + forward_headers=True, + ) + + assert result["Authorization"] == "AWS4-HMAC-SHA256 signed" + assert "authorization" not in result + assert result["Content-Type"] == "application/json" + assert "content-type" not in result + assert result["x-request-id"] == "req-123" + + @pytest.mark.asyncio async def test_vertex_passthrough_custom_model_name_replaced_in_url(): """ From f9407bc0366ef9075e1cd298f2d344d3e11267f2 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 25 May 2026 12:03:17 -0700 Subject: [PATCH 02/13] chore(tests): migrate Bedrock CI to AWS account 941277531214 (#28728) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(tests): migrate Bedrock CI from AWS account 888602223428 to 941277531214 The original account (888602223428) was put under a security restriction by AWS after a root access key leaked in a PR comment. While that account works its way through the AWS Support unlock process, Bedrock-touching CI tests have been migrated to a fresh account (941277531214). Changes: - Replace 26 hardcoded references to 888602223428 with 941277531214 across 8 files (provisioned-model ARNs, imported-model ARNs, AgentCore runtime ARNs, batch execution role ARN, and example proxy config). - The provisioned-model and imported-model ARNs are referenced only from mocked unit tests — no AWS resources to recreate. - The batch execution IAM role has been recreated in the new account with the same name and equivalent permissions. - The two AgentCore runtimes (hosted_agent_r9jvp-3ySZuRHjLC, hosted_agent_13sf6-cALnp38iZD) are being recreated in the new account under the same names — see tools/agentcore-deploy/ in a follow-up. CircleCI env vars AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY / AWS_REGION_NAME were updated separately via the CircleCI API to point at the new account. Smoke-tested locally against the new account: aws bedrock-runtime converse --region us-west-2 \ --model-id us.anthropic.claude-sonnet-4-5-20250929-v1:0 \ --messages '[{"role":"user","content":[{"text":"ping"}]}]' → 200, model returned 'pong' Co-Authored-By: Claude Opus 4.7 * chore(tests): refresh AgentCore ARN suffixes to match newly-deployed runtimes The first migration commit replaced just the account ID, but AgentCore auto-assigns a random 10-char suffix to every runtime on creation — we can't reuse the original suffixes (`3ySZuRHjLC`, `cALnp38iZD`) in the new account. Updated the AgentCore-runtime ARNs in the three files that reference real runtime IDs (not the mock-based unit-test ARNs). Deployed runtimes: arn:aws:bedrock-agentcore:us-west-2:941277531214:runtime/hosted_agent_r9jvp-Rq79QFC2fp arn:aws:bedrock-agentcore:us-west-2:941277531214:runtime/hosted_agent_13sf6-4046UzHSwy Both runtimes are status=READY and pass a smoke invoke: $ aws bedrock-agentcore invoke-agent-runtime --agent-runtime-arn ... --payload '{"prompt":"ping"}' → 200, {"result": "echo: ping"} The agent is a minimal echo (see /tmp/agentcore_deploy/agent.py for the deploy artifacts). Tests that only verify the SDK wiring will pass; if any test asserts on agent output content, swap the echo for the real agent. Co-Authored-By: Claude Opus 4.7 * chore(tests): point Bedrock batch tests at new-account S3 bucket The account migration (888602223428 -> 941277531214) was a flat account-ID swap, which only rewrites ARNs that embed the account number. S3 bucket names carry no account ID, so the live Bedrock batch tests still uploaded to `litellm-proxy` — a bucket that lives in the old account. S3 names are globally unique, and the old account still holds that name, so it can't be recreated in the new account. Rename to `litellm-proxy-941277531214` (account-ID suffix guarantees global uniqueness). The bucket must be created in 941277531214 and the batch execution role granted s3:GetObject/PutObject/ListBucket on it before this job is run in CI. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(tests): point live S3 logging test at new-account bucket Same account-ID-free blind spot as the batch bucket: `load-testing-oct` lives in the old account and its name can't be reused globally. The `logging_testing` CI job is wired into the workflow and runs test_basic_s3_logging, which uploads to this bucket with the CI env creds, then lists and deletes objects — a live dependency. Rename to `load-testing-oct-941277531214`. The bucket must exist in the new account with the CI IAM principal granted s3:PutObject/GetObject/ListBucket/DeleteObject before this job runs. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(tests): repoint Bedrock guardrail IDs to new-account guardrails The migration left guardrail IDs untouched (no account ID in them), so all live guardrail tests failed with "guardrail identifier or version does not exist" against 941277531214. Recreated both guardrails in the new account and updated the hardcoded IDs: - wf0hkdb5x07f -> zgkmukebruil (PII mask: PHONE + CREDIT_DEBIT_CARD, with explicit inputAction=ANONYMIZE so masking applies to INPUT, which is the source litellm's moderation hook sends) - ff6ujrregl1q -> 4w3d1di3snt5 (blocks "coffee"; blocked message set to the exact string the tests assert on) Updated test_bedrock_guardrails.py, otel_test_config.yaml, and the guardrailConfig in test_bedrock_completion.py. Verified locally: the 5 previously-failing guardrail tests now pass. Co-Authored-By: Claude Opus 4.7 (1M context) * test(bedrock): migrate legacy models to current inference profiles The new CI account (941277531214) cannot invoke legacy Bedrock models (AWS gates them: "marked by provider as Legacy... not actively using in the last 30 days"). Migrated the live-call tests: - anthropic.claude-3-sonnet-20240229 -> us.anthropic.claude-sonnet-4-5-20250929-v1:0 - anthropic.claude-3-haiku-20240307 -> us.anthropic.claude-haiku-4-5-20251001-v1:0 Current Claude models on Bedrock require the us. inference-profile prefix (bare on-demand ids are rejected). cohere.command-r-plus has no working replacement (all Cohere is legacy- gated in the new account): swapped to claude-haiku-4-5 in provider- agnostic param lists. amazon.titan-image-generator skipped (no working replacement). Mocked/transformation/cost tests that reference the legacy strings are intentionally left unchanged. Verified live against the new account. Co-Authored-By: Claude Opus 4.7 (1M context) * test(bedrock): repoint SageMaker + Knowledge Base to new-account resources These referenced account-scoped resources by hardcoded id that only existed in the old account, so the migration's account-ID swap missed them. Recreated in 941277531214 and repointed: - SageMaker endpoint jumpstart-dft-hf-textgeneration1-mp-20240815-185614 -> litellm-ci-textgen (gpt2 on a TGI container, ml.g5.xlarge) - Bedrock Knowledge Base T37J8R4WTM -> LCYXFBR2TU (OpenSearch Serverless vector store + titan-embed-text-v2, seeded with a LiteLLM doc) Verified live: test_sagemaker.py (12 passed) and test_bedrock_knowledgebase_hook.py (12 passed). Co-Authored-By: Claude Opus 4.7 (1M context) * test(reasoning_effort_grid): skip bedrock claude-opus-4-7 cells (not entitled on 941277531214) claude-opus-4-7 is listed in the new Bedrock CI account's foundation models but invoke is denied (AccessDeniedException: "not available for this account"). Bedrock access to the flagship Opus requires an AWS Sales request, not the self-serve model-access toggle, so it can't be enabled inline with the rest of the account migration. Add an optional `skip_reason` to ModelEntry and set it on the bedrock-claude-opus-4-7 entry; the grid test honors it via pytest.skip. Cell count (231) and route coverage are unchanged, so the structural asserts still pass. Restore coverage by deleting the one skip_reason line once access is granted. Co-Authored-By: Claude Opus 4.7 (1M context) * test(bedrock): swap/skip legacy-gated models unavailable on new CI account The migrated AWS account (941277531214) cannot access several models that the old account could, so the remaining red CI jobs were hitting real Bedrock "Access denied / Legacy" and "account not authorized" errors: - image_gen: skip both Nova Canvas test classes (amazon.nova-canvas-v1:0 is legacy-gated), matching the existing titan skip. - batches: skip test_async_file_and_batch (Bedrock batch inference is not authorized on the new account; requires an AWS support case). - litellm_overhead: swap legacy claude-3-5-haiku for the active us.anthropic.claude-haiku-4-5 inference profile. - test_completion_claude_3_function_call: swap legacy claude-3-sonnet for the active us.anthropic.claude-sonnet-4-5 inference profile. https://claude.ai/code/session_01Y7zgHYu9GX29YRwV4yiWAa * test(bedrock): fix remaining e2e legacy-model + batch failures on new CI account - e2e_openai_endpoints: skip test_bedrock_batches_api (Bedrock batch inference is not authorized on account 941277531214) and migrate the missed s3_bucket_name in oai_misc_config.yaml to litellm-proxy-941277531214. - build_and_test: swap legacy bedrock claude-3-sonnet for the active us.anthropic.claude-sonnet-4-5 inference profile in the proxy structured output e2e test. https://claude.ai/code/session_01Y7zgHYu9GX29YRwV4yiWAa * test(bedrock): make opus-4-7 + batch cells fail loudly and mock image-gen (#28791) Replace the silent skips added for the new CI account with noisier behavior: - reasoning-effort grid: opus-4-7 cells now fail (when AWS creds are present) instead of skipping, so the missing entitlement stays visible in CI; they still skip when AWS creds are absent (local dev) - Bedrock batch inference tests: drop the skip so they run and fail until batch access is granted - Titan + Nova Canvas image-gen tests: mock the Bedrock HTTP call so the transform + cost-tracking path stays under test without live model access https://claude.ai/code/session_01MT7SWDnXUjv6e6EPG7BDjT Co-authored-by: Claude * test(bedrock): use pytest.xfail for known-failing opus-4-7 cells Replace pytest.fail with pytest.xfail when a model has a fail_reason, so known-broken cells stay visible as XFAIL without keeping CI red. Co-authored-by: Yassin Kortam --------- Co-authored-by: Mateo Co-authored-by: Claude Opus 4.7 Co-authored-by: Cursor Agent Co-authored-by: Yassin Kortam --- .../bedrock/chat/agentcore/transformation.py | 6 +- .../example_config_yaml/oai_misc_config.yaml | 4 +- .../example_config_yaml/otel_test_config.yaml | 2 +- .../test_a2a_completion_bridge.py | 2 +- .../test_bedrock_files_and_batches.py | 8 +-- .../test_bedrock_guardrails.py | 16 ++--- .../test_bedrock_image_gen_unit_tests.py | 35 ++++++++--- .../image_gen_tests/test_image_generation.py | 58 ++++++++++++++++++- .../test_litellm_overhead.py | 4 +- .../reasoning_effort_grid/grid_spec.py | 8 ++- .../test_reasoning_effort_grid.py | 4 +- .../llm_translation/test_bedrock_agentcore.py | 24 ++++---- .../test_bedrock_completion.py | 20 +++---- tests/local_testing/test_completion.py | 21 +++---- .../test_function_call_parsing.py | 3 +- tests/local_testing/test_function_calling.py | 9 ++- tests/local_testing/test_sagemaker.py | 28 ++++----- tests/local_testing/test_streaming.py | 8 +-- .../test_amazing_s3_logs.py | 8 +-- .../test_bedrock_knowledgebase_hook.py | 27 +++++---- .../test_agentcore_transformation.py | 10 ++-- tests/test_openai_endpoints.py | 2 +- .../test_bedrock_vector_store.py | 14 ++--- 23 files changed, 203 insertions(+), 118 deletions(-) diff --git a/litellm/llms/bedrock/chat/agentcore/transformation.py b/litellm/llms/bedrock/chat/agentcore/transformation.py index 44ba1ce3c868..9b9b96aae047 100644 --- a/litellm/llms/bedrock/chat/agentcore/transformation.py +++ b/litellm/llms/bedrock/chat/agentcore/transformation.py @@ -157,8 +157,8 @@ def sign_request( def _get_agent_runtime_arn(self, model: str) -> str: """ Extract ARN from model string - model = "agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC" - returns: "arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC" + model = "agentcore/arn:aws:bedrock-agentcore:us-west-2:941277531214:runtime/hosted_agent_r9jvp-Rq79QFC2fp" + returns: "arn:aws:bedrock-agentcore:us-west-2:941277531214:runtime/hosted_agent_r9jvp-Rq79QFC2fp" """ parts = model.split("/", 1) if len(parts) != 2 or parts[0] != "agentcore": @@ -170,7 +170,7 @@ def _get_agent_runtime_arn(self, model: str) -> str: def _extract_region_from_arn(self, arn: str) -> str: """ Extract region from ARN - arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC + arn:aws:bedrock-agentcore:us-west-2:941277531214:runtime/hosted_agent_r9jvp-Rq79QFC2fp returns: us-west-2 """ parts = arn.split(":") diff --git a/litellm/proxy/example_config_yaml/oai_misc_config.yaml b/litellm/proxy/example_config_yaml/oai_misc_config.yaml index 16cc69c19a55..0b647de8a086 100644 --- a/litellm/proxy/example_config_yaml/oai_misc_config.yaml +++ b/litellm/proxy/example_config_yaml/oai_misc_config.yaml @@ -23,11 +23,11 @@ model_list: model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 ######################################################### ########## batch specific params ######################## - s3_bucket_name: litellm-proxy + s3_bucket_name: litellm-proxy-941277531214 s3_region_name: us-west-2 s3_access_key_id: os.environ/AWS_ACCESS_KEY_ID s3_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY - aws_batch_role_arn: arn:aws:iam::888602223428:role/service-role/AmazonBedrockExecutionRoleForAgents_BB9HNW6V4CV + aws_batch_role_arn: arn:aws:iam::941277531214:role/service-role/AmazonBedrockExecutionRoleForAgents_BB9HNW6V4CV model_info: mode: batch diff --git a/litellm/proxy/example_config_yaml/otel_test_config.yaml b/litellm/proxy/example_config_yaml/otel_test_config.yaml index c05e2b1b5dfe..9c7937efba96 100644 --- a/litellm/proxy/example_config_yaml/otel_test_config.yaml +++ b/litellm/proxy/example_config_yaml/otel_test_config.yaml @@ -55,7 +55,7 @@ guardrails: litellm_params: guardrail: bedrock # supported values: "bedrock", "lakera" mode: "during_call" - guardrailIdentifier: ff6ujrregl1q + guardrailIdentifier: 4w3d1di3snt5 guardrailVersion: "DRAFT" - guardrail_name: "custom-pre-guard" litellm_params: diff --git a/tests/agent_tests/local_only_agent_tests/test_a2a_completion_bridge.py b/tests/agent_tests/local_only_agent_tests/test_a2a_completion_bridge.py index a9268da4c315..95d76ba5804e 100644 --- a/tests/agent_tests/local_only_agent_tests/test_a2a_completion_bridge.py +++ b/tests/agent_tests/local_only_agent_tests/test_a2a_completion_bridge.py @@ -168,7 +168,7 @@ async def test_a2a_completion_bridge_bedrock_agentcore(): litellm._turn_on_debug() # Bedrock AgentCore ARN (streaming-capable runtime) - agentcore_arn = "arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC" + agentcore_arn = "arn:aws:bedrock-agentcore:us-west-2:941277531214:runtime/hosted_agent_r9jvp-Rq79QFC2fp" send_message_payload = { "message": { diff --git a/tests/batches_tests/test_bedrock_files_and_batches.py b/tests/batches_tests/test_bedrock_files_and_batches.py index 5148ea4db91d..97c0802ec99a 100644 --- a/tests/batches_tests/test_bedrock_files_and_batches.py +++ b/tests/batches_tests/test_bedrock_files_and_batches.py @@ -38,7 +38,7 @@ async def test_async_create_file(): file=open(file_path, "rb"), purpose="batch", custom_llm_provider="bedrock", - s3_bucket_name="litellm-proxy", + s3_bucket_name="litellm-proxy-941277531214", ) @@ -55,7 +55,7 @@ async def test_async_file_and_batch(): file=open(file_path, "rb"), purpose="batch", custom_llm_provider="bedrock", - s3_bucket_name="litellm-proxy", + s3_bucket_name="litellm-proxy-941277531214", ) print("CREATED FILE RESPONSE=", file_obj) @@ -70,7 +70,7 @@ async def test_async_file_and_batch(): # bedrock specific params ######################################################### model="us.anthropic.claude-haiku-4-5-20251001-v1:0", - aws_batch_role_arn="arn:aws:iam::888602223428:role/service-role/AmazonBedrockExecutionRoleForAgents_BB9HNW6V4CV", + aws_batch_role_arn="arn:aws:iam::941277531214:role/service-role/AmazonBedrockExecutionRoleForAgents_BB9HNW6V4CV", ) print("CREATED BATCH RESPONSE=", create_batch_response) @@ -129,7 +129,7 @@ async def mock_async_create_file(transformed_request, **kwargs): ), purpose="batch", custom_llm_provider="bedrock", - s3_bucket_name="litellm-proxy", + s3_bucket_name="litellm-proxy-941277531214", ) print(f"PUT URL: {captured_put_url}") diff --git a/tests/guardrails_tests/test_bedrock_guardrails.py b/tests/guardrails_tests/test_bedrock_guardrails.py index 6e78a8c4284c..ea50fe08ae0a 100644 --- a/tests/guardrails_tests/test_bedrock_guardrails.py +++ b/tests/guardrails_tests/test_bedrock_guardrails.py @@ -20,7 +20,7 @@ async def test_bedrock_guardrails_pii_masking(): mock_user_api_key_dict = UserAPIKeyAuth() guardrail = BedrockGuardrail( - guardrailIdentifier="wf0hkdb5x07f", + guardrailIdentifier="zgkmukebruil", guardrailVersion="DRAFT", ) @@ -60,7 +60,7 @@ async def test_bedrock_guardrails_pii_masking_content_list(): mock_user_api_key_dict = UserAPIKeyAuth() guardrail = BedrockGuardrail( - guardrailIdentifier="wf0hkdb5x07f", + guardrailIdentifier="zgkmukebruil", guardrailVersion="DRAFT", ) @@ -115,7 +115,7 @@ async def test_bedrock_guardrails_block_messages_api(): mock_user_api_key_dict = UserAPIKeyAuth() guardrail = BedrockGuardrail( - guardrailIdentifier="ff6ujrregl1q", + guardrailIdentifier="4w3d1di3snt5", guardrailVersion="DRAFT", ) @@ -166,7 +166,7 @@ async def test_bedrock_guardrails_block_responses_api(): mock_user_api_key_dict = UserAPIKeyAuth() guardrail = BedrockGuardrail( - guardrailIdentifier="ff6ujrregl1q", + guardrailIdentifier="4w3d1di3snt5", guardrailVersion="DRAFT", ) @@ -211,7 +211,7 @@ async def test_bedrock_guardrails_with_streaming(): ) guardrail = BedrockGuardrail( - guardrailIdentifier="ff6ujrregl1q", + guardrailIdentifier="4w3d1di3snt5", guardrailVersion="DRAFT", supported_event_hooks=[GuardrailEventHooks.post_call], guardrail_name="bedrock-post-guard", @@ -255,7 +255,7 @@ async def test_bedrock_guardrails_with_streaming_no_violation(): ) guardrail = BedrockGuardrail( - guardrailIdentifier="ff6ujrregl1q", + guardrailIdentifier="4w3d1di3snt5", guardrailVersion="DRAFT", supported_event_hooks=[GuardrailEventHooks.post_call], guardrail_name="bedrock-post-guard", @@ -299,7 +299,7 @@ async def test_bedrock_guardrails_streaming_request_body_mock(): # Create the guardrail guardrail = BedrockGuardrail( - guardrailIdentifier="wf0hkdb5x07f", + guardrailIdentifier="zgkmukebruil", guardrailVersion="DRAFT", supported_event_hooks=[GuardrailEventHooks.post_call], guardrail_name="bedrock-post-guard", @@ -382,7 +382,7 @@ async def test_bedrock_guardrail_aws_param_persistence(): from litellm.types.guardrails import GuardrailEventHooks guardrail = BedrockGuardrail( - guardrailIdentifier="wf0hkdb5x07f", + guardrailIdentifier="zgkmukebruil", guardrailVersion="DRAFT", aws_access_key_id="test-access-key", aws_secret_access_key="test-secret-key", diff --git a/tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py b/tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py index 36ae9e1df675..181691b730d4 100644 --- a/tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py +++ b/tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py @@ -1,3 +1,4 @@ +import json import logging import os import sys @@ -44,6 +45,9 @@ ) from litellm.llms.bedrock.common_utils import BedrockError +# Base64 placeholder used for mocked Bedrock image responses (a 1x1 PNG). +_MOCK_BEDROCK_IMAGE_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + @pytest.mark.parametrize( "model,expected", @@ -528,17 +532,34 @@ def test_backward_compatibility_regular_nova_model(): def test_amazon_titan_image_gen(): - """Test Amazon Titan image generation with cost tracking.""" - from litellm import image_generation + """Test Amazon Titan image generation with cost tracking. + + The Bedrock CI account is not entitled to amazon.titan-image-generator, so + the network call is mocked and only the transform + cost-tracking path is + exercised. + """ + from litellm.llms.custom_httpx.http_handler import HTTPHandler # Use v2 as v1 has reached end of life model_id = "bedrock/amazon.titan-image-generator-v2:0" - response = litellm.image_generation( - model=model_id, - prompt="A serene mountain landscape at sunset with a lake reflection", - aws_region_name="us-east-1", - ) + mock_payload = {"images": [_MOCK_BEDROCK_IMAGE_B64]} + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = mock_payload + mock_response.text = json.dumps(mock_payload) + mock_response.headers = {} + + client = HTTPHandler() + with patch.object(client, "post", return_value=mock_response): + response = litellm.image_generation( + model=model_id, + prompt="A serene mountain landscape at sunset with a lake reflection", + aws_region_name="us-east-1", + aws_access_key_id="fake-access-key-id", + aws_secret_access_key="fake-secret-access-key", + client=client, + ) print(f"response cost: {response._hidden_params['response_cost']}") diff --git a/tests/image_gen_tests/test_image_generation.py b/tests/image_gen_tests/test_image_generation.py index 873777189c9a..23a94ef389a7 100644 --- a/tests/image_gen_tests/test_image_generation.py +++ b/tests/image_gen_tests/test_image_generation.py @@ -7,7 +7,6 @@ import traceback from unittest.mock import AsyncMock, MagicMock, patch - sys.path.insert( 0, os.path.abspath("../..") ) # Adds the parent directory to the system path @@ -136,6 +135,51 @@ def get_base_image_generation_call_args(self) -> dict: } +# Base64 placeholder used for mocked Bedrock image responses (a 1x1 PNG). +_MOCK_BEDROCK_IMAGE_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + + +async def _assert_mocked_bedrock_image_generation(call_args: dict) -> None: + """Run ``aimage_generation`` with the Bedrock HTTP call mocked. + + The CI account is not entitled to Nova Canvas, so the network call is + replaced with a canned Bedrock response. This keeps the request transform, + response transform, and cost-tracking path under test without live access. + """ + mock_payload = {"images": [_MOCK_BEDROCK_IMAGE_B64]} + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = mock_payload + mock_response.text = json.dumps(mock_payload) + mock_response.headers = {} + + custom_logger = TestCustomLogger() + litellm.logging_callback_manager._reset_all_callbacks() + litellm.callbacks = [custom_logger] + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + return_value=mock_response, + ): + response = await litellm.aimage_generation( + **call_args, + prompt="A image of a otter", + aws_access_key_id="fake-access-key-id", + aws_secret_access_key="fake-secret-access-key", + ) + + await asyncio.sleep(1) + + assert custom_logger.standard_logging_payload is not None + assert custom_logger.standard_logging_payload["response_cost"] is not None + assert custom_logger.standard_logging_payload["response_cost"] > 0 + assert response.data is not None + for d in response.data: + assert isinstance(d, Image) + assert d.b64_json is not None or d.url is not None + + class TestBedrockNovaCanvasTextToImage(BaseImageGenTest): def get_base_image_generation_call_args(self) -> dict: litellm.in_memory_llm_clients_cache = InMemoryCache() @@ -148,6 +192,12 @@ def get_base_image_generation_call_args(self) -> dict: "aws_region_name": "us-east-1", } + @pytest.mark.asyncio(scope="module") + async def test_basic_image_generation(self): + await _assert_mocked_bedrock_image_generation( + self.get_base_image_generation_call_args() + ) + class TestBedrockNovaCanvasColorGuidedGeneration(BaseImageGenTest): def get_base_image_generation_call_args(self) -> dict: @@ -162,6 +212,12 @@ def get_base_image_generation_call_args(self) -> dict: "aws_region_name": "us-east-1", } + @pytest.mark.asyncio(scope="module") + async def test_basic_image_generation(self): + await _assert_mocked_bedrock_image_generation( + self.get_base_image_generation_call_args() + ) + class TestOpenAIGPTImage1(BaseImageGenTest): def get_base_image_generation_call_args(self) -> dict: diff --git a/tests/litellm_utils_tests/test_litellm_overhead.py b/tests/litellm_utils_tests/test_litellm_overhead.py index 3a428e9d588b..60ee849f8eba 100644 --- a/tests/litellm_utils_tests/test_litellm_overhead.py +++ b/tests/litellm_utils_tests/test_litellm_overhead.py @@ -82,7 +82,7 @@ async def fake_send(self, request, **kwargs): "bedrock/mistral.mistral-7b-instruct-v0:2", "openai/gpt-4o", "openai/self_hosted", - "bedrock/anthropic.claude-3-5-haiku-20241022-v1:0", + "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "vertex_ai/gemini-1.5-flash", ], ) @@ -147,7 +147,7 @@ async def _run(): [ "bedrock/mistral.mistral-7b-instruct-v0:2", "openai/gpt-4o", - "bedrock/anthropic.claude-3-5-haiku-20241022-v1:0", + "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "openai/self_hosted", ], ) diff --git a/tests/llm_translation/reasoning_effort_grid/grid_spec.py b/tests/llm_translation/reasoning_effort_grid/grid_spec.py index ed5346dad71b..993643e0fc1f 100644 --- a/tests/llm_translation/reasoning_effort_grid/grid_spec.py +++ b/tests/llm_translation/reasoning_effort_grid/grid_spec.py @@ -1,7 +1,6 @@ from dataclasses import dataclass, field from typing import Dict, FrozenSet, List, Optional, Tuple - OMIT = object() @@ -22,6 +21,7 @@ class ModelEntry: extra_params: Tuple[Tuple[str, str], ...] = field(default_factory=tuple) required_env: FrozenSet[str] = field(default_factory=frozenset) caps: FrozenSet[str] = field(default_factory=frozenset) + fail_reason: Optional[str] = None def params(self) -> Dict[str, str]: return dict(self.extra_params) @@ -205,6 +205,12 @@ def expected(model: ModelEntry, effort: str) -> CellExpectation: extra_params=(("aws_region_name", "us-east-1"),), required_env=_BEDROCK_REQ, caps=_CAPS_OPUS_4_7, + fail_reason=( + "claude-opus-4-7 is not entitled on the Bedrock CI account " + "941277531214 (model access requires an AWS Sales request, not " + "self-serve); this cell fails on purpose so it stays loud in CI — " + "remove this fail_reason once access is granted" + ), ), ModelEntry( alias="bedrock-claude-opus-4-6", diff --git a/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py b/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py index 28e2e402d678..e0b6290ad776 100644 --- a/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py +++ b/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py @@ -15,7 +15,6 @@ all_cells, ) - _PROMPT_MESSAGES: List[Dict[str, str]] = [ {"role": "user", "content": "Step by step, calculate 47 * 53. Show your work."} ] @@ -168,6 +167,9 @@ async def test_reasoning_effort_grid( if skip_reason: pytest.skip(skip_reason) + if model.fail_reason: + pytest.xfail(model.fail_reason) + if route_name == "bedrock_invoke_messages": status, exc = await _call_messages(model, effort) else: diff --git a/tests/llm_translation/test_bedrock_agentcore.py b/tests/llm_translation/test_bedrock_agentcore.py index 40774cf3d60e..95a814e97e4c 100644 --- a/tests/llm_translation/test_bedrock_agentcore.py +++ b/tests/llm_translation/test_bedrock_agentcore.py @@ -19,8 +19,8 @@ @pytest.mark.parametrize( "model", [ - "bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_13sf6-cALnp38iZD", # non-streaming invocation - "bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC", # streaming invocation + "bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:941277531214:runtime/hosted_agent_13sf6-4046UzHSwy", # non-streaming invocation + "bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:941277531214:runtime/hosted_agent_r9jvp-Rq79QFC2fp", # streaming invocation ], ) def test_bedrock_agentcore_basic(model): @@ -44,7 +44,7 @@ def test_bedrock_agentcore_basic(model): @pytest.mark.parametrize( "model", [ - "bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_13sf6-cALnp38iZD", # streaming invocation + "bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:941277531214:runtime/hosted_agent_13sf6-4046UzHSwy", # streaming invocation ], ) async def test_bedrock_agentcore_with_streaming(model): @@ -54,7 +54,7 @@ async def test_bedrock_agentcore_with_streaming(model): print("running streming test for model=", model) # litellm._turn_on_debug() response = await litellm.acompletion( - model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC", + model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:941277531214:runtime/hosted_agent_r9jvp-Rq79QFC2fp", messages=[ { "role": "user", @@ -82,7 +82,7 @@ def test_bedrock_agentcore_with_custom_params(): with patch.object(client, "post", return_value=MagicMock()) as mock_post: try: response = litellm.completion( - model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC", + model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:941277531214:runtime/hosted_agent_r9jvp-Rq79QFC2fp", messages=[ { "role": "user", @@ -105,7 +105,7 @@ def test_bedrock_agentcore_with_custom_params(): url = call_kwargs["url"] print(f"URL: {url}") assert ( - "/runtimes/arn%3Aaws%3Abedrock-agentcore%3Aus-west-2%3A888602223428%3Aruntime%2Fhosted_agent_r9jvp-3ySZuRHjLC/invocations" + "/runtimes/arn%3Aaws%3Abedrock-agentcore%3Aus-west-2%3A941277531214%3Aruntime%2Fhosted_agent_r9jvp-Rq79QFC2fp/invocations" in url ) assert "qualifier=DEFAULT" in url @@ -150,7 +150,7 @@ def test_bedrock_agentcore_with_runtime_user_id(): with patch.object(client, "post", return_value=MagicMock()) as mock_post: try: response = litellm.completion( - model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC", + model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:941277531214:runtime/hosted_agent_r9jvp-Rq79QFC2fp", messages=[ { "role": "user", @@ -189,7 +189,7 @@ def test_bedrock_agentcore_with_session_and_user(): with patch.object(client, "post", return_value=MagicMock()) as mock_post: try: response = litellm.completion( - model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC", + model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:941277531214:runtime/hosted_agent_r9jvp-Rq79QFC2fp", messages=[ { "role": "user", @@ -234,7 +234,7 @@ def test_bedrock_agentcore_with_api_key_bearer_token(): with patch.object(client, "post", return_value=MagicMock()) as mock_post: try: response = litellm.completion( - model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC", + model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:941277531214:runtime/hosted_agent_r9jvp-Rq79QFC2fp", messages=[ { "role": "user", @@ -282,7 +282,7 @@ def test_bedrock_agentcore_with_all_parameters(): with patch.object(client, "post", return_value=MagicMock()) as mock_post: try: response = litellm.completion( - model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC", + model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:941277531214:runtime/hosted_agent_r9jvp-Rq79QFC2fp", messages=[ { "role": "user", @@ -350,7 +350,7 @@ def test_bedrock_agentcore_without_api_key_uses_sigv4(): with patch.object(client, "post", return_value=MagicMock()) as mock_post: try: response = litellm.completion( - model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC", + model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:941277531214:runtime/hosted_agent_r9jvp-Rq79QFC2fp", messages=[ { "role": "user", @@ -625,7 +625,7 @@ def test_agentcore_synchronous_non_streaming_response(): with patch.object(client, "post", return_value=mock_response) as mock_post: # Make a synchronous (non-streaming) completion call response = litellm.completion( - model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC", + model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:941277531214:runtime/hosted_agent_r9jvp-Rq79QFC2fp", messages=[ { "role": "user", diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index 15f950224d2a..69c87d1d23fa 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -115,7 +115,7 @@ def test_completion_bedrock_guardrails(streaming): ], max_tokens=10, guardrailConfig={ - "guardrailIdentifier": "ff6ujrregl1q", + "guardrailIdentifier": "4w3d1di3snt5", "guardrailVersion": "DRAFT", "trace": "enabled", }, @@ -144,7 +144,7 @@ def test_completion_bedrock_guardrails(streaming): stream=True, max_tokens=10, guardrailConfig={ - "guardrailIdentifier": "ff6ujrregl1q", + "guardrailIdentifier": "4w3d1di3snt5", "guardrailVersion": "DRAFT", "trace": "enabled", }, @@ -475,7 +475,7 @@ def test_bedrock_claude_3(image_url): ], } response: ModelResponse = completion( - model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0", + model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", num_retries=3, **data, ) # type: ignore @@ -498,7 +498,7 @@ def test_bedrock_claude_3(image_url): @pytest.mark.parametrize( "model", [ - "anthropic.claude-3-sonnet-20240229-v1:0", + "us.anthropic.claude-sonnet-4-5-20250929-v1:0", # "meta.llama3-70b-instruct-v1:0", # "anthropic.claude-v2", # "mistral.mixtral-8x7b-instruct-v0:1", @@ -537,7 +537,7 @@ def test_bedrock_stop_value(stop, model): @pytest.mark.parametrize( "model", [ - "anthropic.claude-3-sonnet-20240229-v1:0", + "us.anthropic.claude-sonnet-4-5-20250929-v1:0", "mistral.mixtral-8x7b-instruct-v0:1", ], ) @@ -602,7 +602,7 @@ def test_bedrock_claude_3_tool_calling(): } ] response: ModelResponse = completion( - model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0", + model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", messages=messages, tools=tools, tool_choice="auto", @@ -630,7 +630,7 @@ def test_bedrock_claude_3_tool_calling(): ) # In the second response, Claude should deduce answer from tool results second_response = completion( - model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0", + model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", messages=messages, tools=tools, tool_choice="auto", @@ -737,7 +737,7 @@ def test_bedrock_ptu(): from openai.types.chat import ChatCompletion model_id = ( - "arn:aws:bedrock:us-west-2:888602223428:provisioned-model/8fxff74qyhs3" + "arn:aws:bedrock:us-west-2:941277531214:provisioned-model/8fxff74qyhs3" ) try: response = litellm.completion( @@ -752,7 +752,7 @@ def test_bedrock_ptu(): assert "url" in mock_client_post.call_args.kwargs assert ( mock_client_post.call_args.kwargs["url"] - == "https://bedrock-runtime.us-west-2.amazonaws.com/model/arn%3Aaws%3Abedrock%3Aus-west-2%3A888602223428%3Aprovisioned-model%2F8fxff74qyhs3/converse" + == "https://bedrock-runtime.us-west-2.amazonaws.com/model/arn%3Aaws%3Abedrock%3Aus-west-2%3A941277531214%3Aprovisioned-model%2F8fxff74qyhs3/converse" ) mock_client_post.assert_called_once() @@ -2327,7 +2327,7 @@ def test_bedrock_cross_region_inference(monkeypatch): def test_bedrock_empty_content_real_call(): completion( - model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0", + model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", messages=[ { "role": "user", diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py index cce6d33e7995..c7abdb5f4936 100644 --- a/tests/local_testing/test_completion.py +++ b/tests/local_testing/test_completion.py @@ -299,7 +299,10 @@ def test_completion_claude_3(): @pytest.mark.parametrize( "model", - ["anthropic/claude-sonnet-4-5-20250929", "anthropic.claude-3-sonnet-20240229-v1:0"], + [ + "anthropic/claude-sonnet-4-5-20250929", + "us.anthropic.claude-sonnet-4-5-20250929-v1:0", + ], ) def test_completion_claude_3_function_call(model): litellm.set_verbose = True @@ -385,7 +388,7 @@ def test_completion_claude_3_function_call(model): [ ("gpt-3.5-turbo", None, None), ("claude-sonnet-4-5-20250929", None, None), - ("anthropic.claude-3-sonnet-20240229-v1:0", None, None), + ("us.anthropic.claude-sonnet-4-5-20250929-v1:0", None, None), # ( # "azure_ai/command-r-plus", # os.getenv("AZURE_COHERE_API_KEY"), @@ -1578,7 +1581,7 @@ def test_completion_openai(): [ # ("gpt-4o-2024-08-06", None), # ("azure/gpt-4.1-mini", None), - ("bedrock/anthropic.claude-3-sonnet-20240229-v1:0", None), + ("bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", None), # ("azure/gpt-4o-new-test", "2024-08-01-preview"), ], ) @@ -1666,15 +1669,13 @@ def custom_callback( ################################################# - print( - f""" + print(f""" Model: {model}, Messages: {messages}, User: {user}, Seed: {kwargs["seed"]}, temperature: {kwargs["temperature"]}, - """ - ) + """) assert kwargs["user"] == "ishaans app" assert kwargs["model"] == "gpt-3.5-turbo-1106" @@ -2699,7 +2700,7 @@ def test_bedrock_deepseek_custom_prompt_dict(): def test_bedrock_deepseek_known_tokenizer_config(monkeypatch): model = ( - "deepseek_r1/arn:aws:bedrock:us-west-2:888602223428:imported-model/bnnr6463ejgf" + "deepseek_r1/arn:aws:bedrock:us-west-2:941277531214:imported-model/bnnr6463ejgf" ) from litellm.llms.custom_httpx.http_handler import HTTPHandler from unittest.mock import Mock @@ -2914,8 +2915,8 @@ def response_format_tests(response: litellm.ModelResponse): "model", [ "bedrock/mistral.mistral-large-2407-v1:0", - "bedrock/cohere.command-r-plus-v1:0", - "anthropic.claude-3-sonnet-20240229-v1:0", + "us.anthropic.claude-haiku-4-5-20251001-v1:0", + "us.anthropic.claude-sonnet-4-5-20250929-v1:0", "mistral.mistral-7b-instruct-v0:2", "meta.llama3-8b-instruct-v1:0", ], diff --git a/tests/local_testing/test_function_call_parsing.py b/tests/local_testing/test_function_call_parsing.py index f9582fcc5746..2453571f1c43 100644 --- a/tests/local_testing/test_function_call_parsing.py +++ b/tests/local_testing/test_function_call_parsing.py @@ -142,7 +142,8 @@ def parse_call(call) -> List[Trade]: @pytest.mark.parametrize( - "model", ["claude-haiku-4-5-20251001", "anthropic.claude-3-haiku-20240307-v1:0"] + "model", + ["claude-haiku-4-5-20251001", "us.anthropic.claude-haiku-4-5-20251001-v1:0"], ) @pytest.mark.flaky(retries=6, delay=10) def test_function_call_parsing(model): diff --git a/tests/local_testing/test_function_calling.py b/tests/local_testing/test_function_calling.py index 3c7e004b62e5..1cad7d1421e0 100644 --- a/tests/local_testing/test_function_calling.py +++ b/tests/local_testing/test_function_calling.py @@ -49,7 +49,7 @@ def get_current_weather(location, unit="fahrenheit"): "mistral/mistral-large-latest", "claude-haiku-4-5-20251001", "gemini/gemini-2.5-flash-lite", - "anthropic.claude-3-sonnet-20240229-v1:0", + "us.anthropic.claude-sonnet-4-5-20250929-v1:0", ], ) @pytest.mark.flaky(retries=3, delay=1) @@ -267,7 +267,6 @@ def test_aaparallel_function_call_with_anthropic_thinking(model): from litellm.types.utils import ChatCompletionMessageToolCall, Function, Message - _PARALLEL_TOOL_HISTORY_MESSAGES = [ { "role": "user", @@ -303,7 +302,7 @@ def test_aaparallel_function_call_with_anthropic_thinking(model): [ # Bedrock Converse still requires modify_params to inject the dummy tool. ( - "anthropic.claude-3-sonnet-20240229-v1:0", + "us.anthropic.claude-sonnet-4-5-20250929-v1:0", _PARALLEL_TOOL_HISTORY_MESSAGES, True, ), @@ -314,7 +313,7 @@ def test_aaparallel_function_call_with_anthropic_thinking(model): False, ), ( - "anthropic.claude-3-sonnet-20240229-v1:0", + "us.anthropic.claude-sonnet-4-5-20250929-v1:0", [ { "role": "user", @@ -579,7 +578,7 @@ def test_groq_parallel_function_call(): @pytest.mark.parametrize( "model", [ - "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", + "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", ], ) def test_passing_tool_result_as_list(model): diff --git a/tests/local_testing/test_sagemaker.py b/tests/local_testing/test_sagemaker.py index d4c5a5a857fb..fdc8347c36aa 100644 --- a/tests/local_testing/test_sagemaker.py +++ b/tests/local_testing/test_sagemaker.py @@ -57,7 +57,7 @@ async def test_completion_sagemaker(sync_mode): print("testing sagemaker") if sync_mode is True: response = litellm.completion( - model="sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", + model="sagemaker/litellm-ci-textgen", messages=[ {"role": "user", "content": "hi"}, ], @@ -67,7 +67,7 @@ async def test_completion_sagemaker(sync_mode): ) else: response = await litellm.acompletion( - model="sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", + model="sagemaker/litellm-ci-textgen", messages=[ {"role": "user", "content": "hi"}, ], @@ -158,7 +158,7 @@ async def test_completion_sagemaker_messages_api(sync_mode): "model", [ # "sagemaker_chat/huggingface-pytorch-tgi-inference-2024-08-23-15-48-59-245", - "sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", + "sagemaker/litellm-ci-textgen", ], ) # @pytest.mark.flaky(retries=3, delay=1) @@ -218,7 +218,7 @@ async def test_completion_sagemaker_stream(sync_mode, model): "model", [ # "sagemaker_chat/huggingface-pytorch-tgi-inference-2024-08-23-15-48-59-245", - "sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", + "sagemaker/litellm-ci-textgen", ], ) async def test_completion_sagemaker_streaming_bad_request(sync_mode, model): @@ -256,7 +256,7 @@ def return_val(): "id": "cmpl-mockid", "object": "text_completion", "created": 1629800000, - "model": "sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", + "model": "sagemaker/litellm-ci-textgen", "choices": [ { "text": "This is a mock response from SageMaker.", @@ -282,7 +282,7 @@ def return_val(): ) as mock_post: # Act: Call the litellm.acompletion function response = await litellm.acompletion( - model="sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", + model="sagemaker/litellm-ci-textgen", messages=[ {"role": "user", "content": "hi"}, ], @@ -302,7 +302,7 @@ def return_val(): assert args_to_sagemaker == expected_payload assert ( kwargs["url"] - == "https://runtime.sagemaker.us-west-2.amazonaws.com/endpoints/jumpstart-dft-hf-textgeneration1-mp-20240815-185614/invocations" + == "https://runtime.sagemaker.us-west-2.amazonaws.com/endpoints/litellm-ci-textgen/invocations" ) @@ -316,7 +316,7 @@ def return_val(): "id": "cmpl-mockid", "object": "text_completion", "created": 1629800000, - "model": "sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", + "model": "sagemaker/litellm-ci-textgen", "choices": [ { "text": "This is a mock response from SageMaker.", @@ -342,7 +342,7 @@ def return_val(): ) as mock_post: # Act: Call the litellm.acompletion function response = litellm.completion( - model="sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", + model="sagemaker/litellm-ci-textgen", messages=[ {"role": "user", "content": "hi"}, ], @@ -362,7 +362,7 @@ def return_val(): assert args_to_sagemaker == expected_payload assert ( kwargs["url"] - == "https://runtime.sagemaker.us-west-2.amazonaws.com/endpoints/jumpstart-dft-hf-textgeneration1-mp-20240815-185614/invocations" + == "https://runtime.sagemaker.us-west-2.amazonaws.com/endpoints/litellm-ci-textgen/invocations" ) @@ -377,7 +377,7 @@ def return_val(): "id": "cmpl-mockid", "object": "text_completion", "created": 1629800000, - "model": "sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", + "model": "sagemaker/litellm-ci-textgen", "choices": [ { "text": "This is a mock response from SageMaker.", @@ -433,7 +433,7 @@ def return_val(): "id": "cmpl-mockid", "object": "text_completion", "created": 1629800000, - "model": "sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", + "model": "sagemaker/litellm-ci-textgen", "choices": [ { "text": "This is a mock response from SageMaker.", @@ -459,7 +459,7 @@ def return_val(): ) as mock_post: # Act: Call the litellm.acompletion function response = litellm.completion( - model="sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", + model="sagemaker/litellm-ci-textgen", messages=[ {"role": "user", "content": "hi"}, ], @@ -482,5 +482,5 @@ def return_val(): assert args_to_sagemaker == expected_payload assert ( kwargs["url"] - == "https://runtime.sagemaker.us-west-5.amazonaws.com/endpoints/jumpstart-dft-hf-textgeneration1-mp-20240815-185614/invocations" + == "https://runtime.sagemaker.us-west-5.amazonaws.com/endpoints/litellm-ci-textgen/invocations" ) diff --git a/tests/local_testing/test_streaming.py b/tests/local_testing/test_streaming.py index 10f351714e13..eb153404a44b 100644 --- a/tests/local_testing/test_streaming.py +++ b/tests/local_testing/test_streaming.py @@ -1174,7 +1174,7 @@ async def test_completion_replicate_llama3_streaming(sync_mode): [ # ["bedrock/ai21.jamba-instruct-v1:0", "us-east-1"], # ["bedrock/cohere.command-r-plus-v1:0", None], - ["anthropic.claude-3-sonnet-20240229-v1:0", None], + ["us.anthropic.claude-sonnet-4-5-20250929-v1:0", None], # ["mistral.mistral-7b-instruct-v0:2", None], # ["meta.llama3-8b-instruct-v1:0", None], ], @@ -1246,7 +1246,7 @@ def test_bedrock_claude_3_streaming(): try: litellm.set_verbose = True response: ModelResponse = completion( # type: ignore - model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0", + model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", messages=messages, max_tokens=10, # type: ignore stream=True, @@ -1276,7 +1276,7 @@ def test_bedrock_claude_3_streaming(): "model", [ "claude-haiku-4-5-20251001", - "cohere.command-r-plus-v1:0", # bedrock + "us.anthropic.claude-haiku-4-5-20251001-v1:0", # bedrock "gpt-3.5-turbo", ], ) @@ -3500,7 +3500,7 @@ def test_unit_test_perplexity_citations_chunk(): [ "gpt-3.5-turbo", "claude-sonnet-4-5-20250929", - "anthropic.claude-3-sonnet-20240229-v1:0", + "us.anthropic.claude-sonnet-4-5-20250929-v1:0", # "vertex_ai/claude-3-5-sonnet@20240620", ], ) diff --git a/tests/logging_callback_tests/test_amazing_s3_logs.py b/tests/logging_callback_tests/test_amazing_s3_logs.py index dab2a0cc0b92..e6291a940499 100644 --- a/tests/logging_callback_tests/test_amazing_s3_logs.py +++ b/tests/logging_callback_tests/test_amazing_s3_logs.py @@ -27,7 +27,7 @@ async def test_basic_s3_logging(sync_mode, streaming): verbose_logger.setLevel(level=logging.DEBUG) litellm.success_callback = ["s3"] litellm.s3_callback_params = { - "s3_bucket_name": "load-testing-oct", + "s3_bucket_name": "load-testing-oct-941277531214", "s3_aws_secret_access_key": "os.environ/AWS_SECRET_ACCESS_KEY", "s3_aws_access_key_id": "os.environ/AWS_ACCESS_KEY_ID", "s3_region_name": "us-west-2", @@ -64,14 +64,14 @@ async def test_basic_s3_logging(sync_mode, streaming): await asyncio.sleep(2) print(f"response: {response}") - total_objects, all_s3_keys = list_all_s3_objects("load-testing-oct") + total_objects, all_s3_keys = list_all_s3_objects("load-testing-oct-941277531214") # assert that atlest one key has response.id in it assert any(response_id in key for key in all_s3_keys) s3 = boto3.client("s3") # delete all objects for key in all_s3_keys: - s3.delete_object(Bucket="load-testing-oct", Key=key) + s3.delete_object(Bucket="load-testing-oct-941277531214", Key=key) @pytest.mark.asyncio @@ -82,7 +82,7 @@ async def test_basic_s3_v2_logging(streaming): from litellm.integrations.s3_v2 import S3Logger litellm.s3_callback_params = { - "s3_bucket_name": "load-testing-oct", + "s3_bucket_name": "load-testing-oct-941277531214", "s3_aws_secret_access_key": "test-secret", "s3_aws_access_key_id": "test-key", "s3_region_name": "us-west-2", diff --git a/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py b/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py index d6d0652ed777..0d4405094b58 100644 --- a/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py +++ b/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py @@ -2,7 +2,6 @@ import os import sys - sys.path.insert(0, os.path.abspath("../..")) import asyncio @@ -67,7 +66,7 @@ def setup_vector_store_registry(): litellm.vector_store_registry = VectorStoreRegistry( vector_stores=[ LiteLLM_ManagedVectorStore( - vector_store_id="T37J8R4WTM", custom_llm_provider="bedrock" + vector_store_id="LCYXFBR2TU", custom_llm_provider="bedrock" ) ] ) @@ -111,7 +110,7 @@ async def test_e2e_bedrock_knowledgebase_retrieval_with_completion( response = await litellm.acompletion( model="anthropic/claude-3.5-sonnet", messages=[{"role": "user", "content": "what is litellm?"}], - vector_store_ids=["T37J8R4WTM"], + vector_store_ids=["LCYXFBR2TU"], client=client, ) except Exception as e: @@ -152,7 +151,7 @@ async def test_e2e_bedrock_knowledgebase_retrieval_with_llm_api_call( response = await litellm.acompletion( model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "what is litellm?"}], - vector_store_ids=["T37J8R4WTM"], + vector_store_ids=["LCYXFBR2TU"], client=async_client, ) print("OPENAI RESPONSE:", json.dumps(dict(response), indent=4, default=str)) @@ -196,7 +195,7 @@ async def test_e2e_bedrock_knowledgebase_retrieval_with_llm_api_call_streaming( response = await litellm.acompletion( model=f"anthropic/{os.environ.get('CI_CD_DEFAULT_ANTHROPIC_MODEL', 'claude-haiku-4-5-20251001')}", messages=[{"role": "user", "content": "what is litellm?"}], - vector_store_ids=["T37J8R4WTM"], + vector_store_ids=["LCYXFBR2TU"], stream=True, client=async_client, ) @@ -255,7 +254,7 @@ async def test_e2e_bedrock_knowledgebase_retrieval_with_llm_api_call_with_tools( model=f"anthropic/{os.environ.get('CI_CD_DEFAULT_ANTHROPIC_MODEL', 'claude-haiku-4-5-20251001')}", messages=[{"role": "user", "content": "what is litellm?"}], max_tokens=10, - tools=[{"type": "file_search", "vector_store_ids": ["T37J8R4WTM"]}], + tools=[{"type": "file_search", "vector_store_ids": ["LCYXFBR2TU"]}], ) assert response is not None @@ -279,7 +278,7 @@ async def test_e2e_bedrock_knowledgebase_retrieval_with_llm_api_call_with_tools_ tools=[ { "type": "file_search", - "vector_store_ids": ["T37J8R4WTM"], + "vector_store_ids": ["LCYXFBR2TU"], "filters": { "key": "user_id", "value": "fake-user-id", @@ -387,7 +386,7 @@ async def fake_async_vector_store_search_handler( tools=[ { "type": "file_search", - "vector_store_ids": ["T37J8R4WTM"], + "vector_store_ids": ["LCYXFBR2TU"], "filters": { "key": "user_id", "value": "fake-user-id", @@ -461,7 +460,7 @@ async def mock_create(**kwargs): await litellm.acompletion( model="gpt-5.5", messages=[{"role": "user", "content": "what is litellm?"}], - vector_store_ids=["T37J8R4WTM"], + vector_store_ids=["LCYXFBR2TU"], client=client, ) except Exception as e: @@ -537,7 +536,7 @@ async def mock_create(**kwargs): await litellm.acompletion( model="gpt-5.5", messages=[{"role": "user", "content": "what is litellm?"}], - tools=[{"type": "file_search", "vector_store_ids": ["T37J8R4WTM"]}], + tools=[{"type": "file_search", "vector_store_ids": ["LCYXFBR2TU"]}], client=client, ) except Exception as e: @@ -611,7 +610,7 @@ async def mock_create(**kwargs): model="gpt-5.5", messages=[{"role": "user", "content": "what is litellm?"}], tools=[ - {"type": "file_search", "vector_store_ids": ["T37J8R4WTM"]}, + {"type": "file_search", "vector_store_ids": ["LCYXFBR2TU"]}, {"type": "file_search", "vector_store_ids": ["unknownVS"]}, ], client=client, @@ -645,7 +644,7 @@ async def mock_create(**kwargs): # model="gpt-5.5", # messages=[{"role": "user", "content": "what is litellm?"}], # vector_store_ids = [ -# "T37J8R4WTM" +# "LCYXFBR2TU" # ], # ) @@ -667,7 +666,7 @@ async def mock_create(**kwargs): # # expect the vector store request metadata object to have the correct values # vector_store_request_metadata = standard_logging_vector_store_request_metadata[0] -# assert vector_store_request_metadata.get("vector_store_id") == "T37J8R4WTM" +# assert vector_store_request_metadata.get("vector_store_id") == "LCYXFBR2TU" # assert vector_store_request_metadata.get("query") == "what is litellm?" # assert vector_store_request_metadata.get("custom_llm_provider") == "bedrock" @@ -723,7 +722,7 @@ async def test_e2e_bedrock_knowledgebase_retrieval_without_vector_store_registry response = await litellm.acompletion( model="anthropic/claude-3.5-sonnet", messages=[{"role": "user", "content": "what is litellm?"}], - vector_store_ids=["T37J8R4WTM"], + vector_store_ids=["LCYXFBR2TU"], client=client, ) except Exception as e: diff --git a/tests/test_litellm/llms/bedrock/chat/agentcore/test_agentcore_transformation.py b/tests/test_litellm/llms/bedrock/chat/agentcore/test_agentcore_transformation.py index 64b43b15dcd0..3287061d37ec 100644 --- a/tests/test_litellm/llms/bedrock/chat/agentcore/test_agentcore_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/agentcore/test_agentcore_transformation.py @@ -76,7 +76,7 @@ def test_accept_header_in_completion_request_jwt(self): with patch.object(client, "post", return_value=MagicMock()) as mock_post: try: litellm.completion( - model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/test_runtime", + model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:941277531214:runtime/test_runtime", messages=[{"role": "user", "content": "test"}], api_key="test-jwt-token", client=client, @@ -281,7 +281,7 @@ def test_sync_streaming_with_json_response(self): with patch.object(client, "post", return_value=mock_response): response = litellm.completion( - model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/test_agent", + model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:941277531214:runtime/test_agent", messages=[{"role": "user", "content": "test"}], stream=True, client=client, @@ -318,7 +318,7 @@ async def test_async_streaming_with_json_response(self): client, "post", new_callable=AsyncMock, return_value=mock_response ): response = await litellm.acompletion( - model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/test_agent", + model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:941277531214:runtime/test_agent", messages=[{"role": "user", "content": "test"}], stream=True, client=client, @@ -353,7 +353,7 @@ def test_sync_streaming_malformed_json_raises_error(self): Exception, match="Failed to read/parse JSON response body" ): litellm.completion( - model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/test_agent", + model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:941277531214:runtime/test_agent", messages=[{"role": "user", "content": "test"}], stream=True, client=client, @@ -383,7 +383,7 @@ async def test_async_streaming_malformed_json_raises_error(self): Exception, match="Failed to read/parse JSON response body" ): await litellm.acompletion( - model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/test_agent", + model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:941277531214:runtime/test_agent", messages=[{"role": "user", "content": "test"}], stream=True, client=client, diff --git a/tests/test_openai_endpoints.py b/tests/test_openai_endpoints.py index e898b88a5567..29875a044133 100644 --- a/tests/test_openai_endpoints.py +++ b/tests/test_openai_endpoints.py @@ -446,7 +446,7 @@ class EventsList(BaseModel): client = AsyncOpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") res = await client.beta.chat.completions.parse( - model="bedrock/us.anthropic.claude-3-sonnet-20240229-v1:0", + model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", messages=messages, response_format=EventsList, timeout=60, diff --git a/tests/vector_store_tests/test_bedrock_vector_store.py b/tests/vector_store_tests/test_bedrock_vector_store.py index d8af1c7188b9..47e73e61c590 100644 --- a/tests/vector_store_tests/test_bedrock_vector_store.py +++ b/tests/vector_store_tests/test_bedrock_vector_store.py @@ -22,7 +22,7 @@ def get_base_create_vector_store_args(self) -> dict: def get_base_request_args(self): return { - "vector_store_id": "T37J8R4WTM", + "vector_store_id": "LCYXFBR2TU", "custom_llm_provider": "bedrock", "query": "what happens after we add a model", } @@ -106,7 +106,7 @@ async def test_bedrock_search_with_router(): _router = Router(model_list=[]) search_response = await _router.avector_store_search( query="what happens after we add a model", - vector_store_id="T37J8R4WTM", + vector_store_id="LCYXFBR2TU", custom_llm_provider="bedrock", ) print(search_response) @@ -150,7 +150,7 @@ async def test_bedrock_search_with_credentials_managed_registry(): # Create vector store with credential reference vector_store = LiteLLM_ManagedVectorStore( - vector_store_id="T37J8R4WTM", + vector_store_id="LCYXFBR2TU", custom_llm_provider="bedrock", created_at=datetime.now(timezone.utc), updated_at=datetime.now(timezone.utc), @@ -162,7 +162,7 @@ async def test_bedrock_search_with_credentials_managed_registry(): litellm.vector_store_registry = registry # Verify credentials can be retrieved from registry - retrieved_credentials = registry.get_credentials_for_vector_store("T37J8R4WTM") + retrieved_credentials = registry.get_credentials_for_vector_store("LCYXFBR2TU") assert retrieved_credentials, "Should retrieve credentials from registry" assert retrieved_credentials.get("aws_access_key_id") == "test_access_key" assert retrieved_credentials.get("aws_secret_access_key") == "test_secret_key" @@ -194,7 +194,7 @@ async def test_bedrock_search_with_credentials_managed_registry(): search_response = await _router.avector_store_search( query="what happens after we add a model", - vector_store_id="T37J8R4WTM", + vector_store_id="LCYXFBR2TU", custom_llm_provider="bedrock", ) @@ -203,7 +203,7 @@ async def test_bedrock_search_with_credentials_managed_registry(): call_kwargs = mock_handler.call_args[1] # Verify that the credential accessor was called with the correct vector store ID - mock_get_creds.assert_called_with("T37J8R4WTM") + mock_get_creds.assert_called_with("LCYXFBR2TU") # Verify the credentials were injected into the search call litellm_params = call_kwargs.get("litellm_params", {}) @@ -224,7 +224,7 @@ async def test_bedrock_search_with_credentials_managed_registry(): assert search_response["data"][0]["id"] == "test_result" print( - f"✅ Test passed: Credential accessor was called with vector store ID: T37J8R4WTM" + f"✅ Test passed: Credential accessor was called with vector store ID: LCYXFBR2TU" ) print(f"✅ Retrieved credentials: {retrieved_credentials}") print(f"✅ Credentials were injected into search call") From 30551de371fe37b1ee8de1e5fe949ddcdad272e8 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Mon, 25 May 2026 12:13:17 -0700 Subject: [PATCH 03/13] fix(otel): export SERVER span on management-endpoint success without http_request (#28794) Co-authored-by: Yassin Kortam --- litellm/proxy/management_helpers/utils.py | 124 ++++++------ .../test_otel_admin_endpoints.py | 176 ++++++++++++++++++ 2 files changed, 245 insertions(+), 55 deletions(-) diff --git a/litellm/proxy/management_helpers/utils.py b/litellm/proxy/management_helpers/utils.py index b7d5cc30c492..ab100615b466 100644 --- a/litellm/proxy/management_helpers/utils.py +++ b/litellm/proxy/management_helpers/utils.py @@ -2,7 +2,7 @@ ## Helper utils for the management endpoints (keys/users/teams) from datetime import datetime from functools import wraps -from typing import List, Optional, Tuple +from typing import Any, Callable, List, Optional, Tuple from fastapi import HTTPException, Request @@ -435,6 +435,58 @@ async def send_management_endpoint_alert( ) +async def _emit_management_endpoint_otel_span( + func: Callable, + kwargs: dict, + parent_otel_span: Any, + start_time: datetime, + end_time: datetime, + result: Any = None, + exception: Optional[Exception] = None, +) -> None: + """Stamp + end the parent OTEL SERVER span for a management endpoint. + + Routes the request/response (or exception) through the OTEL success/failure + hook. Falls back to ``func.__name__`` for the route when the handler has no + ``http_request`` param — endpoints like ``/key/generate`` never receive one, + and gating the hook on it leaked their SERVER span (created in auth, never + ended → never exported). Always emitting keeps both success and failure + paths consistent. + """ + from litellm.proxy.proxy_server import open_telemetry_logger + + if open_telemetry_logger is None: + return + + http_request: Optional[Request] = kwargs.get("http_request") + if http_request is not None: + route = http_request.url.path + request_body: dict = await _read_request_body(request=http_request) + else: + route = func.__name__ + request_body = {} + + logging_payload = ManagementEndpointLoggingPayload( + route=route, + request_data=request_body, + response=None, + start_time=start_time, + end_time=end_time, + exception=exception, + ) + + if exception is None: + await open_telemetry_logger.async_management_endpoint_success_hook( + logging_payload=logging_payload, + parent_otel_span=parent_otel_span, + ) + else: + await open_telemetry_logger.async_management_endpoint_failure_hook( + logging_payload=logging_payload, + parent_otel_span=parent_otel_span, + ) + + def management_endpoint_wrapper(func): """ This wrapper does the following: @@ -446,13 +498,10 @@ def management_endpoint_wrapper(func): @wraps(func) async def wrapper(*args, **kwargs): start_time = datetime.now() - _http_request: Optional[Request] = None try: result = await func(*args, **kwargs) end_time = datetime.now() try: - if kwargs is None: - kwargs = {} user_api_key_dict: UserAPIKeyAuth = ( kwargs.get("user_api_key_dict") or UserAPIKeyAuth() ) @@ -462,31 +511,16 @@ async def wrapper(*args, **kwargs): user_api_key_dict=user_api_key_dict, function_name=func.__name__, ) - _http_request = kwargs.get("http_request", None) parent_otel_span = getattr(user_api_key_dict, "parent_otel_span", None) if parent_otel_span is not None: - from litellm.proxy.proxy_server import open_telemetry_logger - - if open_telemetry_logger is not None: - if _http_request: - _route = _http_request.url.path - _request_body: dict = await _read_request_body( - request=_http_request - ) - _response = dict(result) if result is not None else None - - logging_payload = ManagementEndpointLoggingPayload( - route=_route, - request_data=_request_body, - response=_response, - start_time=start_time, - end_time=end_time, - ) - - await open_telemetry_logger.async_management_endpoint_success_hook( # type: ignore - logging_payload=logging_payload, - parent_otel_span=parent_otel_span, - ) + await _emit_management_endpoint_otel_span( + func=func, + kwargs=kwargs, + parent_otel_span=parent_otel_span, + start_time=start_time, + end_time=end_time, + result=result, + ) # Delete updated/deleted info from cache _delete_api_key_from_cache(kwargs=kwargs) @@ -502,39 +536,19 @@ async def wrapper(*args, **kwargs): except Exception as e: end_time = datetime.now() - if kwargs is None: - kwargs = {} user_api_key_dict: UserAPIKeyAuth = ( kwargs.get("user_api_key_dict") or UserAPIKeyAuth() ) parent_otel_span = getattr(user_api_key_dict, "parent_otel_span", None) if parent_otel_span is not None: - from litellm.proxy.proxy_server import open_telemetry_logger - - if open_telemetry_logger is not None: - _http_request = kwargs.get("http_request") - if _http_request: - _route = _http_request.url.path - _request_body: dict = await _read_request_body( - request=_http_request - ) - else: - _route = func.__name__ - _request_body = {} - - logging_payload = ManagementEndpointLoggingPayload( - route=_route, - request_data=_request_body, - response=None, - start_time=start_time, - end_time=end_time, - exception=e, - ) - - await open_telemetry_logger.async_management_endpoint_failure_hook( # type: ignore - logging_payload=logging_payload, - parent_otel_span=parent_otel_span, - ) + await _emit_management_endpoint_otel_span( + func=func, + kwargs=kwargs, + parent_otel_span=parent_otel_span, + start_time=start_time, + end_time=end_time, + exception=e, + ) raise e diff --git a/tests/test_litellm/integrations/open_telemetry/test_otel_admin_endpoints.py b/tests/test_litellm/integrations/open_telemetry/test_otel_admin_endpoints.py index b1a3b834c3da..34103449dad1 100644 --- a/tests/test_litellm/integrations/open_telemetry/test_otel_admin_endpoints.py +++ b/tests/test_litellm/integrations/open_telemetry/test_otel_admin_endpoints.py @@ -3,6 +3,7 @@ import asyncio from datetime import datetime +from unittest.mock import MagicMock import pytest @@ -14,6 +15,7 @@ from ._helpers import ( HttpStatusException, assert_server_span_attrs, + get_server_span, make_fastapi_http_exception, make_httpx_status_error, ) @@ -28,6 +30,10 @@ def _real_user_api_key_dict(parent_span): ) +async def _noop_alert(*args, **kwargs): + return None + + async def _drive_admin_failure(*, otel, exception, parent_span, route): payload = ManagementEndpointLoggingPayload( route=route, @@ -180,3 +186,173 @@ def test_admin_endpoint_failure_stamps_server_span( expected_url_path=path, where=f"{path} {expected_status}", ) + + +def test_management_wrapper_success_ends_server_span_without_http_request( + server_span_factory, otel_with_exporter, monkeypatch +): + """Regression: management endpoints whose handler does not declare an + ``http_request`` parameter (``/key/generate``, ``/user/new``, ``/mcp/*``, + ...) must still get their parent SERVER span stamped + ended on success. + + The success hook itself stamps 200 and ``end()``s the parent, but the + wrapper only invoked it when ``http_request`` was present — so on success + the span (created in auth) was never ended and never exported. This drives + the real wrapper around an ``http_request``-less handler and asserts the + SERVER span reaches the exporter with status 200. + """ + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.management_helpers import utils as mgmt_utils + + otel, exporter = otel_with_exporter + monkeypatch.setattr(proxy_server, "open_telemetry_logger", otel, raising=False) + monkeypatch.setattr(mgmt_utils, "send_management_endpoint_alert", _noop_alert) + + server_span = server_span_factory(KEY_GENERATE_PATH) + + @mgmt_utils.management_endpoint_wrapper + async def fake_generate_key_fn(data=None, user_api_key_dict=None): + # No ``http_request`` parameter — mirrors generate_key_fn et al. + return {"key": "sk-xyz", "key_name": "k"} + + asyncio.run( + fake_generate_key_fn( + data={}, + user_api_key_dict=_real_user_api_key_dict(server_span), + ) + ) + + assert_server_span_attrs( + exporter, + expected_status=200, + expected_url_path=KEY_GENERATE_PATH, + where="management wrapper success without http_request", + ) + + +def test_management_wrapper_failure_ends_server_span( + server_span_factory, otel_with_exporter, monkeypatch +): + """When the handler raises, the wrapper must route through the failure hook + and stamp + end the parent SERVER span with the error status — even for an + ``http_request``-less handler (route falls back to ``func.__name__``).""" + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.management_helpers import utils as mgmt_utils + + otel, exporter = otel_with_exporter + monkeypatch.setattr(proxy_server, "open_telemetry_logger", otel, raising=False) + + server_span = server_span_factory(KEY_GENERATE_PATH) + + @mgmt_utils.management_endpoint_wrapper + async def failing_fn(data=None, user_api_key_dict=None): + raise HttpStatusException(500, "boom") + + with pytest.raises(HttpStatusException): + asyncio.run( + failing_fn(data={}, user_api_key_dict=_real_user_api_key_dict(server_span)) + ) + + assert_server_span_attrs( + exporter, + expected_status=500, + expected_url_path=KEY_GENERATE_PATH, + where="management wrapper failure", + ) + + +def test_management_wrapper_success_with_http_request( + server_span_factory, otel_with_exporter, monkeypatch +): + """Cover the branch where the handler DOES declare ``http_request``: the + route comes from ``http_request.url.path`` and the body is read from it.""" + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.management_helpers import utils as mgmt_utils + + otel, exporter = otel_with_exporter + monkeypatch.setattr(proxy_server, "open_telemetry_logger", otel, raising=False) + monkeypatch.setattr(mgmt_utils, "send_management_endpoint_alert", _noop_alert) + + async def _fake_body(request=None): + return {"team_alias": "t"} + + monkeypatch.setattr(mgmt_utils, "_read_request_body", _fake_body) + + server_span = server_span_factory("/team/new") + http_request = MagicMock() + http_request.url.path = "/team/new" + + @mgmt_utils.management_endpoint_wrapper + async def fake_new_team(data=None, http_request=None, user_api_key_dict=None): + return {"team_id": "t-1"} + + asyncio.run( + fake_new_team( + data={}, + http_request=http_request, + user_api_key_dict=_real_user_api_key_dict(server_span), + ) + ) + + assert_server_span_attrs( + exporter, + expected_status=200, + expected_url_path="/team/new", + where="management wrapper success with http_request", + ) + + +def test_management_wrapper_noop_when_otel_logger_absent( + server_span_factory, otel_with_exporter, monkeypatch +): + """When no OTEL logger is registered, the helper early-returns and no SERVER + span is exported — and the handler result is still returned unchanged.""" + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.management_helpers import utils as mgmt_utils + + _otel, exporter = otel_with_exporter + monkeypatch.setattr(proxy_server, "open_telemetry_logger", None, raising=False) + monkeypatch.setattr(mgmt_utils, "send_management_endpoint_alert", _noop_alert) + + server_span = server_span_factory(KEY_GENERATE_PATH) + + @mgmt_utils.management_endpoint_wrapper + async def fake_fn(data=None, user_api_key_dict=None): + return {"ok": True} + + result = asyncio.run( + fake_fn(data={}, user_api_key_dict=_real_user_api_key_dict(server_span)) + ) + + assert result == {"ok": True} + assert get_server_span(exporter) is None + + +def test_management_wrapper_swallows_post_success_errors( + server_span_factory, otel_with_exporter, monkeypatch +): + """A failure in post-success bookkeeping (cache invalidation, alerting) must + not propagate — the handler result is returned regardless (non-blocking).""" + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.management_helpers import utils as mgmt_utils + + otel, _exporter = otel_with_exporter + monkeypatch.setattr(proxy_server, "open_telemetry_logger", otel, raising=False) + monkeypatch.setattr(mgmt_utils, "send_management_endpoint_alert", _noop_alert) + + def _boom(*args, **kwargs): + raise RuntimeError("cache backend down") + + monkeypatch.setattr(mgmt_utils, "_delete_api_key_from_cache", _boom) + + server_span = server_span_factory(KEY_GENERATE_PATH) + + @mgmt_utils.management_endpoint_wrapper + async def fake_fn(data=None, user_api_key_dict=None): + return {"ok": True} + + result = asyncio.run( + fake_fn(data={}, user_api_key_dict=_real_user_api_key_dict(server_span)) + ) + + assert result == {"ok": True} From 5f75be5c1c979262901c56b1f5ee727ca4007f5a Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 25 May 2026 13:44:49 -0700 Subject: [PATCH 04/13] chore(ci): merge dev branch (#28801) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(proxy): route path-dependent call sites through get_request_route Replace direct ``request.url.path`` reads in auth, ACL, routing, and audit-log decisions with ``get_request_route(request)`` — the helper already added in ``auth/auth_utils.py`` that returns the ASGI ``scope["path"]`` with ``root_path`` stripped. Starlette reconstructs ``url.path`` from the Host header; ``scope["path"]`` is uvicorn's parse of the request line and matches what FastAPI dispatches on, so it's the authoritative route for any decision that should agree with the actual handler. Sites: - _experimental/mcp_server/auth/user_api_key_auth_mcp.py - management_endpoints/mcp_management_endpoints.py - vector_store_endpoints/utils.py - pass_through_endpoints/pass_through_endpoints.py - auth/route_checks.py - litellm_pre_call_utils.py - spend_tracking/spend_management_endpoints.py - common_utils/http_parsing_utils.py - management_helpers/utils.py - health_endpoints/_health_endpoints.py Adds regression tests in tests/proxy_unit_tests/test_proxy_routes.py that construct a Request with scope["path"] set to a benign route and the Host header crafted so url.path would resolve differently; each site's decision is asserted against scope["path"]. * chore(proxy): make get_request_route imports lazy at call sites Move the ``from litellm.proxy.auth.auth_utils import get_request_route`` imports added in the prior commit back to the function bodies that use them. The module-level form participates in a long-standing import cycle through ``auth_utils -> _types -> ...`` and was flagged by CodeQL on the PR; the lazy form matches the pattern the proxy already uses for ``user_api_key_auth`` and related helpers elsewhere in these files. Also drop the ``RouteChecks._is_assistants_api_request`` delegation in ``_get_metadata_variable_name`` introduced in the prior commit — the delegation pulled ``RouteChecks`` into the same cycle, and the call site reuses the resolved route for its other branches, so inlining the substring check is both cycle-free and avoids a redundant second ``get_request_route`` call. Comment in test_proxy_routes.py acknowledges that the two MCP table entries exercise ``get_request_route`` directly rather than the full production handler (which needs ASGI scope + MCP state to invoke). --------- Co-authored-by: shin-berri Co-authored-by: user <70670632+stuxf@users.noreply.github.com> --- .../mcp_server/auth/user_api_key_auth_mcp.py | 14 +- litellm/proxy/auth/auth_utils.py | 15 ++- litellm/proxy/auth/route_checks.py | 6 +- .../proxy/common_utils/http_parsing_utils.py | 5 +- .../health_endpoints/_health_endpoints.py | 5 +- litellm/proxy/litellm_pre_call_utils.py | 4 +- .../mcp_management_endpoints.py | 5 +- litellm/proxy/management_helpers/utils.py | 7 +- .../pass_through_endpoints.py | 5 +- .../spend_management_endpoints.py | 5 +- litellm/proxy/vector_store_endpoints/utils.py | 18 ++- tests/proxy_unit_tests/test_proxy_routes.py | 121 +++++++++++++++++- 12 files changed, 185 insertions(+), 25 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index 708ec7f11765..70fc2c233e7b 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -118,15 +118,19 @@ async def mock_body(): return b"{}" request.body = mock_body # type: ignore + # Inline import — auth_utils participates in a proxy import cycle. + from litellm.proxy.auth.auth_utils import ( # noqa: PLC0415 + get_request_route, + ) + + request_route = get_request_route(request) # Only OAuth metadata routes registered under /.well-known/ are public. - # Match on request.url.path (path-only, exact prefix) so the substring - # cannot be smuggled via query string, hostname, or a deeper URL segment. - if request.url.path.startswith("/.well-known/"): + if request_route.startswith("/.well-known/"): validated_user_api_key_auth = UserAPIKeyAuth() elif ( not litellm_api_key and MCPRequestHandler._target_servers_delegate_auth_to_upstream( # noqa: E501 - path=request.url.path, mcp_servers=mcp_servers + path=request_route, mcp_servers=mcp_servers ) ): # Operator opted this oauth2 server into upstream-delegated auth @@ -174,7 +178,7 @@ async def mock_body(): "401", "403", ) and MCPRequestHandler._target_servers_use_oauth2( - path=request.url.path, mcp_servers=mcp_servers + path=request_route, mcp_servers=mcp_servers ): verbose_logger.debug( "MCP OAuth2: target server is OAuth2-mode, treating " diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index c4dcca764b29..1e87dcaef1c3 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -498,9 +498,18 @@ def route_in_additonal_public_routes(current_route: str): def get_request_route(request: Request) -> str: """ - Helper to get the route from the request - - remove base url from path if set e.g. `/genai/chat/completions` -> `/chat/completions + Resolve the request route from the ASGI scope, with ``root_path`` stripped. + + Prefer this over ``request.url.path`` for any auth, ACL, routing, or + audit-log decision: Starlette reconstructs ``url.path`` by interpolating + the Host header into a URL string and re-parsing with ``urlsplit``, so a + malformed Host (e.g. ``localhost/?x=1``) collapses ``url.path`` to ``"/"`` + while FastAPI continues to dispatch on ``scope["path"]``. ``scope["path"]`` + is uvicorn's parse of the HTTP request line and matches the actual + handler, so it's the authoritative route. + + Also normalizes sub-path deployments by stripping ``scope["root_path"]`` + e.g. ``/genai/chat/completions`` -> ``/chat/completions``. """ try: scope = request.scope diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index b2878ba0ae63..3144c5cd25b4 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -627,7 +627,11 @@ def _is_assistants_api_request(request: Request) -> bool: Returns: bool: True if `thread` or `assistant` is in the request path, False otherwise """ - if "thread" in request.url.path or "assistant" in request.url.path: + # Inline import — auth_utils participates in a proxy import cycle. + from .auth_utils import get_request_route # noqa: PLC0415 + + route = get_request_route(request) + if "thread" in route or "assistant" in route: return True return False diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 2ce3fda62972..678ff2896491 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -546,7 +546,10 @@ def _add_vector_store_id_from_path(request_data: dict, request: Request) -> None request_data: The request data dictionary to populate request: The FastAPI Request object """ - path = request.url.path + # Inline import — auth_utils participates in a proxy import cycle. + from litellm.proxy.auth.auth_utils import get_request_route # noqa: PLC0415 + + path = get_request_route(request) vector_store_match = re.search(r"/vector_stores/([^/]+)/", path) if vector_store_match: vector_store_id = vector_store_match.group(1) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index ff3df11c4485..ba3aee750470 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -151,7 +151,10 @@ async def test_endpoint(request: Request): dict: A dictionary containing the route of the request URL. """ # ping the proxy server to check if its healthy - return {"route": request.url.path} + # Inline import — auth_utils participates in a proxy import cycle. + from litellm.proxy.auth.auth_utils import get_request_route # noqa: PLC0415 + + return {"route": get_request_route(request)} @router.get( diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 2b840b5495e2..0d27b283c47e 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -333,8 +333,10 @@ def _get_metadata_variable_name(request: Request) -> str: For ALL other endpoints we call this "metadata" """ - path = request.url.path + # Inline imports — auth_utils/route_checks participate in a proxy import cycle. + from litellm.proxy.auth.auth_utils import get_request_route # noqa: PLC0415 + path = get_request_route(request) if "thread" in path or "assistant" in path: return "litellm_metadata" diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index e9d9c243e7cd..431ff49c7cec 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -1568,6 +1568,9 @@ async def _mcp_oauth_user_api_key_auth(request: Request) -> UserAPIKeyAuth: from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415 global_mcp_server_manager, ) + from litellm.proxy.auth.auth_utils import ( # noqa: PLC0415 + get_request_route, + ) server_id = request.path_params.get("server_id", "") if server_id: @@ -1584,7 +1587,7 @@ async def _mcp_oauth_user_api_key_auth(request: Request) -> UserAPIKeyAuth: ): # For /token, require PKCE authorization_code; refresh_token # grants must NOT bypass auth (see comment above). - path_lower = (request.url.path or "").rstrip("/").lower() + path_lower = get_request_route(request).rstrip("/").lower() if path_lower.endswith("/token"): body_data = await _read_request_body(request=request) grant_type = (body_data or {}).get("grant_type", "") diff --git a/litellm/proxy/management_helpers/utils.py b/litellm/proxy/management_helpers/utils.py index ab100615b466..495bce2f00e3 100644 --- a/litellm/proxy/management_helpers/utils.py +++ b/litellm/proxy/management_helpers/utils.py @@ -460,7 +460,12 @@ async def _emit_management_endpoint_otel_span( http_request: Optional[Request] = kwargs.get("http_request") if http_request is not None: - route = http_request.url.path + # Inline import — auth_utils participates in a proxy import cycle. + from litellm.proxy.auth.auth_utils import ( # noqa: PLC0415 + get_request_route, + ) + + route = get_request_route(http_request) request_body: dict = await _read_request_body(request=http_request) else: route = func.__name__ diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 51dbf5ce890b..00eaba09acd9 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -1307,11 +1307,14 @@ async def endpoint_func( # type: ignore user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), subpath: str = "", # captures sub-paths when include_subpath=True ): + from litellm.proxy.auth.auth_utils import ( # noqa: PLC0415 + get_request_route, + ) from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( InitPassThroughEndpointHelpers, ) - path = request.url.path + path = get_request_route(request) # Parse request data based on content type ( diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index e3019801aae1..36beb5e9aba8 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -1817,7 +1817,10 @@ async def ui_view_spend_logs( # noqa: PLR0915 ) try: - is_v2 = "/spend/logs/v2" in request.url.path + # Inline import — auth_utils participates in a proxy import cycle. + from litellm.proxy.auth.auth_utils import get_request_route # noqa: PLC0415 + + is_v2 = "/spend/logs/v2" in get_request_route(request) formats = ["%Y-%m-%d %H:%M:%S", "%Y-%m-%d"] if is_v2 else ["%Y-%m-%d %H:%M:%S"] def parse_date(date_str: str) -> datetime: diff --git a/litellm/proxy/vector_store_endpoints/utils.py b/litellm/proxy/vector_store_endpoints/utils.py index 657b520b2717..1221ccf119f5 100644 --- a/litellm/proxy/vector_store_endpoints/utils.py +++ b/litellm/proxy/vector_store_endpoints/utils.py @@ -330,11 +330,16 @@ def is_allowed_to_call_vector_store_endpoint( provider_config.get_vector_store_endpoints_by_type() ) + # Inline import — auth_utils participates in a proxy import cycle. + from litellm.proxy.auth.auth_utils import get_request_route # noqa: PLC0415 + + request_route = get_request_route(request) + # Determine the permission type based on the request permission_type = None for endpoint in provider_vector_store_endpoints["read"]: if request.method == endpoint[0] and _does_endpoint_match( - endpoint[1], request.url.path + endpoint[1], request_route ): permission_type = "read" break @@ -342,7 +347,7 @@ def is_allowed_to_call_vector_store_endpoint( if permission_type is None: for endpoint in provider_vector_store_endpoints["write"]: if request.method == endpoint[0] and _does_endpoint_match( - endpoint[1], request.url.path + endpoint[1], request_route ): permission_type = "write" break @@ -392,10 +397,15 @@ def is_allowed_to_call_vector_store_files_endpoint( provider_config.get_vector_store_file_endpoints_by_type() ) + # Inline import — auth_utils participates in a proxy import cycle. + from litellm.proxy.auth.auth_utils import get_request_route # noqa: PLC0415 + + request_route = get_request_route(request) + permission_type: Optional[str] = None for endpoint in provider_vector_store_endpoints.get("read", ()): if request.method == endpoint[0] and _does_endpoint_match( - endpoint[1], request.url.path + endpoint[1], request_route ): permission_type = "read" break @@ -403,7 +413,7 @@ def is_allowed_to_call_vector_store_files_endpoint( if permission_type is None: for endpoint in provider_vector_store_endpoints.get("write", ()): if request.method == endpoint[0] and _does_endpoint_match( - endpoint[1], request.url.path + endpoint[1], request_route ): permission_type = "write" break diff --git a/tests/proxy_unit_tests/test_proxy_routes.py b/tests/proxy_unit_tests/test_proxy_routes.py index 34123e992c26..db41bd654098 100644 --- a/tests/proxy_unit_tests/test_proxy_routes.py +++ b/tests/proxy_unit_tests/test_proxy_routes.py @@ -217,9 +217,120 @@ def _create_request_with_host_header(path: str, host_header: str) -> Request: ], ) def test_get_request_route_not_bypassed_by_malformed_host(host_header: str): - for protected_path in ["/health", "/user/new", "/key/generate", "/get/internal_user_settings"]: - request = _create_request_with_host_header(path=protected_path, host_header=host_header) - result = get_request_route(request) - assert result == protected_path, ( - f"Host: {host_header!r} caused route {protected_path!r} to resolve as {result!r}" + for protected_path in [ + "/health", + "/user/new", + "/key/generate", + "/get/internal_user_settings", + ]: + request = _create_request_with_host_header( + path=protected_path, host_header=host_header ) + result = get_request_route(request) + assert ( + result == protected_path + ), f"Host: {host_header!r} caused route {protected_path!r} to resolve as {result!r}" + + +# --------------------------------------------------------------------------- +# Regression tests for variant call sites that previously read request.url.path +# (Host-derived) instead of the ASGI scope path. Each test sends a Host header +# crafted to collapse url.path to a substring the call site's decision logic +# would match on, while scope["path"] is the real (unmatching) route. +# --------------------------------------------------------------------------- + +_BYPASS_HOSTS = [ + "localhost/?x=1", + "localhost:4000/?x=1", + "localhost/#test", + "localhost:4000/#test", +] + + +def _is_assistants(req): + return RouteChecks._is_assistants_api_request(req) + + +def _metadata_var_name(req): + from litellm.proxy.litellm_pre_call_utils import _get_metadata_variable_name + + return _get_metadata_variable_name(req) + + +def _vector_store_id_in_path(req): + from litellm.proxy.common_utils.http_parsing_utils import ( + _add_vector_store_id_from_path, + ) + + data: dict = {} + _add_vector_store_id_from_path(request_data=data, request=req) + return "vector_store_id" in data + + +# (label, scope_path, host_suffix_template, predicate, expected) — host_suffix_template +# receives the host_header via %s substitution. The predicate is invoked on a Request +# whose scope["path"] is scope_path and whose Host header is the formatted suffix. +# +# The MCP entries (well_known_mcp_bypass, pkce_token_suffix) call +# get_request_route directly rather than the surrounding production handler +# (MCPRequestHandler.process_mcp_request / _mcp_oauth_user_api_key_auth) — +# those handlers require an ASGI scope plus MCP state to invoke, and the call +# sites do nothing with the path except feed it to this helper. The helper- +# level assertion is the relevant signal. +_CALL_SITES = [ + ("assistants_classification", "/key/generate", "%s/thread", _is_assistants, False), + ( + "metadata_variable_name", + "/chat/completions", + "%s/thread", + _metadata_var_name, + "metadata", + ), + ( + "vector_store_id_extraction", + "/key/generate", + "%s/vector_stores/x/files", + _vector_store_id_in_path, + False, + ), + ( + "well_known_mcp_bypass", + "/mcp/tools/call", + "/.well-known/%s", + lambda r: get_request_route(r).startswith("/.well-known/"), + False, + ), + ( + "pkce_token_suffix", + "/mcp/server-id/token", + "%s", + lambda r: get_request_route(r).rstrip("/").lower().endswith("/token"), + True, + ), + ( + "spend_logs_v2_classification", + "/spend/logs", + "%s/spend/logs/v2", + lambda r: "/spend/logs/v2" in get_request_route(r), + False, + ), + ("health_route_echo", "/test", "%s", lambda r: get_request_route(r), "/test"), +] + + +@pytest.mark.parametrize("host_header", _BYPASS_HOSTS) +@pytest.mark.parametrize( + "label,scope_path,host_suffix_template,predicate,expected", + _CALL_SITES, + ids=[c[0] for c in _CALL_SITES], +) +def test_call_site_uses_scope_path( + label, scope_path, host_suffix_template, predicate, expected, host_header +): + """Each call site that previously read request.url.path must now make its + decision against scope["path"]. The Host header is crafted so url.path + would resolve to a value that flips the decision under the old code.""" + request = _create_request_with_host_header( + path=scope_path, host_header=host_suffix_template % host_header + ) + assert predicate(request) == expected From d98ada8c3f7ff6cad3c9643b965fea118dc5f188 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 25 May 2026 13:48:47 -0700 Subject: [PATCH 05/13] chore(ci): merge dev branch (#28657) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(dashboard): navbar hierarchy + Agent Platform notifications (#27543) * feat(dashboard): refine navbar zones and Agent Platform notice Restructure the admin navbar for production users: clear product vs community vs personal columns with vertical dividers, icon-only Slack/GitHub in a shared chip, and Docs/Blog typography aligned on an 8px rhythm. Add a notifications bell with popover linking to the LiteLLM Agent Platform repo and optional mark-as-read persistence. Promote the account control with initials avatar, single-line display name, and navDisplayName mapping for placeholder user ids (e.g. default_user_id). Co-authored-by: Cursor * fix(dashboard): address PR review — AntD buttons, public page guard, dedupe regex - Replace raw + ); }; diff --git a/ui/litellm-dashboard/src/components/Navbar/CommunityEngagementButtons/CommunityEngagementButtons.test.tsx b/ui/litellm-dashboard/src/components/Navbar/CommunityEngagementButtons/CommunityEngagementButtons.test.tsx index 6994def858b1..4f07f0e2daa2 100644 --- a/ui/litellm-dashboard/src/components/Navbar/CommunityEngagementButtons/CommunityEngagementButtons.test.tsx +++ b/ui/litellm-dashboard/src/components/Navbar/CommunityEngagementButtons/CommunityEngagementButtons.test.tsx @@ -29,14 +29,14 @@ describe("CommunityEngagementButtons", () => { expect(joinSlackLink).toHaveAttribute("rel", "noopener noreferrer"); }); - it("should render Star us on GitHub button with correct link", () => { + it("should render GitHub link with correct href", () => { renderWithProviders(); - const starOnGithubLink = screen.getByRole("link", { name: /star us on github/i }); - expect(starOnGithubLink).toBeInTheDocument(); - expect(starOnGithubLink).toHaveAttribute("href", "https://github.com/BerriAI/litellm"); - expect(starOnGithubLink).toHaveAttribute("target", "_blank"); - expect(starOnGithubLink).toHaveAttribute("rel", "noopener noreferrer"); + const githubLink = screen.getByRole("link", { name: /litellm on github/i }); + expect(githubLink).toBeInTheDocument(); + expect(githubLink).toHaveAttribute("href", "https://github.com/BerriAI/litellm"); + expect(githubLink).toHaveAttribute("target", "_blank"); + expect(githubLink).toHaveAttribute("rel", "noopener noreferrer"); }); it("should not render buttons when prompts are disabled", () => { @@ -45,6 +45,6 @@ describe("CommunityEngagementButtons", () => { renderWithProviders(); expect(screen.queryByRole("link", { name: /join slack/i })).not.toBeInTheDocument(); - expect(screen.queryByRole("link", { name: /star us on github/i })).not.toBeInTheDocument(); + expect(screen.queryByRole("link", { name: /litellm on github/i })).not.toBeInTheDocument(); }); }); diff --git a/ui/litellm-dashboard/src/components/Navbar/CommunityEngagementButtons/CommunityEngagementButtons.tsx b/ui/litellm-dashboard/src/components/Navbar/CommunityEngagementButtons/CommunityEngagementButtons.tsx index 649bcc0b589c..f6a43196a326 100644 --- a/ui/litellm-dashboard/src/components/Navbar/CommunityEngagementButtons/CommunityEngagementButtons.tsx +++ b/ui/litellm-dashboard/src/components/Navbar/CommunityEngagementButtons/CommunityEngagementButtons.tsx @@ -1,36 +1,45 @@ import { useDisableShowPrompts } from "@/app/(dashboard)/hooks/useDisableShowPrompts"; import { GithubOutlined, SlackOutlined } from "@ant-design/icons"; -import { Button } from "antd"; +import { Tooltip } from "antd"; import React from "react"; +const iconBtnClass = + "inline-flex h-9 w-9 shrink-0 items-center justify-center rounded-md border-0 bg-transparent text-gray-500 transition-colors hover:bg-gray-100 hover:text-gray-700 cursor-pointer"; + export const CommunityEngagementButtons: React.FC = () => { const disableShowPrompts = useDisableShowPrompts(); - // Hide buttons if prompts are disabled if (disableShowPrompts) { return null; } return ( - <> - - - +
+ + + + + + + + + + +
); }; diff --git a/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.test.tsx b/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.test.tsx new file mode 100644 index 000000000000..4ead085d9774 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.test.tsx @@ -0,0 +1,69 @@ +import { renderWithProviders, screen } from "../../../../tests/test-utils"; +import { NotificationsBell, AGENT_PLATFORM_URL } from "./NotificationsBell"; +import React from "react"; +import userEvent from "@testing-library/user-event"; + +describe("NotificationsBell", () => { + beforeEach(() => { + localStorage.clear(); + }); + + it("should open notifications with Agent Platform details and GitHub link", async () => { + const user = userEvent.setup(); + renderWithProviders(); + await user.click(screen.getByRole("button", { name: /^notifications$/i })); + expect(screen.getByText(/LiteLLM Agent Platform/i)).toBeInTheDocument(); + const githubBtn = screen.getByRole("link", { name: /^GitHub$/i }); + expect(githubBtn).toHaveAttribute("href", AGENT_PLATFORM_URL); + expect(githubBtn).toHaveAttribute("target", "_blank"); + expect(githubBtn).toHaveAttribute("rel", "noopener noreferrer"); + }); + + it("should offer mark as read when announcement is unread", async () => { + const user = userEvent.setup(); + renderWithProviders(); + await user.click(screen.getByRole("button", { name: /^notifications$/i })); + expect(screen.getByRole("button", { name: /^mark as read$/i })).toBeInTheDocument(); + }); + + it("should hide mark as read and persist after marking read", async () => { + const user = userEvent.setup(); + renderWithProviders(); + await user.click(screen.getByRole("button", { name: /^notifications$/i })); + await user.click(screen.getByRole("button", { name: /^mark as read$/i })); + expect(localStorage.getItem("litellmHideAgentPlatformBanner")).toBe("true"); + await user.click(screen.getByRole("button", { name: /^notifications$/i })); + expect(screen.queryByRole("button", { name: /^mark as read$/i })).not.toBeInTheDocument(); + }); + + it("should not show mark as read when previously dismissed", async () => { + localStorage.setItem("litellmHideAgentPlatformBanner", "true"); + const user = userEvent.setup(); + renderWithProviders(); + await user.click(screen.getByRole("button", { name: /^notifications$/i })); + expect(screen.queryByRole("button", { name: /^mark as read$/i })).not.toBeInTheDocument(); + }); + + it("should sync sibling instances when one is dismissed", async () => { + const user = userEvent.setup(); + renderWithProviders( + <> +
+ +
+
+ +
+ , + ); + + // Both bells start unread → both render the "Mark as read" affordance once opened. + const [bellA, bellB] = screen.getAllByRole("button", { name: /^notifications$/i }); + await user.click(bellA); + await user.click(screen.getByRole("button", { name: /^mark as read$/i })); + + // Dismissing in bell A must also clear bell B without a remount. + await user.click(bellB); + expect(screen.queryByRole("button", { name: /^mark as read$/i })).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.tsx b/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.tsx new file mode 100644 index 000000000000..a3b7678e1e7e --- /dev/null +++ b/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.tsx @@ -0,0 +1,59 @@ +"use client"; + +import { + HIDE_AGENT_PLATFORM_BANNER_KEY, + useHideAgentPlatformBanner, +} from "@/app/(dashboard)/hooks/useHideAgentPlatformBanner"; +import { emitLocalStorageChange, setLocalStorageItem } from "@/utils/localStorageUtils"; +import { BellOutlined } from "@ant-design/icons"; +import { Badge, Button, Popover, Typography } from "antd"; +import React, { useState } from "react"; + +export const AGENT_PLATFORM_URL = "https://github.com/BerriAI/litellm-agent-platform"; + +export const NotificationsBell: React.FC = () => { + const hidden = useHideAgentPlatformBanner(); + const hasUnread = !hidden; + const [open, setOpen] = useState(false); + + const markDismissed = () => { + setLocalStorageItem(HIDE_AGENT_PLATFORM_BANNER_KEY, "true"); + emitLocalStorageChange(HIDE_AGENT_PLATFORM_BANNER_KEY); + setOpen(false); + }; + + const content = ( +
+ + LiteLLM Agent Platform + + + Open-source agent infra — sandboxes, durable sessions, and workers on AWS Fargate. + +
+ + {hasUnread ? ( + + ) : null} +
+
+ ); + + return ( + + + + ); +}; diff --git a/ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.test.tsx b/ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.test.tsx index de853303c15c..31ddae317985 100644 --- a/ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.test.tsx +++ b/ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.test.tsx @@ -37,6 +37,8 @@ vi.mock("@/utils/localStorageUtils", () => ({ describe("UserDropdown", () => { const mockOnLogout = vi.fn(); + const getAccountTrigger = () => screen.getByRole("button", { name: /account menu/i }); + beforeEach(() => { vi.clearAllMocks(); mockUseAuthorizedImpl = () => ({ @@ -55,22 +57,23 @@ describe("UserDropdown", () => { it("should render", () => { renderWithProviders(); - expect(screen.getByRole("button")).toBeInTheDocument(); + expect(getAccountTrigger()).toBeInTheDocument(); }); - it("should display user button with User text", () => { + it("should surface initials and account menu affordance", () => { renderWithProviders(); - expect(screen.getByText("User")).toBeInTheDocument(); + expect(getAccountTrigger()).toBeInTheDocument(); + expect(screen.getByText("TE")).toBeInTheDocument(); }); it("should show user email when dropdown is opened", async () => { const user = userEvent.setup(); renderWithProviders(); - await user.click(screen.getByText("User")); + await user.click(getAccountTrigger()); await waitFor(() => { - expect(screen.getByText("test@example.com")).toBeInTheDocument(); + expect(screen.getAllByText("test@example.com").length).toBeGreaterThan(0); }); }); @@ -78,7 +81,7 @@ describe("UserDropdown", () => { const user = userEvent.setup(); renderWithProviders(); - await user.click(screen.getByText("User")); + await user.click(getAccountTrigger()); await waitFor(() => { expect(screen.getByText("test-user-id")).toBeInTheDocument(); @@ -89,10 +92,10 @@ describe("UserDropdown", () => { const user = userEvent.setup(); renderWithProviders(); - await user.click(screen.getByText("User")); + await user.click(getAccountTrigger()); await waitFor(() => { - expect(screen.getByText("Admin")).toBeInTheDocument(); + expect(screen.getAllByText("Admin").length).toBeGreaterThan(0); }); }); @@ -100,7 +103,7 @@ describe("UserDropdown", () => { const user = userEvent.setup(); renderWithProviders(); - await user.click(screen.getByText("User")); + await user.click(getAccountTrigger()); await waitFor(() => { expect(screen.getByText("Standard")).toBeInTheDocument(); @@ -118,7 +121,7 @@ describe("UserDropdown", () => { renderWithProviders(); - await user.click(screen.getByText("User")); + await user.click(getAccountTrigger()); await waitFor(() => { expect(screen.getByText("Premium")).toBeInTheDocument(); @@ -129,10 +132,10 @@ describe("UserDropdown", () => { const user = userEvent.setup(); renderWithProviders(); - await user.click(screen.getByText("User")); + await user.click(getAccountTrigger()); await waitFor(() => { - expect(screen.getByText("test@example.com")).toBeInTheDocument(); + expect(screen.getAllByText("test@example.com").length).toBeGreaterThan(0); }); await user.click(screen.getByText("Logout")); @@ -144,10 +147,10 @@ describe("UserDropdown", () => { const user = userEvent.setup(); renderWithProviders(); - await user.click(screen.getByText("User")); + await user.click(getAccountTrigger()); await waitFor(() => { - expect(screen.getByText("test@example.com")).toBeInTheDocument(); + expect(screen.getAllByText("test@example.com").length).toBeGreaterThan(0); }); const toggle = screen.getByLabelText("Toggle hide new feature indicators"); @@ -169,10 +172,10 @@ describe("UserDropdown", () => { renderWithProviders(); - await user.click(screen.getByText("User")); + await user.click(getAccountTrigger()); await waitFor(() => { - expect(screen.getByText("test@example.com")).toBeInTheDocument(); + expect(screen.getAllByText("test@example.com").length).toBeGreaterThan(0); }); const toggle = screen.getByLabelText("Toggle hide new feature indicators"); @@ -189,10 +192,10 @@ describe("UserDropdown", () => { const user = userEvent.setup(); renderWithProviders(); - await user.click(screen.getByText("User")); + await user.click(getAccountTrigger()); await waitFor(() => { - expect(screen.getByText("test@example.com")).toBeInTheDocument(); + expect(screen.getAllByText("test@example.com").length).toBeGreaterThan(0); }); const toggle = screen.getByLabelText("Toggle hide all prompts"); @@ -215,10 +218,10 @@ describe("UserDropdown", () => { renderWithProviders(); - await user.click(screen.getByText("User")); + await user.click(getAccountTrigger()); await waitFor(() => { - expect(screen.getByText("test@example.com")).toBeInTheDocument(); + expect(screen.getAllByText("test@example.com").length).toBeGreaterThan(0); }); const toggle = screen.getByLabelText("Toggle hide all prompts"); @@ -231,6 +234,17 @@ describe("UserDropdown", () => { expect(localStorageUtils.emitLocalStorageChange).toHaveBeenCalledWith("disableShowPrompts"); }); + it("should show Account in the trigger when user id is the default placeholder", () => { + mockUseAuthorizedImpl = () => ({ + userId: "default_user_id", + userEmail: null as any, + userRole: "Admin", + premiumUser: false, + }); + renderWithProviders(); + expect(screen.getByText("Account")).toBeInTheDocument(); + }); + it("should display dash when user email is not available", async () => { const user = userEvent.setup(); mockUseAuthorizedImpl = () => ({ @@ -242,7 +256,7 @@ describe("UserDropdown", () => { renderWithProviders(); - await user.click(screen.getByText("User")); + await user.click(getAccountTrigger()); await waitFor(() => { expect(screen.getByText("-")).toBeInTheDocument(); @@ -260,7 +274,7 @@ describe("UserDropdown", () => { renderWithProviders(); - await user.click(screen.getByText("User")); + await user.click(getAccountTrigger()); await waitFor(() => { const dashElements = screen.getAllByText("-"); @@ -277,10 +291,10 @@ describe("UserDropdown", () => { renderWithProviders(); - await user.click(screen.getByText("User")); + await user.click(getAccountTrigger()); await waitFor(() => { - expect(screen.getByText("test@example.com")).toBeInTheDocument(); + expect(screen.getAllByText("test@example.com").length).toBeGreaterThan(0); }); const toggle = screen.getByLabelText("Toggle hide new feature indicators"); diff --git a/ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.tsx b/ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.tsx index 6490cd32fa7c..64a2f1260ba4 100644 --- a/ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.tsx +++ b/ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.tsx @@ -9,6 +9,7 @@ import { removeLocalStorageItem, setLocalStorageItem, } from "@/utils/localStorageUtils"; +import { navAccountDisplayName } from "@/components/Navbar/navDisplayName"; import { CrownOutlined, DownOutlined, @@ -23,6 +24,39 @@ import React, { useEffect, useState } from "react"; const { Text } = Typography; +function hueFromString(seed: string): number { + let h = 0; + for (let i = 0; i < seed.length; i += 1) { + h = seed.charCodeAt(i) + ((h << 5) - h); + } + return Math.abs(h) % 360; +} + +function initialsFromIdentity(email: string | null, userId: string | null): string { + const local = email?.split("@")[0]?.trim(); + if (local) { + const parts = local + .replace(/[^a-zA-Z0-9]+/g, " ") + .trim() + .split(/\s+/) + .filter(Boolean); + if (parts.length >= 2) { + return `${parts[0]!.charAt(0)}${parts[1]!.charAt(0)}`.toUpperCase(); + } + if (parts.length === 1) { + const p = parts[0]!; + return p.length >= 2 ? p.slice(0, 2).toUpperCase() : `${p.charAt(0)}`.toUpperCase(); + } + } + if (userId && userId.length >= 2) { + return userId.slice(0, 2).toUpperCase(); + } + if (userId && userId.length === 1) { + return `${userId.toUpperCase()}•`; + } + return "?"; +} + interface UserDropdownProps { onLogout: () => void; } @@ -61,19 +95,12 @@ const UserDropdown: React.FC = ({ onLogout }) => { {userEmail || "-"} {premiumUser ? ( - } - color="gold" - > + } color="gold"> Premium ) : ( - } - > - Standard - + }>Standard )} @@ -83,12 +110,7 @@ const UserDropdown: React.FC = ({ onLogout }) => { User ID - + {userId || "-"} @@ -189,13 +211,17 @@ const UserDropdown: React.FC = ({ onLogout }) => { ); + const seed = userEmail || userId || "user"; + const initials = initialsFromIdentity(userEmail, userId); + const hue = hueFromString(seed); + const displayName = navAccountDisplayName(userEmail, userId); + return ( ( -
+
{renderUserInfoSection()} {React.cloneElement(menu as React.ReactElement, { @@ -204,12 +230,23 @@ const UserDropdown: React.FC = ({ onLogout }) => {
)} > - ); diff --git a/ui/litellm-dashboard/src/components/Navbar/navDisplayName.test.ts b/ui/litellm-dashboard/src/components/Navbar/navDisplayName.test.ts new file mode 100644 index 000000000000..96e768c5d315 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Navbar/navDisplayName.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from "vitest"; +import { navAccountDisplayName } from "./navDisplayName"; + +describe("navAccountDisplayName", () => { + it("should prefer email when present", () => { + expect(navAccountDisplayName("x@y.com", "ignored")).toBe("x@y.com"); + }); + + it("should map default_user_id placeholder to Account", () => { + expect(navAccountDisplayName(null, "default_user_id")).toBe("Account"); + expect(navAccountDisplayName(null, "DEFAULT_USER_ID")).toBe("Account"); + }); + + it("should show a sensible token when user id is non-placeholder", () => { + expect(navAccountDisplayName(null, "user-uuid-123")).toBe("user-uuid-123"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/Navbar/navDisplayName.ts b/ui/litellm-dashboard/src/components/Navbar/navDisplayName.ts new file mode 100644 index 000000000000..d6f51dc37a83 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Navbar/navDisplayName.ts @@ -0,0 +1,15 @@ +/** Primary label for the navbar account control — avoids raw placeholder JWT/user IDs in the UI. */ +export function navAccountDisplayName(userEmail: string | null, userId: string | null): string { + const email = userEmail?.trim(); + if (email) { + return email; + } + const id = userId?.trim(); + if (!id) { + return "Account"; + } + if (/^default[_\s-]?user[_\s-]?id$/i.test(id)) { + return "Account"; + } + return id; +} diff --git a/ui/litellm-dashboard/src/components/Navbar/navProductLinkClass.ts b/ui/litellm-dashboard/src/components/Navbar/navProductLinkClass.ts new file mode 100644 index 000000000000..ca4b2e5d1f39 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Navbar/navProductLinkClass.ts @@ -0,0 +1,3 @@ +/** Shared styling for Docs / Blog in the top nav (product navigation zone). */ +export const NAV_PRODUCT_LINK_CLASS = + "inline-flex h-9 shrink-0 items-center justify-center gap-1 rounded-md px-2 text-sm font-medium leading-none text-gray-800 transition-colors hover:bg-gray-100 hover:text-gray-950"; diff --git a/ui/litellm-dashboard/src/components/navbar.test.tsx b/ui/litellm-dashboard/src/components/navbar.test.tsx index 2e122164969b..274e81db527f 100644 --- a/ui/litellm-dashboard/src/components/navbar.test.tsx +++ b/ui/litellm-dashboard/src/components/navbar.test.tsx @@ -30,6 +30,7 @@ const mockUserDropdownData = vi.hoisted(() => ({ vi.mock("./Navbar/UserDropdown/UserDropdown", async (importOriginal) => { const React = await import("react"); const { useState } = React; + const { Button } = await import("antd"); const localStorageUtils = await import("@/utils/localStorageUtils"); return { default: function MockUserDropdown({ onLogout }: { onLogout: () => void }) { @@ -37,9 +38,9 @@ vi.mock("./Navbar/UserDropdown/UserDropdown", async (importOriginal) => { const [open, setOpen] = useState(false); return (
- + {open && (
{userId} @@ -136,30 +137,25 @@ Object.defineProperty(window, "location", { describe("Navbar", () => { const defaultProps = { - userID: "test-user", - userEmail: "test@example.com", - userRole: "Admin", - premiumUser: false, proxySettings: {}, setProxySettings: vi.fn(), accessToken: "test-token", isPublicPage: false, - isDarkMode: false, - toggleDarkMode: vi.fn(), }; it("should render without crashing", () => { renderWithProviders(); + expect(screen.getByRole("button", { name: /^notifications$/i })).toBeInTheDocument(); expect(screen.getByText("Docs")).toBeInTheDocument(); - expect(screen.getByText("User")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /open account menu/i })).toBeInTheDocument(); }); it("should display user information in dropdown", async () => { const user = userEvent.setup(); renderWithProviders(); - await user.click(screen.getByText("User")); + await user.click(screen.getByRole("button", { name: /open account menu/i })); await waitFor(() => { expect(screen.getByText("test-user")).toBeInTheDocument(); @@ -198,7 +194,7 @@ describe("Navbar", () => { }); renderWithProviders(); - await user.click(screen.getByText("User")); + await user.click(screen.getByRole("button", { name: /open account menu/i })); await waitFor(() => { expect(screen.getByText("Premium")).toBeInTheDocument(); @@ -247,11 +243,12 @@ describe("Navbar", () => { mockUseThemeImpl = () => ({ logoUrl: null }); }); - it("should hide user dropdown on public pages", () => { + it("should hide user dropdown and notifications on public pages", () => { const publicPageProps = { ...defaultProps, isPublicPage: true }; renderWithProviders(); - expect(screen.queryByText("User")).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: /open account menu/i })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: /^notifications$/i })).not.toBeInTheDocument(); }); it("should handle hide new features toggle", async () => { @@ -265,7 +262,7 @@ describe("Navbar", () => { renderWithProviders(); - await user.click(screen.getByText("User")); + await user.click(screen.getByRole("button", { name: /open account menu/i })); await waitFor(() => { expect(screen.getByText("test-user")).toBeInTheDocument(); @@ -290,7 +287,7 @@ describe("Navbar", () => { renderWithProviders(); - await user.click(screen.getByText("User")); + await user.click(screen.getByRole("button", { name: /open account menu/i })); await waitFor(() => { expect(screen.getByText("test-user")).toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/components/navbar.tsx b/ui/litellm-dashboard/src/components/navbar.tsx index 055038b6d881..e5a1490788c0 100644 --- a/ui/litellm-dashboard/src/components/navbar.tsx +++ b/ui/litellm-dashboard/src/components/navbar.tsx @@ -1,47 +1,39 @@ import { useHealthReadinessDetails } from "@/app/(dashboard)/hooks/healthReadiness/useHealthReadinessDetails"; import { useDisableBouncingIcon } from "@/app/(dashboard)/hooks/useDisableBouncingIcon"; +import { useDisableShowPrompts } from "@/app/(dashboard)/hooks/useDisableShowPrompts"; +import { useWorker } from "@/hooks/useWorker"; import { getProxyBaseUrl } from "@/components/networking"; import { useTheme } from "@/contexts/ThemeContext"; import { clearTokenCookies } from "@/utils/cookieUtils"; import { clearStoredReturnUrl } from "@/utils/returnUrlUtils"; import { fetchProxySettings } from "@/utils/proxyUtils"; -import { MenuFoldOutlined, MenuUnfoldOutlined, MoonOutlined, SunOutlined } from "@ant-design/icons"; -import { Button, Switch, Tag } from "antd"; +import { DownOutlined, MenuFoldOutlined, MenuUnfoldOutlined } from "@ant-design/icons"; +import { Tag } from "antd"; import Link from "next/link"; import React, { useEffect, useState } from "react"; import { BlogDropdown } from "./Navbar/BlogDropdown/BlogDropdown"; import { CommunityEngagementButtons } from "./Navbar/CommunityEngagementButtons/CommunityEngagementButtons"; +import { NAV_PRODUCT_LINK_CLASS } from "./Navbar/navProductLinkClass"; +import { NotificationsBell } from "./Navbar/NotificationsBell/NotificationsBell"; import UserDropdown from "./Navbar/UserDropdown/UserDropdown"; import WorkerDropdown from "./Navbar/WorkerDropdown/WorkerDropdown"; interface NavbarProps { - userID: string | null; - userEmail: string | null; - userRole: string | null; - premiumUser: boolean; proxySettings: any; setProxySettings: React.Dispatch>; accessToken: string | null; isPublicPage: boolean; sidebarCollapsed?: boolean; onToggleSidebar?: () => void; - isDarkMode: boolean; - toggleDarkMode: () => void; } const Navbar: React.FC = ({ - userID, - userEmail, - userRole, - premiumUser, proxySettings, setProxySettings, accessToken, isPublicPage = false, sidebarCollapsed = false, onToggleSidebar, - isDarkMode, - toggleDarkMode, }) => { const baseUrl = getProxyBaseUrl(); const [logoutUrl, setLogoutUrl] = useState(""); @@ -49,8 +41,10 @@ const Navbar: React.FC = ({ const { data: healthData } = useHealthReadinessDetails(accessToken); const version = healthData?.litellm_version; const disableBouncingIcon = useDisableBouncingIcon(); + const hideCommunityLinks = useDisableShowPrompts(); + const { isControlPlane, selectedWorker } = useWorker(); + const showWorkerSwitch = isControlPlane && selectedWorker !== null; - // Simple logo URL: use custom logo if available, otherwise default const imageUrl = logoUrl || `${baseUrl}/get_image`; useEffect(() => { @@ -87,14 +81,14 @@ const Navbar: React.FC = ({ }; return ( -