Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion tests/llm_translation/reasoning_effort_grid/grid_spec.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
from dataclasses import dataclass, field
from typing import Dict, FrozenSet, List, Optional, Tuple


OMIT = object()


Expand All @@ -22,6 +21,7 @@ class ModelEntry:
extra_params: Tuple[Tuple[str, str], ...] = field(default_factory=tuple)
required_env: FrozenSet[str] = field(default_factory=frozenset)
caps: FrozenSet[str] = field(default_factory=frozenset)
unavailable_error: Optional[str] = None
bedrock_effort_ceiling: Optional[str] = None

def params(self) -> Dict[str, str]:
Expand Down Expand Up @@ -234,6 +234,7 @@ def expected(model: ModelEntry, effort: str) -> CellExpectation:
extra_params=(("aws_region_name", "us-east-1"),),
required_env=_BEDROCK_REQ,
caps=_CAPS_OPUS_4_7,
unavailable_error="is not available for this account",
),
ModelEntry(
alias="bedrock-claude-opus-4-6",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
all_cells,
)


_PROMPT_MESSAGES: List[Dict[str, str]] = [
{"role": "user", "content": "Step by step, calculate 47 * 53. Show your work."}
]
Expand Down Expand Up @@ -133,6 +132,12 @@ def _classify_status(exc: Exception) -> int:
return 500


def _model_unavailable(model: ModelEntry, exc: Optional[Exception]) -> bool:
if not model.unavailable_error or exc is None:
return False
return model.unavailable_error in str(exc)


async def _call_chat(model: ModelEntry, effort: str) -> Tuple[int, Optional[Exception]]:
kwargs = _build_completion_kwargs(model, effort)
try:
Expand Down Expand Up @@ -173,6 +178,9 @@ async def test_reasoning_effort_grid(
else:
status, exc = await _call_chat(model, effort)

if _model_unavailable(model, exc):
pytest.skip(f"{model.alias}: {model.unavailable_error}")

record = wire_capture.latest()
body = record["body"] if record else None
if route_name == "bedrock_converse" and isinstance(body, str):
Expand Down Expand Up @@ -205,3 +213,30 @@ def test_grid_route_coverage() -> None:
"bedrock_invoke_chat",
"bedrock_invoke_messages",
}


def test_model_unavailable_tolerates_only_the_declared_error() -> None:
gated = ModelEntry(
alias="bedrock-claude-opus-4-7",
model="bedrock/converse/us.anthropic.claude-opus-4-7",
mode="adaptive",
unavailable_error="is not available for this account",
)
entitlement_error = Exception(
"litellm.APIConnectionError: BedrockException - "
'{"message":"anthropic.claude-opus-4-7 is not available for this account."}'
)

assert _model_unavailable(gated, entitlement_error) is True
assert (
_model_unavailable(gated, Exception("ThrottlingException: rate exceeded"))
is False
)
assert _model_unavailable(gated, None) is False

ungated = ModelEntry(
alias="bedrock-claude-opus-4-6",
model="bedrock/converse/us.anthropic.claude-opus-4-6-v1",
mode="adaptive",
)
assert _model_unavailable(ungated, entitlement_error) is False
34 changes: 32 additions & 2 deletions tests/logging_callback_tests/test_log_db_redis_services.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
import os
import sys


sys.path.insert(0, os.path.abspath("../.."))

import asyncio
Expand Down Expand Up @@ -59,7 +58,38 @@ async def test_log_db_metrics_success():
assert isinstance(call_args["duration"], float)
assert isinstance(call_args["start_time"], datetime)
assert isinstance(call_args["end_time"], datetime)
assert "function_name" in call_args["event_metadata"]
assert call_args["event_metadata"] is None


@pytest.mark.asyncio
async def test_log_db_metrics_event_metadata_is_safe():
"""event_metadata must surface only the table name, never the raw
kwargs/args which carry live clients (Prisma, OTel spans) and secrets.

Regression guard for #28909: a previous version dumped function_kwargs and
function_args onto the span.
"""
with patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging:
mock_proxy_logging.service_logging_obj.async_service_success_hook = AsyncMock()

@log_db_metrics
async def db_call(**kwargs):
return "success"

await db_call(
parent_otel_span="test_span",
table_name="LiteLLM_SpendLogs",
token="sk-secret-should-not-leak",
prisma_client=object(),
)
await asyncio.sleep(0)

call_args = (
mock_proxy_logging.service_logging_obj.async_service_success_hook.call_args[
1
]
)
assert call_args["event_metadata"] == {"table_name": "LiteLLM_SpendLogs"}


@pytest.mark.asyncio
Expand Down
Loading