Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
from starlette.types import Scope

from litellm._logging import verbose_logger
from litellm.constants import DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL
from litellm.proxy._types import (
LiteLLM_TeamTable,
ProxyException,
Expand All @@ -17,6 +16,7 @@
)
from litellm.proxy.auth.ip_address_utils import IPAddressUtils
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.user_api_key_cache import get_management_object_ttl
from litellm.repositories.table_repositories import (
AgentsRepository,
MCPServerRepository,
Expand Down Expand Up @@ -1414,7 +1414,7 @@ async def _get_agent_object_permission(
key=cache_key,
value=object_permission_id
or MCPRequestHandler._AGENT_NO_PERMISSION_SENTINEL,
ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL,
ttl=get_management_object_ttl(user_api_key_cache),
)
if not object_permission_id:
return None
Expand Down
7 changes: 3 additions & 4 deletions litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@
)
from litellm.proxy.auth.ip_address_utils import IPAddressUtils
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper
from litellm.proxy.common_utils.user_api_key_cache import get_management_object_ttl
from litellm.proxy.utils import ProxyLogging
from litellm.repositories.table_repositories import MCPServerRepository
from litellm.types.llms.custom_http import httpxSpecialProvider
Expand Down Expand Up @@ -1498,7 +1499,6 @@ async def resolve_toolset_tool_permissions(
Redis-backed ``DualCache`` in production) so that cache entries are
shared across workers and cold-cache DB hits are minimised.
"""
from litellm.constants import DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL
from litellm.proxy._experimental.mcp_server.toolset_db import list_mcp_toolsets
from litellm.proxy.proxy_server import prisma_client, user_api_key_cache

Expand All @@ -1524,7 +1524,7 @@ async def resolve_toolset_tool_permissions(
await user_api_key_cache.async_set_cache(
key=cache_key,
value=tool_permissions,
ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL,
ttl=get_management_object_ttl(user_api_key_cache),
)
return tool_permissions
except Exception as e:
Expand Down Expand Up @@ -1581,7 +1581,6 @@ async def get_toolset_by_name_cached(
deployments. On a cache hit we reconstruct the ``MCPToolset`` Pydantic object
so callers can always use attribute access (e.g. ``toolset.toolset_id``).
"""
from litellm.constants import DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL
from litellm.proxy.proxy_server import user_api_key_cache
from litellm.types.mcp_server.mcp_toolset import MCPToolset

Expand Down Expand Up @@ -1609,7 +1608,7 @@ async def get_toolset_by_name_cached(
if toolset is not None
else "__not_found__"
),
ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL,
ttl=get_management_object_ttl(user_api_key_cache),
)
return toolset

Expand Down
18 changes: 10 additions & 8 deletions litellm/proxy/auth/auth_checks.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
CLI_SESSION_KEY_PREFIX,
DEFAULT_ACCESS_GROUP_CACHE_TTL,
DEFAULT_IN_MEMORY_TTL,
DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL,
DEFAULT_MAX_RECURSE_DEPTH,
EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE,
)
Expand Down Expand Up @@ -67,7 +66,10 @@
_safe_get_request_headers,
_safe_get_request_query_params,
)
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.common_utils.user_api_key_cache import (
UserApiKeyCache,
get_management_object_ttl,
)
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
from litellm.proxy.guardrails.tool_name_extraction import (
TOOL_CAPABLE_CALL_TYPES,
Expand Down Expand Up @@ -994,7 +996,7 @@ async def get_default_end_user_budget(
key=cache_key,
value=_budget_obj,
model_type=LiteLLM_BudgetTable,
ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL,
ttl=get_management_object_ttl(user_api_key_cache),
)

return _budget_obj
Expand Down Expand Up @@ -1050,7 +1052,7 @@ async def get_team_member_default_budget(
await user_api_key_cache.async_set_cache(
key=cache_key,
value=budget_record.dict(),
ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL,
ttl=get_management_object_ttl(user_api_key_cache),
)

return LiteLLM_BudgetTable(**budget_record.dict())
Expand Down Expand Up @@ -1761,7 +1763,7 @@ async def get_user_object(
key=user_id,
value=_response,
model_type=LiteLLM_UserTable,
ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL,
ttl=get_management_object_ttl(user_api_key_cache),
)

# save to db access time
Expand Down Expand Up @@ -1796,7 +1798,7 @@ async def _cache_management_object(
key=key,
value=value,
model_type=model_type,
ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL,
ttl=get_management_object_ttl(user_api_key_cache),
)


Expand Down Expand Up @@ -2700,7 +2702,7 @@ async def get_object_permission(
key=key,
value=_perm_obj,
model_type=LiteLLM_ObjectPermissionTable,
ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL,
ttl=get_management_object_ttl(user_api_key_cache),
)

return _perm_obj
Expand Down Expand Up @@ -2765,7 +2767,7 @@ async def get_managed_vector_store_rows_by_uuids(
key=key,
value=cached_obj,
model_type=LiteLLM_ManagedVectorStoresTable,
ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL,
ttl=get_management_object_ttl(user_api_key_cache),
)
result.append(cached_obj)

Expand Down
10 changes: 6 additions & 4 deletions litellm/proxy/auth/handle_jwt.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
from jwt.api_jwk import PyJWK

from litellm._logging import verbose_proxy_logger
from litellm.constants import DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL
from litellm.litellm_core_utils.dot_notation_indexing import get_nested_value
from litellm.llms.custom_httpx.httpx_handler import HTTPHandler
from litellm.proxy._types import (
Expand All @@ -48,7 +47,10 @@
)
from litellm.proxy.auth.auth_checks import can_team_access_model
from litellm.proxy.auth.route_checks import RouteChecks
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.common_utils.user_api_key_cache import (
UserApiKeyCache,
get_management_object_ttl,
)
from litellm.proxy.utils import PrismaClient, ProxyLogging
from litellm.repositories.user_repository import UserRepository

Expand Down Expand Up @@ -1824,7 +1826,7 @@ async def sync_user_role_and_teams(
key=user_object.user_id,
value=user_object,
model_type=LiteLLM_UserTable,
ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL,
ttl=get_management_object_ttl(user_api_key_cache),
)

# Sync team memberships
Expand All @@ -1848,7 +1850,7 @@ async def sync_user_role_and_teams(
key=user_object.user_id,
value=user_object,
model_type=LiteLLM_UserTable,
ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL,
ttl=get_management_object_ttl(user_api_key_cache),
)
return None

Expand Down
15 changes: 15 additions & 0 deletions litellm/proxy/common_utils/user_api_key_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

from litellm._logging import verbose_proxy_logger
from litellm.caching.dual_cache import DualCache
from litellm.constants import DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL
from litellm.proxy.common_utils.cache_pydantic_utils import CacheCodec

T = TypeVar("T", bound=BaseModel)
Expand Down Expand Up @@ -160,3 +161,17 @@ async def async_set_cache_pipeline( # type: ignore[override]
return await super().async_set_cache_pipeline(
cache_list=normalized, local_only=local_only, **kwargs
)


def get_management_object_ttl(cache: DualCache) -> float:
"""
In-memory TTL for management-object cache writes (keys, teams, users, budgets, ...).

Honors ``general_settings.user_api_key_cache_ttl``, which ``proxy_server``
propagates onto ``default_in_memory_ttl`` at startup, and falls back to
``DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL`` when no default is configured.
"""
configured: Optional[float] = getattr(cache, "default_in_memory_ttl", None)
if configured is not None:
return configured
return DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL
19 changes: 10 additions & 9 deletions litellm/proxy/proxy_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -316,7 +316,10 @@ def generate_feedback_box():
from litellm.proxy.common_utils.reset_budget_job import ResetBudgetJob
from litellm.proxy.common_utils.swagger_utils import ERROR_RESPONSES
from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.common_utils.user_api_key_cache import (
UserApiKeyCache,
get_management_object_ttl,
)
from litellm.proxy.container_endpoints.endpoints import router as container_router
from litellm.proxy.credential_endpoints.endpoints import router as credential_router
from litellm.proxy.db.db_transaction_queue.spend_log_cleanup import SpendLogCleanup
Expand Down Expand Up @@ -3105,7 +3108,7 @@ async def _update_tag_cache():
asyncio.create_task(
user_api_key_cache.async_set_cache_pipeline(
cache_list=values_to_update_in_cache,
ttl=60,
ttl=get_management_object_ttl(user_api_key_cache),
litellm_parent_otel_span=parent_otel_span,
)
)
Expand Down Expand Up @@ -17043,17 +17046,15 @@ async def _resolve_mcp_csv_tokens(
async def _is_mcp_access_group_cached(name: str) -> bool:
"""Return True if *name* is a known MCP access group tag.

Positive results are cached for ``DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL``
seconds. Negative results are cached for a short
Positive results are cached for the configured management-object TTL
(``get_management_object_ttl(user_api_key_cache)``). Negative results are
cached for a short
``DEFAULT_MCP_ACCESS_GROUP_NEGATIVE_CACHE_TTL`` window so unauthenticated
callers cannot force a fresh DB lookup per request for unknown names, while
bounding staleness so a transient DB error (which surfaces as an empty
list) cannot hide a real group for long.
"""
from litellm.constants import (
DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL,
DEFAULT_MCP_ACCESS_GROUP_NEGATIVE_CACHE_TTL,
)
from litellm.constants import DEFAULT_MCP_ACCESS_GROUP_NEGATIVE_CACHE_TTL
from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
MCPRequestHandler,
)
Expand All @@ -17067,7 +17068,7 @@ async def _is_mcp_access_group_cached(name: str) -> bool:
key=cache_key,
value=result,
ttl=(
DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL
get_management_object_ttl(user_api_key_cache)
if result
else DEFAULT_MCP_ACCESS_GROUP_NEGATIVE_CACHE_TTL
),
Expand Down
56 changes: 56 additions & 0 deletions tests/test_litellm/proxy/auth/test_auth_checks.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
)
from litellm.proxy.auth.auth_checks import (
ExperimentalUIJWTToken,
_cache_management_object,
_can_object_call_model,
_can_object_call_vector_stores,
_check_end_user_budget,
Expand All @@ -48,7 +49,10 @@
get_user_object,
vector_store_access_check,
)
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.constants import DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.utils import get_utc_datetime


Expand Down Expand Up @@ -3934,3 +3938,55 @@ async def test_virtual_key_max_budget_not_exceeded_does_not_raise():
valid_token=valid_token,
proxy_logging_obj=proxy_logging_obj,
)


class _TTLCapturingInMemoryCache(InMemoryCache):
"""Records the ``ttl`` DualCache forwards into the in-memory layer."""

def __init__(self) -> None:
super().__init__()
self.last_ttl = None

def set_cache(self, key, value, **kwargs): # type: ignore[override]
self.last_ttl = kwargs.get("ttl")
super().set_cache(key, value, **kwargs)


class TestManagementObjectTTLHonored:
"""
Regression for LIT-3338. ``_cache_management_object`` is the central writer on
the reported ``get_key_object -> _cache_key_object -> _cache_management_object``
path. It must cache for the configured ``user_api_key_cache_ttl`` (propagated to
``default_in_memory_ttl``) rather than the hardcoded 60s management default.
"""

@pytest.mark.asyncio
async def test_uses_configured_user_api_key_cache_ttl(self):
mem = _TTLCapturingInMemoryCache()
cache = UserApiKeyCache(in_memory_cache=mem, default_in_memory_ttl=300)

await _cache_management_object(
key="team_id:lit-3338",
value=UserAPIKeyAuth(token="hash-lit-3338"),
user_api_key_cache=cache,
proxy_logging_obj=None,
model_type=UserAPIKeyAuth,
)

assert mem.last_ttl == 300

@pytest.mark.asyncio
async def test_falls_back_to_management_default_when_unconfigured(self):
mem = _TTLCapturingInMemoryCache()
cache = UserApiKeyCache(in_memory_cache=mem)
assert cache.default_in_memory_ttl is None

await _cache_management_object(
key="team_id:lit-3338-default",
value=UserAPIKeyAuth(token="hash-default"),
user_api_key_cache=cache,
proxy_logging_obj=None,
model_type=UserAPIKeyAuth,
)

assert mem.last_ttl == DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL
Loading
Loading