Skip to content
Merged
15 changes: 7 additions & 8 deletions litellm/proxy/agent_endpoints/a2a_endpoints.py
Original file line number Diff line number Diff line change
Expand Up @@ -168,14 +168,13 @@ def _jsonrpc_error(
)


def _get_agent(agent_id: str):
async def _get_agent(agent_id: str) -> "AgentResponse | None":
"""Look up an agent by ID or name. Returns None if not found."""
from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry
from litellm.proxy.common_utils.registry_read_through import (
get_agent_with_read_through,
)

agent = global_agent_registry.get_agent_by_id(agent_id=agent_id)
if agent is None:
agent = global_agent_registry.get_agent_by_name(agent_name=agent_id)
return agent
return await get_agent_with_read_through(agent_id)


def _enforce_inbound_trace_id(agent: "AgentResponse", request: Request) -> None:
Expand Down Expand Up @@ -559,7 +558,7 @@ async def get_agent_card(
)

try:
agent: Final = _get_agent(agent_id)
agent: Final = await _get_agent(agent_id)
if agent is None:
raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")

Expand Down Expand Up @@ -673,7 +672,7 @@ async def invoke_agent_a2a(
params.pop(key)

# Find the agent
agent: Final = _get_agent(agent_id)
agent: Final = await _get_agent(agent_id)
if agent is None:
return _jsonrpc_error(request_id, -32000, f"Agent '{agent_id}' not found", 404)

Expand Down
10 changes: 6 additions & 4 deletions litellm/proxy/agent_endpoints/a2a_routing.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,12 @@ async def route_a2a_agent_request(
Returns None if not an A2A request (allows normal routing to continue).
"""
# Import here to avoid circular imports
from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry
from litellm.proxy.agent_endpoints.auth.agent_permission_handler import (
AgentRequestHandler,
)
from litellm.proxy.common_utils.registry_read_through import (
get_agent_with_read_through,
)
from litellm.proxy.route_llm_request import (
ROUTE_ENDPOINT_MAPPING,
ProxyModelNotFoundError,
Expand All @@ -44,11 +46,11 @@ async def route_a2a_agent_request(
agent_name: Final = model_name[4:]

# Look up agent in registry
agent: Final = global_agent_registry.get_agent_by_name(agent_name)
agent: Final = await get_agent_with_read_through(agent_name)
if agent is None:
verbose_proxy_logger.error("[A2A] Agent '%s' not found in registry", agent_name)
route_name = ROUTE_ENDPOINT_MAPPING.get(route_type, route_type)
raise ProxyModelNotFoundError(route=route_name, model_name=model_name)
raise ProxyModelNotFoundError(route=route_name, model_name=model_name, retryable_with_model_read_through=False)

# Verify the caller is permitted to use this agent (admins bypass the check)
is_admin: Final = user_api_key_dict is not None and (
Expand All @@ -70,7 +72,7 @@ async def route_a2a_agent_request(
if not agent.agent_card_params or "url" not in agent.agent_card_params:
verbose_proxy_logger.error("[A2A] Agent '%s' has no URL configured", agent_name)
route_name = ROUTE_ENDPOINT_MAPPING.get(route_type, route_type)
raise ProxyModelNotFoundError(route=route_name, model_name=model_name)
raise ProxyModelNotFoundError(route=route_name, model_name=model_name, retryable_with_model_read_through=False)

# Inject API base and route to litellm
data["api_base"] = agent.agent_card_params["url"]
Expand Down
1 change: 1 addition & 0 deletions litellm/proxy/agent_endpoints/agent_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -600,3 +600,4 @@ def get_agent_by_name(self, agent_name: str) -> AgentResponse | None:


global_agent_registry: Final = AgentRegistry()
AGENT_RECONCILE_LOCK: Final = asyncio.Lock()
218 changes: 218 additions & 0 deletions litellm/proxy/common_utils/registry_read_through.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,218 @@
"""Read-through recovery for in-memory registries in multi-replica deployments.

A management write (POST /model/new, /guardrails, /v1/agents) lands on one
replica and reaches Postgres, but sibling replicas only refresh their in-memory
registries on the periodic config reload, so a request using the new object
immediately can land on a sibling that has never heard of it and fail 400/404.
On a registry miss, callers here fetch the missing row from the DB and load it
into the local registry before giving up. A short negative-result TTL per key
plus a global resync budget per window bound the DB load from lookups of
genuinely unknown names.
"""

import asyncio
import time
from collections.abc import Awaitable, Callable
from typing import TYPE_CHECKING, Final

from litellm._logging import verbose_proxy_logger
from litellm.caching.in_memory_cache import InMemoryCache

if TYPE_CHECKING:
from prisma.types import (
LiteLLM_AgentsTableInclude,
LiteLLM_AgentsTableWhereUniqueInput,
LiteLLM_ProxyModelTableWhereInput,
)

from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.types.agents import AgentResponse

READ_THROUGH_MISS_TTL_SECONDS: Final = 2.0
READ_THROUGH_RESYNC_WINDOW_SECONDS: Final = 5.0
READ_THROUGH_MAX_RESYNCS_PER_WINDOW: Final = 20


class RegistryReadThrough:
__slots__ = (
"_lock",
"_max_resyncs_per_window",
"_miss_ttl_seconds",
"_recent_misses",
"_resync",
"_resync_window_seconds",
"_window_resyncs",
"_window_started_at",
)

def __init__(
self,
resync: Callable[[str], Awaitable[bool]],
miss_ttl_seconds: float = READ_THROUGH_MISS_TTL_SECONDS,
max_resyncs_per_window: int = READ_THROUGH_MAX_RESYNCS_PER_WINDOW,
resync_window_seconds: float = READ_THROUGH_RESYNC_WINDOW_SECONDS,
) -> None:
self._resync = resync
self._miss_ttl_seconds = miss_ttl_seconds
self._max_resyncs_per_window = max_resyncs_per_window
self._resync_window_seconds = resync_window_seconds
self._lock = asyncio.Lock()
self._recent_misses = InMemoryCache(max_size_in_memory=1000)
self._window_started_at = float("-inf")
self._window_resyncs = 0

def _consume_resync_budget(self) -> bool:
now: Final = time.monotonic()
if now - self._window_started_at >= self._resync_window_seconds:
self._window_started_at = now
self._window_resyncs = 0
if self._window_resyncs >= self._max_resyncs_per_window:
return False
self._window_resyncs += 1
return True

async def attempt(self, key: str) -> bool:
if self._recent_misses.get_cache(key) is not None:
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
return False
async with self._lock:
if self._recent_misses.get_cache(key) is not None:
return False
if not self._consume_resync_budget():
verbose_proxy_logger.warning(
"registry read-through for %r skipped: resync budget of %s per %ss exhausted",
key,
self._max_resyncs_per_window,
self._resync_window_seconds,
)
return False
try:
found: Final = await self._resync(key)
except Exception as e: # noqa: BLE001 # a failed read-through must surface the original miss error, not a 500
verbose_proxy_logger.warning("registry read-through for %r failed: %s", key, e)
return False
if not found:
self._recent_misses.set_cache(key, True, ttl=self._miss_ttl_seconds)
return found


def _db_backed_registries_enabled(object_type: str) -> bool:
from litellm.proxy import proxy_server

if proxy_server.prisma_client is None or proxy_server.store_model_in_db is not True:
return False
return proxy_server.should_load_db_object(object_type=object_type)


async def _resync_model_deployments(model_name: str) -> bool:
from litellm.proxy import proxy_server
from litellm.repositories.model_repository import ModelRepository
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved

if not _db_backed_registries_enabled("models"):
return False
prisma_client: Final = proxy_server.prisma_client
assert prisma_client is not None
table: Final = ModelRepository(prisma_client).table
name_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_name": model_name}
id_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_id": model_name}
rows: Final = await table.find_many(where=name_filter) or await table.find_many(where=id_filter)
if not rows:
return False
router: Final = proxy_server.llm_router
if router is None:
await proxy_server.proxy_config.add_deployment(
prisma_client=prisma_client, proxy_logging_obj=proxy_server.proxy_logging_obj
)
return proxy_server.llm_router is not None
async with proxy_server.MODEL_RECONCILE_LOCK:
proxy_server.proxy_config._add_deployment(db_models=rows)
proxy_server.llm_model_list = router.get_model_list()
return True
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved


async def _resync_guardrails(guardrail_name: str) -> bool:
from litellm.proxy import proxy_server
from litellm.proxy.guardrails.guardrail_registry import (
IN_MEMORY_GUARDRAIL_HANDLER,
GuardrailRegistry,
)

if not _db_backed_registries_enabled("guardrails"):
return False
prisma_client: Final = proxy_server.prisma_client
assert prisma_client is not None
row: Final = await GuardrailRegistry().get_guardrail_by_name_from_db(
Comment thread
veria-ai[bot] marked this conversation as resolved.
Resolved
guardrail_name=guardrail_name, prisma_client=prisma_client
)
if row is None:
return False
IN_MEMORY_GUARDRAIL_HANDLER.sync_guardrail_from_db(guardrail=row)
return _initialized_guardrail(guardrail_name) is not None
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved


async def _resync_agents(agent_id_or_name: str) -> bool:
from litellm.proxy import proxy_server
from litellm.proxy.agent_endpoints.agent_registry import (
AGENT_RECONCILE_LOCK,
agents_table,
global_agent_registry,
)
from litellm.types.agents import AgentResponse

if not _db_backed_registries_enabled("agents"):
return False
if _agent_from_registry(agent_id_or_name) is not None:
return True
prisma_client: Final = proxy_server.prisma_client
assert prisma_client is not None
table: Final = agents_table(prisma_client)
id_filter: Final[LiteLLM_AgentsTableWhereUniqueInput] = {"agent_id": agent_id_or_name}
name_filter: Final[LiteLLM_AgentsTableWhereUniqueInput] = {"agent_name": agent_id_or_name}
include_permission: Final[LiteLLM_AgentsTableInclude] = {"object_permission": True}
async with AGENT_RECONCILE_LOCK:
if _agent_from_registry(agent_id_or_name) is not None:
return True
row: Final = await table.find_unique(where=id_filter, include=include_permission) or await table.find_unique(
where=name_filter, include=include_permission
)
if row is None:
return False
global_agent_registry.register_agent(agent_config=AgentResponse.model_validate(row.model_dump()))
return True


model_registry_read_through: Final = RegistryReadThrough(resync=_resync_model_deployments)
guardrail_registry_read_through: Final = RegistryReadThrough(resync=_resync_guardrails)
agent_registry_read_through: Final = RegistryReadThrough(resync=_resync_agents)


def _agent_from_registry(agent_id_or_name: str) -> "AgentResponse | None":
from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry

by_id: Final = global_agent_registry.get_agent_by_id(agent_id=agent_id_or_name)
if by_id is not None:
return by_id
return global_agent_registry.get_agent_by_name(agent_name=agent_id_or_name)


async def get_agent_with_read_through(agent_id_or_name: str) -> "AgentResponse | None":
agent: Final = _agent_from_registry(agent_id_or_name)
if agent is not None:
return agent
if not await agent_registry_read_through.attempt(agent_id_or_name):
return None
return _agent_from_registry(agent_id_or_name)


def _initialized_guardrail(guardrail_name: str) -> "CustomGuardrail | None":
from litellm.proxy.guardrails import guardrail_endpoints

return guardrail_endpoints.GUARDRAIL_REGISTRY.get_initialized_guardrail_callback(guardrail_name=guardrail_name)


async def get_initialized_guardrail_with_read_through(guardrail_name: str) -> "CustomGuardrail | None":
active: Final = _initialized_guardrail(guardrail_name)
if active is not None:
return active
if not await guardrail_registry_read_through.attempt(guardrail_name):
return None
return _initialized_guardrail(guardrail_name)
6 changes: 5 additions & 1 deletion litellm/proxy/guardrails/guardrail_endpoints.py
Original file line number Diff line number Diff line change
Expand Up @@ -2305,8 +2305,12 @@ async def apply_guardrail(
litellm_logging_obj = None
start_time: Final = datetime.now(timezone.utc)

from litellm.proxy.common_utils.registry_read_through import (
get_initialized_guardrail_with_read_through,
)

try:
active_guardrail: Final[CustomGuardrail | None] = GUARDRAIL_REGISTRY.get_initialized_guardrail_callback(
active_guardrail: Final[CustomGuardrail | None] = await get_initialized_guardrail_with_read_through(
guardrail_name=request.guardrail_name
)
if active_guardrail is None:
Expand Down
Loading
Loading