Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
876fa55
fix(scheduler): remove a request's queue entry once it stops waiting
RachelHuangZW Sep 24, 2026
da4dc27
test(router): allowlist _wait_for_scheduler_turn in the router covera…
RachelHuangZW Sep 24, 2026
629b9ef
fix(scheduler): admit healthy requests before reading the queue, and …
RachelHuangZW Sep 24, 2026
f5383a8
refactor(scheduler): move the wait loop into Scheduler.wait_for_turn
RachelHuangZW Sep 24, 2026
471cc69
Merge remote-tracking branch 'upstream/main' into litellm_scheduler_r…
RachelHuangZW Sep 25, 2026
d4596aa
Merge remote-tracking branch 'origin/main' into litellm_scheduler_rem…
RachelHuangZW Sep 29, 2026
e7db391
test(scheduler): move scheduler tests into tests/unit
RachelHuangZW Sep 29, 2026
0c9ab36
fix(scheduler): finish the queue removal when cancellation is deliver…
RachelHuangZW Sep 29, 2026
fe16915
Merge remote-tracking branch 'upstream/main' into litellm_scheduler_r…
RachelHuangZW Sep 30, 2026
f9b785a
Merge remote-tracking branch 'upstream/main' into litellm_scheduler_r…
RachelHuangZW Oct 7, 2026
5de5644
Merge remote-tracking branch 'origin/main' into litellm_scheduler_rem…
mateo-berri Oct 7, 2026
e6cdc06
fix(scheduler): read queue entries back from redis as tuples
mateo-berri Oct 7, 2026
1313444
fix(scheduler): admit a request whose queue entry vanished during coo…
mateo-berri Oct 7, 2026
c835b73
fix(scheduler): re-enqueue a request whose queue entry vanished durin…
mateo-berri Oct 7, 2026
0be0db2
test(integration): cover priority scheduler queue cleanup across inst…
mateo-berri Oct 8, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
151 changes: 41 additions & 110 deletions litellm/router.py
Original file line number Diff line number Diff line change
Expand Up @@ -4528,61 +4528,21 @@ async def schedule_acompletion(
stream=False,
**kwargs,
):
parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs)
### FLOW ITEM ###
_request_id: Final = str(uuid.uuid4())
item: Final = FlowItem(
priority=priority, # 👈 SET PRIORITY FOR REQUEST
request_id=_request_id, # 👈 SET REQUEST ID
model_name=model, # 👈 SAME as 'Router'
)
### [fin] ###

## ADDS REQUEST TO QUEUE ##
await self.scheduler.add_request(request=item)

## POLL QUEUE
end_time: Final = time.monotonic() + self.timeout
curr_time = time.monotonic()
poll_interval: Final = self.scheduler.polling_interval # poll every 3ms
make_request = False

while curr_time < end_time:
_healthy_deployments, _ = await self._async_get_healthy_deployments(
model=model, parent_otel_span=parent_otel_span
)
make_request = await self.scheduler.poll( ## POLL QUEUE ## - returns 'True' if there's healthy deployments OR if request is at top of queue
id=item.request_id,
model_name=item.model_name,
health_deployments=_healthy_deployments,
)
if make_request: ## IF TRUE -> MAKE REQUEST
break
else: ## ELSE -> loop till default_timeout
await asyncio.sleep(poll_interval)
curr_time = time.monotonic()

if make_request:
try:
_response: Final = await self.acompletion(model=model, messages=messages, stream=stream, **kwargs)
response_hidden_params: Final = get_hidden_params(_response)
if response_hidden_params is not None:
additional_headers: Final = cast( # cast-ok: router headers are stored as a mutable mapping
dict[str, object], response_hidden_params.setdefault("additional_headers", {})
)
additional_headers.update({"x-litellm-request-prioritization-used": True})
return _response
except Exception as e:
setattr(e, "priority", priority)
raise e
else:
# Clean up the request from the scheduler queue also before raising the timeout exception
await self.scheduler.remove_request(request_id=item.request_id, model_name=item.model_name)
raise litellm.Timeout(
message="Request timed out while polling queue",
model=model,
llm_provider="openai",
)
await self._wait_for_scheduler_turn(
model=model, priority=priority, parent_otel_span=get_parent_otel_span_from_kwargs(kwargs)
)
try:
_response: Final = await self.acompletion(model=model, messages=messages, stream=stream, **kwargs)
response_hidden_params: Final = get_hidden_params(_response)
if response_hidden_params is not None:
additional_headers: Final = cast( # cast-ok: router headers are stored as a mutable mapping
dict[str, object], response_hidden_params.setdefault("additional_headers", {})
)
additional_headers.update({"x-litellm-request-prioritization-used": True})
return _response
except Exception as e:
setattr(e, "priority", priority)
raise e

async def _schedule_factory(
self,
Expand All @@ -4592,61 +4552,32 @@ async def _schedule_factory(
args: tuple[object, ...],
kwargs: dict[str, object],
):
parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs)
### FLOW ITEM ###
_request_id: Final = str(uuid.uuid4())
item: Final = FlowItem(
priority=priority, # 👈 SET PRIORITY FOR REQUEST
request_id=_request_id, # 👈 SET REQUEST ID
model_name=model, # 👈 SAME as 'Router'
)
### [fin] ###

## ADDS REQUEST TO QUEUE ##
await self.scheduler.add_request(request=item)

## POLL QUEUE
end_time: Final = time.monotonic() + self.timeout
curr_time = time.monotonic()
poll_interval: Final = self.scheduler.polling_interval # poll every 3ms
make_request = False

while curr_time < end_time:
_healthy_deployments, _ = await self._async_get_healthy_deployments(
model=model, parent_otel_span=parent_otel_span
)
make_request = await self.scheduler.poll( ## POLL QUEUE ## - returns 'True' if there's healthy deployments OR if request is at top of queue
id=item.request_id,
model_name=item.model_name,
health_deployments=_healthy_deployments,
)
if make_request: ## IF TRUE -> MAKE REQUEST
break
else: ## ELSE -> loop till default_timeout
await asyncio.sleep(poll_interval)
curr_time = time.monotonic()

if make_request:
try:
_response: Final = await original_function(*args, **kwargs)
response_hidden_params: Final = get_hidden_params(_response)
if response_hidden_params is not None:
additional_headers: Final = cast( # cast-ok: router headers are stored as a mutable mapping
dict[str, object], response_hidden_params.setdefault("additional_headers", {})
)
additional_headers.update({"x-litellm-request-prioritization-used": True})
return _response
except Exception as e:
setattr(e, "priority", priority)
raise e
else:
# Clean up the request from the scheduler queue also before raising the timeout exception
await self.scheduler.remove_request(request_id=item.request_id, model_name=item.model_name)
raise litellm.Timeout(
message="Request timed out while polling queue",
model=model,
llm_provider="openai",
)
await self._wait_for_scheduler_turn(
model=model, priority=priority, parent_otel_span=get_parent_otel_span_from_kwargs(kwargs)
)
try:
_response: Final = await original_function(*args, **kwargs)
response_hidden_params: Final = get_hidden_params(_response)
if response_hidden_params is not None:
additional_headers: Final = cast( # cast-ok: router headers are stored as a mutable mapping
dict[str, object], response_hidden_params.setdefault("additional_headers", {})
)
additional_headers.update({"x-litellm-request-prioritization-used": True})
return _response
except Exception as e:
setattr(e, "priority", priority)
raise e

async def _wait_for_scheduler_turn(self, model: str, priority: int, parent_otel_span: Span | None) -> None:
async def healthy_deployments() -> Sequence[object]:
deployments, _ = await self._async_get_healthy_deployments(model=model, parent_otel_span=parent_otel_span)
return deployments

await self.scheduler.wait_for_turn(
request=FlowItem(priority=priority, request_id=str(uuid.uuid4()), model_name=model),
timeout=self.timeout,
get_healthy_deployments=healthy_deployments,
)

def _is_prompt_management_model(self, model: str) -> bool:
model_list: Final = self.get_model_list(model_name=model)
Expand Down
82 changes: 55 additions & 27 deletions litellm/scheduler.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,22 @@
import asyncio
import enum
import heapq
from typing import Final
import time
from collections.abc import Awaitable, Callable, Sequence
from typing import Final, TypeAlias

from pydantic import TypeAdapter

from litellm import print_verbose
from litellm._internal_context import with_service_target
from litellm.caching.caching import DualCache, RedisCache
from litellm.constants import DEFAULT_IN_MEMORY_TTL, DEFAULT_POLLING_INTERVAL
from litellm.exceptions import Timeout
from litellm.types.llms.base import LiteLLMBaseModel

SCHEDULER_QUEUE_TARGET: Final = "scheduler_queue"
QueueEntry: TypeAlias = tuple[int, str]
_QUEUE_ENTRIES: Final = TypeAdapter(list[QueueEntry])


class SchedulerCacheKeys(enum.Enum):
Expand All @@ -33,7 +41,7 @@ def __init__(
"""
polling_interval: float or null - frequency of polling queue. Default is 3ms.
"""
self.queue: list = []
self.queue: list[QueueEntry] = []
default_in_memory_ttl: float | None = None
if redis_cache is not None:
# if redis-cache available frequently poll that instead of using in-memory.
Expand All @@ -51,7 +59,7 @@ async def add_request(self, request: FlowItem):
# save the queue
await self.save_queue(queue=queue, model_name=request.model_name)

async def poll(self, id: str, model_name: str, health_deployments: list) -> bool:
async def poll(self, request: FlowItem, health_deployments: Sequence[object]) -> bool:
"""
Return if request can be processed.

Expand All @@ -62,30 +70,48 @@ async def poll(self, id: str, model_name: str, health_deployments: list) -> bool
- False:
* If no healthy deployments available
* AND request not at the top of queue
"""
queue: Final = await self.get_queue(model_name=model_name)
if not queue:
raise Exception(f"Incorrectly setup. Queue is invalid. Queue={queue}")

# ------------
# Setup values
# ------------

A request the queue no longer holds (its cache key expired or a concurrent writer erased the entry)
is put back at its priority so it keeps its place in the order instead of failing or jumping ahead
"""
print_verbose(f"len(health_deployments): {len(health_deployments)}")
if len(health_deployments) == 0:
print_verbose(f"queue: {queue}, seeking id={id}")
# Check if the id is at the top of the heap
if queue[0][1] == id:
# Remove the item from the queue
heapq.heappop(queue)
await self.save_queue(queue=queue, model_name=model_name)
print_verbose(f"Popped id: {id}")
return True
else:
if len(health_deployments) > 0:
return True

queue: Final = await self.get_queue(model_name=request.model_name)
entry: Final = (request.priority, request.request_id)
print_verbose(f"queue: {queue}, seeking {entry}")
if entry not in queue:
print_verbose(f"queue no longer holds {entry}, re-enqueueing it")
heapq.heappush(queue, entry)
if queue[0] != entry:
await self.save_queue(queue=queue, model_name=request.model_name)
return False
elif queue[0] != entry:
return False

heapq.heappop(queue)
await self.save_queue(queue=queue, model_name=request.model_name)
print_verbose(f"Popped id: {request.request_id}")
return True

async def wait_for_turn(
self,
request: FlowItem,
timeout: float,
get_healthy_deployments: Callable[[], Awaitable[Sequence[object]]],
) -> None:
try:
await self.add_request(request=request)
end_time: Final = time.monotonic() + timeout
while time.monotonic() < end_time:
if await self.poll(request=request, health_deployments=await get_healthy_deployments()):
return
await asyncio.sleep(self.polling_interval)
finally:
await asyncio.shield(self.remove_request(request_id=request.request_id, model_name=request.model_name))
raise Timeout(message="Request timed out while polling queue", model=request.model_name, llm_provider="openai")

async def remove_request(self, request_id: str, model_name: str) -> None:
"""
Remove a specific request from the priority queue for a model.
Expand Down Expand Up @@ -118,21 +144,23 @@ def get_queue_status(self):
return self.queue

@with_service_target(SCHEDULER_QUEUE_TARGET)
async def get_queue(self, model_name: str) -> list:
async def get_queue(self, model_name: str) -> list[QueueEntry]:
"""
Return a queue for that specific model group
Return a queue for that specific model group.

Redis hands the queue back as JSON lists, so every entry is validated into the
(priority, request_id) tuple the heap operations compare against.
"""
if self.cache is not None:
_cache_key: Final = f"{SchedulerCacheKeys.queue.value}:{model_name}"
response: Final = await self.cache.async_get_cache(key=_cache_key)
if response is None or not isinstance(response, list):
if not isinstance(response, list):
return []
elif isinstance(response, list):
return response
return _QUEUE_ENTRIES.validate_python(response)
return self.queue

@with_service_target(SCHEDULER_QUEUE_TARGET)
async def save_queue(self, queue: list, model_name: str) -> None:
async def save_queue(self, queue: list[QueueEntry], model_name: str) -> None:
"""
Save the updated queue of the model group
"""
Expand Down
1 change: 1 addition & 0 deletions tests/code_coverage_tests/router_code_coverage.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ def get_functions_from_router(file_path):
"_resolve_claude_code_session_router", # Tested through Claude Code session routing in test_router.py
"_get_claude_code_session_router_binding", # Tested through the two-worker session routing test in test_router.py
"_apply_updated_routing_strategy_args", # Tested via update_settings in test_lowest_latency.py (file lacks "router" in name)
"_wait_for_scheduler_turn", # Tested through prioritized acompletion and atext_completion in test_router.py
"arm_routing_read_prefetch", # Tested in tests/unit/caching/test_request_redis_batch_pre_call.py (file lacks "router" in name)
"_configured_model_info", # Tested through get_configured_service_tiers in test_router.py
"_routable_deployments", # Tested through get_configured_service_tiers and get_routable_upstream_model in test_router.py
Expand Down
Loading
Loading