Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
876fa55
fix(scheduler): remove a request's queue entry once it stops waiting
RachelHuangZW Sep 24, 2026
da4dc27
test(router): allowlist _wait_for_scheduler_turn in the router covera…
RachelHuangZW Sep 24, 2026
629b9ef
fix(scheduler): admit healthy requests before reading the queue, and …
RachelHuangZW Sep 24, 2026
f5383a8
refactor(scheduler): move the wait loop into Scheduler.wait_for_turn
RachelHuangZW Sep 24, 2026
471cc69
Merge remote-tracking branch 'upstream/main' into litellm_scheduler_r…
RachelHuangZW Sep 25, 2026
d4596aa
Merge remote-tracking branch 'origin/main' into litellm_scheduler_rem…
RachelHuangZW Sep 29, 2026
e7db391
test(scheduler): move scheduler tests into tests/unit
RachelHuangZW Sep 29, 2026
0c9ab36
fix(scheduler): finish the queue removal when cancellation is deliver…
RachelHuangZW Sep 29, 2026
fe16915
Merge remote-tracking branch 'upstream/main' into litellm_scheduler_r…
RachelHuangZW Sep 30, 2026
f9b785a
Merge remote-tracking branch 'upstream/main' into litellm_scheduler_r…
RachelHuangZW Oct 7, 2026
5de5644
Merge remote-tracking branch 'origin/main' into litellm_scheduler_rem…
mateo-berri Oct 7, 2026
e6cdc06
fix(scheduler): read queue entries back from redis as tuples
mateo-berri Oct 7, 2026
1313444
fix(scheduler): admit a request whose queue entry vanished during coo…
mateo-berri Oct 7, 2026
c835b73
fix(scheduler): re-enqueue a request whose queue entry vanished durin…
mateo-berri Oct 7, 2026
0be0db2
test(integration): cover priority scheduler queue cleanup across inst…
mateo-berri Oct 8, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
144 changes: 40 additions & 104 deletions litellm/router.py
Original file line number Diff line number Diff line change
Expand Up @@ -4313,57 +4313,17 @@ async def schedule_acompletion(
stream=False,
**kwargs,
):
parent_otel_span: Final = _get_parent_otel_span_from_kwargs(kwargs)
### FLOW ITEM ###
_request_id: Final = str(uuid.uuid4())
item: Final = FlowItem(
priority=priority, # 👈 SET PRIORITY FOR REQUEST
request_id=_request_id, # 👈 SET REQUEST ID
model_name=model, # 👈 SAME as 'Router'
)
### [fin] ###

## ADDS REQUEST TO QUEUE ##
await self.scheduler.add_request(request=item)

## POLL QUEUE
end_time: Final = time.monotonic() + self.timeout
curr_time = time.monotonic()
poll_interval: Final = self.scheduler.polling_interval # poll every 3ms
make_request = False

while curr_time < end_time:
_healthy_deployments, _ = await self._async_get_healthy_deployments(
model=model, parent_otel_span=parent_otel_span
)
make_request = await self.scheduler.poll( ## POLL QUEUE ## - returns 'True' if there's healthy deployments OR if request is at top of queue
id=item.request_id,
model_name=item.model_name,
health_deployments=_healthy_deployments,
)
if make_request: ## IF TRUE -> MAKE REQUEST
break
else: ## ELSE -> loop till default_timeout
await asyncio.sleep(poll_interval)
curr_time = time.monotonic()

if make_request:
try:
_response: Final = await self.acompletion(model=model, messages=messages, stream=stream, **kwargs)
_response._hidden_params.setdefault("additional_headers", {})
_response._hidden_params["additional_headers"].update({"x-litellm-request-prioritization-used": True})
return _response
except Exception as e:
setattr(e, "priority", priority)
raise e
else:
# Clean up the request from the scheduler queue also before raising the timeout exception
await self.scheduler.remove_request(request_id=item.request_id, model_name=item.model_name)
raise litellm.Timeout(
message="Request timed out while polling queue",
model=model,
llm_provider="openai",
)
await self._wait_for_scheduler_turn(
model=model, priority=priority, parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs)
)
try:
_response: Final = await self.acompletion(model=model, messages=messages, stream=stream, **kwargs)
_response._hidden_params.setdefault("additional_headers", {})
_response._hidden_params["additional_headers"].update({"x-litellm-request-prioritization-used": True})
return _response
except Exception as e:
setattr(e, "priority", priority)
raise e

async def _schedule_factory(
self,
Expand All @@ -4373,60 +4333,36 @@ async def _schedule_factory(
args: tuple[object, ...],
kwargs: dict[str, object],
):
parent_otel_span: Final = _get_parent_otel_span_from_kwargs(kwargs)
### FLOW ITEM ###
_request_id: Final = str(uuid.uuid4())
item: Final = FlowItem(
priority=priority, # 👈 SET PRIORITY FOR REQUEST
request_id=_request_id, # 👈 SET REQUEST ID
model_name=model, # 👈 SAME as 'Router'
)
### [fin] ###

## ADDS REQUEST TO QUEUE ##
await self.scheduler.add_request(request=item)

## POLL QUEUE
end_time: Final = time.monotonic() + self.timeout
curr_time = time.monotonic()
poll_interval: Final = self.scheduler.polling_interval # poll every 3ms
make_request = False

while curr_time < end_time:
_healthy_deployments, _ = await self._async_get_healthy_deployments(
model=model, parent_otel_span=parent_otel_span
)
make_request = await self.scheduler.poll( ## POLL QUEUE ## - returns 'True' if there's healthy deployments OR if request is at top of queue
id=item.request_id,
model_name=item.model_name,
health_deployments=_healthy_deployments,
)
if make_request: ## IF TRUE -> MAKE REQUEST
break
else: ## ELSE -> loop till default_timeout
await asyncio.sleep(poll_interval)
curr_time = time.monotonic()
await self._wait_for_scheduler_turn(
model=model, priority=priority, parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs)
)
try:
_response: Final = await original_function(*args, **kwargs)
if isinstance(_response._hidden_params, dict):
_response._hidden_params.setdefault("additional_headers", {})
_response._hidden_params["additional_headers"].update({"x-litellm-request-prioritization-used": True})
return _response
except Exception as e:
setattr(e, "priority", priority)
raise e

if make_request:
try:
_response: Final = await original_function(*args, **kwargs)
if isinstance(_response._hidden_params, dict):
_response._hidden_params.setdefault("additional_headers", {})
_response._hidden_params["additional_headers"].update(
{"x-litellm-request-prioritization-used": True}
)
return _response
except Exception as e:
setattr(e, "priority", priority)
raise e
else:
# Clean up the request from the scheduler queue also before raising the timeout exception
await self.scheduler.remove_request(request_id=item.request_id, model_name=item.model_name)
raise litellm.Timeout(
message="Request timed out while polling queue",
model=model,
llm_provider="openai",
)
async def _wait_for_scheduler_turn(self, model: str, priority: int, parent_otel_span: Span | None) -> None:
item: Final = FlowItem(priority=priority, request_id=str(uuid.uuid4()), model_name=model)
try:
await self.scheduler.add_request(request=item)
end_time: Final = time.monotonic() + self.timeout
while time.monotonic() < end_time:
healthy_deployments, _ = await self._async_get_healthy_deployments(
model=model, parent_otel_span=parent_otel_span
)
if await self.scheduler.poll(
id=item.request_id, model_name=model, health_deployments=healthy_deployments
):
return
await asyncio.sleep(self.scheduler.polling_interval)
finally:
await self.scheduler.remove_request(request_id=item.request_id, model_name=model)
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
raise litellm.Timeout(message="Request timed out while polling queue", model=model, llm_provider="openai")

def _is_prompt_management_model(self, model: str) -> bool:
model_list: Final = self.get_model_list(model_name=model)
Expand Down
26 changes: 10 additions & 16 deletions litellm/scheduler.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,27 +61,21 @@ async def poll(self, id: str, model_name: str, health_deployments: list) -> bool
* If no healthy deployments available
* AND request not at the top of queue
"""
print_verbose(f"len(health_deployments): {len(health_deployments)}")
if len(health_deployments) > 0:
return True

queue: Final = await self.get_queue(model_name=model_name)
if not queue:
raise Exception(f"Incorrectly setup. Queue is invalid. Queue={queue}")

# ------------
# Setup values
# ------------

print_verbose(f"len(health_deployments): {len(health_deployments)}")
if len(health_deployments) == 0:
print_verbose(f"queue: {queue}, seeking id={id}")
# Check if the id is at the top of the heap
if queue[0][1] == id:
# Remove the item from the queue
heapq.heappop(queue)
await self.save_queue(queue=queue, model_name=model_name)
print_verbose(f"Popped id: {id}")
return True
else:
return False
print_verbose(f"queue: {queue}, seeking id={id}")
if queue[0][1] != id:
return False

heapq.heappop(queue)
await self.save_queue(queue=queue, model_name=model_name)
print_verbose(f"Popped id: {id}")
return True

async def remove_request(self, request_id: str, model_name: str) -> None:
Expand Down
1 change: 1 addition & 0 deletions tests/code_coverage_tests/router_code_coverage.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ def get_functions_from_router(file_path):
"_resolve_claude_code_session_router", # Tested through Claude Code session routing in test_router.py
"_get_claude_code_session_router_binding", # Tested through the two-worker session routing test in test_router.py
"_apply_updated_routing_strategy_args", # Tested via update_settings in test_lowest_latency.py (file lacks "router" in name)
"_wait_for_scheduler_turn", # Tested through prioritized acompletion and atext_completion in test_router.py
]


Expand Down
86 changes: 86 additions & 0 deletions tests/test_litellm/test_router.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
from litellm.router_utils.client_initalization_utils import MaxParallelRequestsLimit
from litellm.router_utils.cooldown_handlers import _async_get_cooldown_deployments
from litellm.router_utils.router_callbacks.track_deployment_metrics import get_deployment_successes_for_current_minute
from litellm.scheduler import FlowItem, Scheduler
from litellm.types.llms.openai import ChatCompletionRequest
from litellm.types.router import Deployment, DeploymentTypedDict, LiteLLM_Params, ModelInfo, PreRoutingHookResponse, RetryPolicy

Expand Down Expand Up @@ -17862,3 +17863,88 @@ def test_access_windows_filter_reserved_deployments_method():
request_team_id="team-a",
)
] == ["reserved-deployment", "open-deployment"]


def _scheduled_router(timeout: float) -> Router:
return Router(
model_list=[
{
"model_name": "sched-model",
"litellm_params": {"model": "openai/sched-model", "api_key": "sk-fake", "mock_response": "hi"},
"model_info": {"id": "sched-deployment"},
}
],
timeout=timeout,
)


async def _send_scheduled_chat(router: Router, priority: int) -> object:
return await router.acompletion(
model="sched-model", messages=[{"role": "user", "content": "hi"}], priority=priority
)


async def _send_scheduled_text(router: Router, priority: int) -> object:
return await router.atext_completion(model="sched-model", prompt="hi", priority=priority)


@pytest.mark.parametrize(
"send", [_send_scheduled_chat, _send_scheduled_text], ids=["schedule_acompletion", "schedule_factory"]
)
@pytest.mark.asyncio
async def test_admitted_prioritized_request_does_not_block_later_request_during_cooldown(
send: Callable[[Router, int], Awaitable[object]],
):
from litellm.types.router import RouterRateLimitError

router: Final = _scheduled_router(timeout=1)
await send(router, 1)
_cool_down(router, "sched-deployment")

with pytest.raises(RouterRateLimitError, match="cooldown"):
await send(router, 2)


@pytest.mark.parametrize("stop_waiting", ["cancel", "timeout"])
@pytest.mark.asyncio
async def test_prioritized_request_leaves_queue_when_it_stops_waiting(stop_waiting: Literal["cancel", "timeout"]):
router: Final = _scheduled_router(timeout=0.5)
_cool_down(router, "sched-deployment")
await router.scheduler.add_request(FlowItem(priority=0, request_id="head-of-queue", model_name="sched-model"))
waiting: Final = asyncio.create_task(_send_scheduled_chat(router, 5))
await asyncio.sleep(0.05)
assert len(await router.scheduler.get_queue("sched-model")) == 2

if stop_waiting == "cancel":
waiting.cancel()
with pytest.raises(asyncio.CancelledError if stop_waiting == "cancel" else litellm.Timeout):
await waiting

assert await router.scheduler.get_queue("sched-model") == [(0, "head-of-queue")]


class _PausesAfterEnqueueScheduler(Scheduler):
def __init__(self) -> None:
super().__init__()
self.enqueued: Final = asyncio.Event()

async def add_request(self, request: FlowItem) -> None:
await super().add_request(request)
self.enqueued.set()
await asyncio.Event().wait()


@pytest.mark.asyncio
async def test_prioritized_request_cancelled_while_enqueueing_leaves_queue():
router: Final = _scheduled_router(timeout=5)
scheduler: Final = _PausesAfterEnqueueScheduler()
router.scheduler = scheduler
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
waiting: Final = asyncio.create_task(_send_scheduled_chat(router, 1))
await scheduler.enqueued.wait()
assert len(await scheduler.get_queue("sched-model")) == 1

waiting.cancel()
with pytest.raises(asyncio.CancelledError):
await waiting

assert await scheduler.get_queue("sched-model") == []
25 changes: 25 additions & 0 deletions tests/test_litellm/test_scheduler.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
from typing import Final

import pytest

from litellm.scheduler import FlowItem, Scheduler


@pytest.mark.asyncio
async def test_poll_admits_request_missing_from_queue_while_a_deployment_is_healthy():
scheduler: Final = Scheduler()

assert await scheduler.poll(
id="erased-by-concurrent-write", model_name="sched-model", health_deployments=[{"model_info": {"id": "a"}}]
)


@pytest.mark.asyncio
async def test_poll_during_cooldown_admits_only_the_head_of_the_queue():
scheduler: Final = Scheduler()
await scheduler.add_request(FlowItem(priority=2, request_id="later", model_name="sched-model"))
await scheduler.add_request(FlowItem(priority=1, request_id="head", model_name="sched-model"))

assert not await scheduler.poll(id="later", model_name="sched-model", health_deployments=[])
assert await scheduler.poll(id="head", model_name="sched-model", health_deployments=[])
assert await scheduler.get_queue("sched-model") == [(2, "later")]
Loading