diff --git a/components/src/dynamo/common/configuration/groups/kv_router_args.py b/components/src/dynamo/common/configuration/groups/kv_router_args.py index 2067357f2ea8..cfe81864d909 100644 --- a/components/src/dynamo/common/configuration/groups/kv_router_args.py +++ b/components/src/dynamo/common/configuration/groups/kv_router_args.py @@ -11,6 +11,7 @@ """ import argparse +import logging import os import warnings from typing import Optional @@ -51,9 +52,24 @@ "serve_indexer", "shared_cache_multiplier", "shared_cache_type", + "conditional_disagg_enabled", + "conditional_disagg_policy", + "conditional_disagg_eff_isl_threshold", + "conditional_disagg_eff_isl_ratio_threshold", + "conditional_disagg_prefill_busy_threshold", + "conditional_disagg_decode_busy_threshold", "router_predicted_ttl_secs", ) +CONDITIONAL_DISAGG_POLICY_CHOICES: tuple[str, ...] = ( + "isl_bounding", + "prefill_load", + "isl_or_load", +) +LOAD_AWARE_CONDITIONAL_DISAGG_POLICIES: frozenset[str] = frozenset( + {"prefill_load", "isl_or_load"} +) + _DEPRECATED_OVERLAP_WEIGHT_MESSAGE = ( "router KV overlap score weight is deprecated; use " "--router-prefill-load-scale or DYN_ROUTER_PREFILL_LOAD_SCALE for equivalent behavior" @@ -105,6 +121,38 @@ def _default_prefill_load_scale() -> float: return 1.0 +def warn_conditional_disagg_prefill_busy_threshold_resolution( + *, + policy: str, + busy_threshold: Optional[float], + queue_threshold: Optional[float], +) -> None: + if policy not in LOAD_AWARE_CONDITIONAL_DISAGG_POLICIES: + return + if busy_threshold is not None: + logging.getLogger(__name__).info( + "conditional_disagg prefill-load condition using " + "--router-conditional-disagg-prefill-busy-threshold=%s", + busy_threshold, + ) + elif queue_threshold is not None: + logging.getLogger(__name__).info( + "conditional_disagg prefill-load condition using " + "--router-queue-threshold=%s because " + "--router-conditional-disagg-prefill-busy-threshold is unset", + queue_threshold, + ) + else: + warnings.warn( + f"--router-conditional-disagg-policy={policy!r} consumes the " + "prefill-worker busy signal, but neither " + "--router-conditional-disagg-prefill-busy-threshold nor " + "--router-queue-threshold is set; the prefill-load condition " + "will be disabled.", + stacklevel=3, + ) + + class KvRouterConfigBase(ConfigBase): """Mixin carrying the shared KvRouterConfig fields.""" @@ -134,6 +182,12 @@ class KvRouterConfigBase(ConfigBase): serve_indexer: bool = False shared_cache_multiplier: float = 0.0 shared_cache_type: str = "none" + conditional_disagg_enabled: bool = False + conditional_disagg_policy: str = "isl_bounding" + conditional_disagg_eff_isl_threshold: int = 2048 + conditional_disagg_eff_isl_ratio_threshold: float = 0.7 + conditional_disagg_prefill_busy_threshold: Optional[float] = None + conditional_disagg_decode_busy_threshold: Optional[float] = None router_predicted_ttl_secs: Optional[float] = None load_aware: bool = False @@ -418,6 +472,88 @@ def add_arguments(self, parser) -> None: ), arg_type=str, ) + add_negatable_bool_argument( + g, + flag_name="--router-conditional-disagg", + env_var="DYN_ROUTER_CONDITIONAL_DISAGG", + default=False, + help=( + "[EXPERIMENTAL] KV Router: Enable conditional disaggregation. " + "When enabled, the frontend may skip remote prefill and route " + "a request directly to the cache-hot decode worker when the " + "configured policy decides local prefill+decode is preferable." + ), + dest="conditional_disagg_enabled", + ) + add_argument( + g, + flag_name="--router-conditional-disagg-policy", + env_var="DYN_ROUTER_CONDITIONAL_DISAGG_POLICY", + default="isl_bounding", + help=( + "KV Router: Conditional-disagg bypass policy. " + "'isl_bounding': bypass when effective ISL is below both the " + "absolute and ratio thresholds. 'prefill_load': bypass when " + "the chosen prefill worker is busy. 'isl_or_load': bypass " + "when either policy would bypass." + ), + arg_type=str, + choices=list(CONDITIONAL_DISAGG_POLICY_CHOICES), + dest="conditional_disagg_policy", + ) + add_argument( + g, + flag_name="--router-conditional-disagg-eff-isl-threshold", + env_var="DYN_ROUTER_CONDITIONAL_DISAGG_EFF_ISL_THRESHOLD", + default=2048, + help=( + "KV Router: For 'isl_bounding' and the ISL arm of " + "'isl_or_load', require effective ISL to be below this many tokens." + ), + arg_type=int, + dest="conditional_disagg_eff_isl_threshold", + ) + add_argument( + g, + flag_name="--router-conditional-disagg-eff-isl-ratio-threshold", + env_var="DYN_ROUTER_CONDITIONAL_DISAGG_EFF_ISL_RATIO_THRESHOLD", + default=0.7, + help=( + "KV Router: For 'isl_bounding' and the ISL arm of " + "'isl_or_load', require effective ISL / raw ISL to be below " + "this value. Must be in [0.0, 1.0]." + ), + arg_type=float, + dest="conditional_disagg_eff_isl_ratio_threshold", + ) + add_argument( + g, + flag_name="--router-conditional-disagg-prefill-busy-threshold", + env_var="DYN_ROUTER_CONDITIONAL_DISAGG_PREFILL_BUSY_THRESHOLD", + default=None, + help=( + "KV Router: Dedicated prefill-worker busy-line fraction for " + "'prefill_load' and 'isl_or_load'. A worker is busy when " + "active_tokens(W) > threshold * max_num_batched_tokens(W). " + "If unset, falls back to --router-queue-threshold." + ), + arg_type=float, + dest="conditional_disagg_prefill_busy_threshold", + ) + add_argument( + g, + flag_name="--router-conditional-disagg-decode-busy-threshold", + env_var="DYN_ROUTER_CONDITIONAL_DISAGG_DECODE_BUSY_THRESHOLD", + default=None, + help=( + "KV Router: Decode-busy guard for conditional disagg. " + "When set, conditional-disagg bypass is disabled if the selected " + "decode worker's projected decode load exceeds " + "threshold * total_kv_blocks(W). If unset, the guard is disabled." + ), + arg_type=float, + dest="conditional_disagg_decode_busy_threshold", + ) add_argument( g, flag_name="--router-event-threads", diff --git a/components/src/dynamo/frontend/frontend_args.py b/components/src/dynamo/frontend/frontend_args.py index 25e3030f86e5..4112f1280425 100644 --- a/components/src/dynamo/frontend/frontend_args.py +++ b/components/src/dynamo/frontend/frontend_args.py @@ -13,8 +13,10 @@ AicPerfConfigBase, ) from dynamo.common.configuration.groups.kv_router_args import ( + CONDITIONAL_DISAGG_POLICY_CHOICES, KvRouterArgGroup, KvRouterConfigBase, + warn_conditional_disagg_prefill_busy_threshold_resolution, ) from dynamo.common.configuration.groups.router_args import ( RouterArgGroup, @@ -157,6 +159,30 @@ def validate(self) -> None: raise ValueError( "--serve-indexer and --use-remote-indexer are mutually exclusive" ) + if self.conditional_disagg_policy not in CONDITIONAL_DISAGG_POLICY_CHOICES: + raise ValueError( + "--router-conditional-disagg-policy must be one of " + + ", ".join( + f"'{choice}'" for choice in CONDITIONAL_DISAGG_POLICY_CHOICES + ) + ) + if self.conditional_disagg_eff_isl_threshold < 0: + raise ValueError( + "--router-conditional-disagg-eff-isl-threshold must be >= 0" + ) + if not 0.0 <= self.conditional_disagg_eff_isl_ratio_threshold <= 1.0: + raise ValueError( + "--router-conditional-disagg-eff-isl-ratio-threshold must be in [0.0, 1.0]" + ) + if self.conditional_disagg_enabled and self.router_mode != "kv": + raise ValueError("--router-conditional-disagg requires --router-mode=kv") + if self.conditional_disagg_enabled: + warn_conditional_disagg_prefill_busy_threshold_resolution( + policy=self.conditional_disagg_policy, + busy_threshold=self.conditional_disagg_prefill_busy_threshold, + queue_threshold=self.router_queue_threshold, + ) + self.apply_admission_control() self.validate_rejection_thresholds() self.log_rejection_thresholds() diff --git a/components/src/dynamo/sglang/backend_args.py b/components/src/dynamo/sglang/backend_args.py index 769fc9586c19..6cd64b034a5e 100644 --- a/components/src/dynamo/sglang/backend_args.py +++ b/components/src/dynamo/sglang/backend_args.py @@ -165,6 +165,13 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None: "(1=minimal, 2=per-request [default], 3=+decode_loop, 4=full).", ) + g.add_argument( + "--enable-conditional-disagg", + action=argparse.BooleanOptionalAction, + default=False, + help=argparse.SUPPRESS, + ) + class DynamoSGLangConfig(ConfigBase): """Configuration for Dynamo SGLang wrapper (SGLang-specific only).""" @@ -184,6 +191,7 @@ class DynamoSGLangConfig(ConfigBase): video_generation_worker: bool enable_rl: bool frontend_decoding: bool = False + enable_conditional_disagg: bool = False sglang_trace_level: int # Extra served names beyond the primary, parsed from --served-model-name. @@ -196,6 +204,13 @@ def validate(self) -> None: str(self.embedding_transfer_mode) ) + if self.enable_conditional_disagg: + raise ValueError( + "--enable-conditional-disagg is not supported by the SGLang " + "backend yet. SGLang decode workers cannot run " + "conditional-disagg bypass requests as local prefill+decode." + ) + if (self.disagg_config is not None) ^ (self.disagg_config_key is not None): raise ValueError( "Both 'disagg_config' and 'disagg_config_key' must be provided together." diff --git a/components/src/dynamo/sglang/request_handlers/llm/decode_handler.py b/components/src/dynamo/sglang/request_handlers/llm/decode_handler.py index be1e1f7954e5..3ebc84e32669 100644 --- a/components/src/dynamo/sglang/request_handlers/llm/decode_handler.py +++ b/components/src/dynamo/sglang/request_handlers/llm/decode_handler.py @@ -39,6 +39,17 @@ "top_k", "min_p", ) +BYPASS_REMOTE_PREFILL_ANNOTATION = "x-bypass-remote-prefill" + + +def _raise_if_conditional_disagg_bypass(request: Dict[str, Any]) -> None: + if BYPASS_REMOTE_PREFILL_ANNOTATION not in (request.get("annotations") or []): + return + raise RuntimeError( + "SGLang backend does not support conditional-disagg bypass yet. " + "Disable --router-conditional-disagg or use a backend with " + "decode-side local prefill support." + ) def _nvext_extra_field_requested(request: Dict[str, Any], field: str) -> bool: @@ -341,6 +352,7 @@ async def generate( RuntimeError: If no bootstrap info received from prefill worker. """ logging.debug(f"New Request ID: {context.id()}") + _raise_if_conditional_disagg_bypass(request) trace_id = context.trace_id sampling_params = self._build_sampling_params(request) input_param = self._get_input_param(request) diff --git a/components/src/dynamo/trtllm/request_handlers/handler_base.py b/components/src/dynamo/trtllm/request_handlers/handler_base.py index dc1a0b99e2a1..0e96f5a68f14 100644 --- a/components/src/dynamo/trtllm/request_handlers/handler_base.py +++ b/components/src/dynamo/trtllm/request_handlers/handler_base.py @@ -70,6 +70,8 @@ logger = logging.getLogger(__name__) +BYPASS_REMOTE_PREFILL_ANNOTATION = "x-bypass-remote-prefill" + class TRTLLMEnginePauseController: """Adapts TRT-LLM sleep/wake to the standard pause controller interface. @@ -703,10 +705,20 @@ def _setup_disaggregated_params_for_mode( disaggregated_params = None epd_metadata: dict[str, Any] = {} - # Canary probe: use its pre-built disagg params (skip prefill_result decode - # and skip the mode-specific request_type overrides). - if request.get(HEALTH_CHECK_KEY) and request.get("disaggregated_params"): - return LlmDisaggregatedParams(**request["disaggregated_params"]), None, {} + use_request_disagg_params = request.get(HEALTH_CHECK_KEY) or ( + self.disaggregation_mode == DisaggregationMode.DECODE + and BYPASS_REMOTE_PREFILL_ANNOTATION in (request.get("annotations") or []) + ) + + # Canary probes and conditional-disagg bypasses run a full + # context+generation request on a disagg-mode worker, so they use + # the pre-built params and skip the normal prefill-result handoff. + if use_request_disagg_params and request.get("disaggregated_params"): + return ( + LlmDisaggregatedParams(**request["disaggregated_params"]), + ep_disaggregated_params, + {}, + ) # PREFILL mode: setup context_only params if self.disaggregation_mode == DisaggregationMode.PREFILL: @@ -984,6 +996,16 @@ async def _generate_locally_impl( # Normalize OpenAI format to TRT-LLM internal format self._normalize_request_format(request) + bypass_remote_prefill = ( + self.disaggregation_mode == DisaggregationMode.DECODE + and BYPASS_REMOTE_PREFILL_ANNOTATION in (request.get("annotations") or []) + ) + if bypass_remote_prefill: + logging.debug( + "DECODE: conditional-disagg bypass annotation present; " + "running request as AGG (prefill+decode on this worker)." + ) + request["disaggregated_params"] = {"request_type": "context_and_generation"} # Setup disaggregated params based on PREFILL/DECODE mode ( @@ -1013,6 +1035,7 @@ async def _generate_locally_impl( if ( self.disaggregation_mode == DisaggregationMode.DECODE and disaggregated_params is None + and not bypass_remote_prefill ): logging.error("DECODE: disaggregated_params is None but required!") logging.error(f"DECODE: Request keys: {list(request.keys())}") @@ -1171,11 +1194,13 @@ async def _generate_locally_impl( cache_salt=cache_salt, ) - # In disagg decode mode, wrap abort() to defer until first token - # (KV transfer complete). + # In disagg decode mode with remote prefill, wrap abort() to defer + # until the first token is received (KV transfer complete). abort_guard = ( _DeferredAbort(generation_result) if self.disaggregation_mode == DisaggregationMode.DECODE + and disaggregated_params is not None + and not bypass_remote_prefill else None ) diff --git a/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py b/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py index b07fe5abd50c..45676ed1c4d5 100644 --- a/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py +++ b/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py @@ -24,7 +24,10 @@ from dynamo.trtllm.constants import DisaggregationMode from dynamo.trtllm.health_check import TrtllmHealthCheckPayload from dynamo.trtllm.llm_engine import TrtllmLLMEngine -from dynamo.trtllm.request_handlers.handler_base import HandlerBase +from dynamo.trtllm.request_handlers.handler_base import ( + BYPASS_REMOTE_PREFILL_ANNOTATION, + HandlerBase, +) pytestmark = [ pytest.mark.unit, @@ -717,6 +720,13 @@ def _make_prefill_handler(self, machine_id: int = 42) -> HandlerBase: handler.disaggregation_mode = DisaggregationMode.PREFILL return handler + def _make_decode_handler(self) -> HandlerBase: + config = MagicMock() + config.shutdown_event = None + handler = _ConcreteHandler(config) + handler.disaggregation_mode = DisaggregationMode.DECODE + return handler + def test_disagg_request_id_populated_in_prefill_mode(self): """When mode is PREFILL and no ep_disaggregated_params, disagg_request_id is set.""" handler = self._make_prefill_handler() @@ -782,6 +792,39 @@ def test_different_machine_ids_produce_different_id_ranges(self): ) assert params_a.disagg_request_id != params_b.disagg_request_id + def test_decode_conditional_bypass_uses_request_disagg_params(self): + """Conditional-disagg bypass runs full context+generation on decode.""" + handler = self._make_decode_handler() + params, _, _ = handler._setup_disaggregated_params_for_mode( + request={ + "annotations": [BYPASS_REMOTE_PREFILL_ANNOTATION], + "disaggregated_params": {"request_type": "context_and_generation"}, + }, + ep_disaggregated_params=None, + ) + assert params.request_type == "context_and_generation" + + def test_decode_conditional_bypass_preserves_epd_params(self): + """Conditional-disagg bypass keeps EPD params for multimodal handling.""" + handler = self._make_decode_handler() + ep_disaggregated_params = MagicMock() + + ( + params, + returned_ep_params, + epd_metadata, + ) = handler._setup_disaggregated_params_for_mode( + request={ + "annotations": [BYPASS_REMOTE_PREFILL_ANNOTATION], + "disaggregated_params": {"request_type": "context_and_generation"}, + }, + ep_disaggregated_params=ep_disaggregated_params, + ) + + assert params.request_type == "context_and_generation" + assert returned_ep_params is ep_disaggregated_params + assert epd_metadata == {} + class TestHealthCheckPriority: """Verify generate_locally forwards the correct priority to generate_async. diff --git a/components/src/dynamo/vllm/args.py b/components/src/dynamo/vllm/args.py index 682b06002f14..ef160c614052 100644 --- a/components/src/dynamo/vllm/args.py +++ b/components/src/dynamo/vllm/args.py @@ -401,10 +401,13 @@ def create_kv_events_config( dynamo_config: Config, engine_config: AsyncEngineArgs ) -> Optional[KVEventsConfig]: """Create KVEventsConfig for prefix caching if needed.""" - if dynamo_config.disaggregation_mode == DisaggregationMode.DECODE: + if ( + dynamo_config.disaggregation_mode == DisaggregationMode.DECODE + and not dynamo_config.enable_conditional_disagg + ): logger.info( "Decode worker detected (disaggregation_mode=decode): " - "kv_events_config disabled (decode workers don't publish KV events)" + "kv_events_config disabled (pass --enable-conditional-disagg to opt in)" ) return None diff --git a/components/src/dynamo/vllm/backend_args.py b/components/src/dynamo/vllm/backend_args.py index a0122664a71a..91cd0dd9ba9e 100644 --- a/components/src/dynamo/vllm/backend_args.py +++ b/components/src/dynamo/vllm/backend_args.py @@ -82,6 +82,18 @@ def add_arguments(self, parser) -> None: "Mark this as a decode worker which does not publish KV events.", ) + add_negatable_bool_argument( + g, + flag_name="--enable-conditional-disagg", + env_var="DYN_VLLM_ENABLE_CONDITIONAL_DISAGG", + default=False, + help=( + "Opt this decode worker into conditional-disagg support. Decode " + "workers publish KV events so the Dynamo router can see decode-side " + "cache, and annotated bypass requests run as local prefill+decode." + ), + ) + add_negatable_bool_argument( g, flag_name="--use-vllm-tokenizer", @@ -418,6 +430,7 @@ class DynamoVllmConfig(ConfigBase): ] # None when not provided; resolved to enum in validate() is_prefill_worker: bool is_decode_worker: bool + enable_conditional_disagg: bool use_vllm_tokenizer: bool # Multimodal diff --git a/components/src/dynamo/vllm/handlers.py b/components/src/dynamo/vllm/handlers.py index 70971b866a35..de5786fa4798 100644 --- a/components/src/dynamo/vllm/handlers.py +++ b/components/src/dynamo/vllm/handlers.py @@ -101,6 +101,11 @@ configure_dynamo_logging() logger = logging.getLogger(__name__) +# Marker set by the Rust conditional-disagg bypass path. When present on a +# DECODE-mode worker, the request runs as local prefill+decode instead of +# expecting KV-transfer metadata from an upstream prefill worker. +BYPASS_REMOTE_PREFILL_ANNOTATION = "x-bypass-remote-prefill" + _GENERATE_REASONING_SUPPORT_CACHE_ATTR = "_dynamo_generate_reasoning_support" _DELTA_REQUEST_OUTPUT_KIND = RequestOutputKind.DELTA _RL_INIT_WEIGHTS_TIMEOUT_ENV = "DYN_RL_INIT_WEIGHTS_TIMEOUT_S" @@ -2998,6 +3003,15 @@ async def _generate_token_mode(self, request, context, request_id): mode = cast(DisaggregationMode, self.config.disaggregation_mode) is_decode_only = mode == DisaggregationMode.DECODE + if is_decode_only and BYPASS_REMOTE_PREFILL_ANNOTATION in ( + request.get("annotations") or [] + ): + logger.debug( + "DECODE: conditional-disagg bypass annotation present; " + "running request as AGG (prefill+decode on this worker)." + ) + is_decode_only = False + mode = DisaggregationMode.AGGREGATED has_mm_data = request.get("multi_modal_data") is not None mixed_embeds: tuple[torch.Tensor, list[int], list[bool]] | None = None @@ -3214,11 +3228,20 @@ async def _generate_text_mode(self, request, context, request_id): trace_headers = context.trace_headers() + is_decode_only = self.config.disaggregation_mode == DisaggregationMode.DECODE + if is_decode_only and BYPASS_REMOTE_PREFILL_ANNOTATION in ( + request.get("annotations") or [] + ): + logger.debug( + "DECODE: conditional-disagg bypass annotation present; " + "running text-mode request as AGG (prefill+decode on this worker)." + ) + is_decode_only = False + # Mirror _generate_token_mode: in disagg decode mode route aborts through # the per-request deferred guard so engine_client.abort() never fires in # the unsafe pre-first-token window, and the admin abort_request route can # reach this request via self._deferred_aborts. - is_decode_only = self.config.disaggregation_mode == DisaggregationMode.DECODE async with _deferred_abort_guard( self.engine_client, request_id, diff --git a/components/src/dynamo/vllm/main.py b/components/src/dynamo/vllm/main.py index 7ce04ab01f1a..16360e7295bd 100644 --- a/components/src/dynamo/vllm/main.py +++ b/components/src/dynamo/vllm/main.py @@ -367,9 +367,16 @@ def setup_kv_event_publisher( if not config.engine_args.enable_prefix_caching: return None - # Skip KV event publishing for decode workers - if config.disaggregation_mode == DisaggregationMode.DECODE: - logger.info("Skipping KV event publisher setup for decode worker") + # Decode workers do not publish KV events by default. Conditional disagg + # opts in because decode-side cache visibility is needed for bypass routing. + if ( + config.disaggregation_mode == DisaggregationMode.DECODE + and not config.enable_conditional_disagg + ): + logger.info( + "Skipping KV event publisher setup for decode worker " + "(pass --enable-conditional-disagg to opt in)" + ) return None if config.engine_args.kv_events_config is None: diff --git a/components/src/dynamo/vllm/tests/test_vllm_worker_handler.py b/components/src/dynamo/vllm/tests/test_vllm_worker_handler.py index bddca4415cd3..eba23f3dd546 100644 --- a/components/src/dynamo/vllm/tests/test_vllm_worker_handler.py +++ b/components/src/dynamo/vllm/tests/test_vllm_worker_handler.py @@ -10,6 +10,7 @@ import asyncio import base64 import json +from contextlib import asynccontextmanager from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch @@ -800,6 +801,111 @@ async def test_aggregated_mode_calls_extract_multimodal_data(self): assert len(chunks) == 1 assert chunks[0]["status"] == "error" + async def test_decode_only_bypass_annotation_runs_as_agg(self): + """Decode worker with conditional-disagg bypass annotation runs as AGG.""" + handler = _make_decode_handler( + model="Qwen/Qwen3-VL-2B-Instruct", + disaggregation_mode="DECODE", + ) + handler._multimodal_request_processor.extract_multimodal_data = AsyncMock( + return_value=None + ) + handler._build_prompt_from_request = MagicMock( + return_value=(None, None, {"status": "error", "message": "test stop"}) + ) + + request = { + "token_ids": [1, 2, 3], + "multi_modal_data": {"image_url": [{"Url": "http://img.png"}]}, + "sampling_options": {}, + "stop_conditions": {}, + "output_options": {}, + "annotations": [mod.BYPASS_REMOTE_PREFILL_ANNOTATION], + } + context = MagicMock() + + chunks = [] + async for chunk in handler._generate_token_mode(request, context, "req-1"): + chunks.append(chunk) + + handler._multimodal_request_processor.extract_multimodal_data.assert_awaited_once() + assert len(chunks) == 1 + assert chunks[0]["message"] == "test stop" + + async def test_decode_only_bypass_annotation_text_only_does_not_require_prefill_kv_params( + self, + ): + """Text-only bypass does not require incoming prefill KV params.""" + handler = _make_decode_handler(disaggregation_mode="DECODE") + handler._build_prompt_from_request = MagicMock( + return_value=(None, None, {"status": "error", "message": "stop"}) + ) + + request = { + "token_ids": [1, 2, 3], + "sampling_options": {}, + "stop_conditions": {}, + "output_options": {}, + "annotations": [mod.BYPASS_REMOTE_PREFILL_ANNOTATION], + } + context = MagicMock() + + chunks = [] + async for chunk in handler._generate_token_mode(request, context, "req-1"): + chunks.append(chunk) + + assert len(chunks) == 1 + assert chunks[0]["message"] == "stop" + + async def test_decode_only_bypass_annotation_text_mode_runs_as_agg(self): + """Text-mode conditional-disagg bypass is not treated as decode-only.""" + handler = _make_decode_handler(disaggregation_mode="DECODE") + handler.input_param_manager = MagicMock() + handler.input_param_manager.get_input_param.return_value = [1, 2, 3] + handler.engine_client = MagicMock() + handler.default_sampling_params = {} + + async def _empty_generate(*args, **kwargs): + if False: + yield None + + handler.engine_client.generate = _empty_generate + + killed_future = asyncio.get_event_loop().create_future() + killed_future.set_result(None) + context = MagicMock() + context.async_killed_or_stopped.return_value = killed_future + context.trace_headers.return_value = {} + + decode_only_values = [] + + @asynccontextmanager + async def _capture_guard( + engine_client, + request_id, + is_decode_only, + registry=None, + on_engine_dead=None, + ): + decode_only_values.append(is_decode_only) + yield None + + request = { + "token_ids": [1, 2, 3], + "sampling_options": {}, + "stop_conditions": {}, + "output_options": {}, + "annotations": [mod.BYPASS_REMOTE_PREFILL_ANNOTATION], + } + + with patch.object(mod, "_deferred_abort_guard", _capture_guard): + chunks = [] + async for chunk in handler._generate_text_mode(request, context, "req-1"): + chunks.append(chunk) + + assert chunks == [] + assert decode_only_values == [False] + @pytest.mark.parametrize( "mm_processor_kwargs", [None, {"use_audio_in_video": True}], diff --git a/deploy/inference-gateway/ext-proc/src/epp.rs b/deploy/inference-gateway/ext-proc/src/epp.rs index 97fd9c4447e2..a5a414477e93 100644 --- a/deploy/inference-gateway/ext-proc/src/epp.rs +++ b/deploy/inference-gateway/ext-proc/src/epp.rs @@ -174,6 +174,7 @@ impl Router { Some(prefill_config), None, None, + None, model_name.clone(), actual_namespace.to_string(), enable_eagle, diff --git a/docs/backends/trtllm/README.md b/docs/backends/trtllm/README.md index 9b9f52e6e517..c56d44fc80c0 100644 --- a/docs/backends/trtllm/README.md +++ b/docs/backends/trtllm/README.md @@ -20,7 +20,7 @@ Dynamo TensorRT-LLM integrates [TensorRT-LLM](https://github.com/NVIDIA/TensorRT | Feature | TensorRT-LLM | Notes | |---------|--------------|-------| | [**Disaggregated Serving**](../../design-docs/disagg-serving.md) | ✅ | | -| [**Conditional Disaggregation**](../../design-docs/disagg-serving.md) | 🚧 | Not supported yet | +| [**Conditional Disaggregation**](../../components/router/router-disaggregated-serving.md#conditional-disaggregation) | ✅ | Requires KV-aware routing and `--publish-kv-events` on prefill and decode workers | | [**KV-Aware Routing**](../../components/router/README.md) | ✅ | | | [**SLA-Based Planner**](../../components/planner/planner-guide.md) | ✅ | | | [**Load Based Planner**](../../components/planner/README.md) | 🚧 | Planned | diff --git a/docs/components/router/router-configuration.md b/docs/components/router/router-configuration.md index a02f2e12527c..0248ff175252 100644 --- a/docs/components/router/router-configuration.md +++ b/docs/components/router/router-configuration.md @@ -21,6 +21,7 @@ This page collects the main router flags for frontend-embedded and standalone de - `--router-queue-threshold`: Optional queue threshold fraction for prefill token capacity. Queueing is disabled by default; setting a numeric value enables it. The router holds incoming requests in a priority queue while all eligible workers exceed `threshold * max_num_batched_tokens`, releasing them when capacity frees up. This defers dispatch rather than rejecting work, so routing decisions use the freshest load metrics at the moment a request is sent to a worker. `nvext.agent_hints.strict_priority` selects an absolute pending-queue tier, while `nvext.agent_hints.priority` adjusts ordering within the configured policy. Must be greater than or equal to 0; use `0.0` for maximum queueing sensitivity. See the SGLang note under [Tuning Guidelines](#tuning-guidelines) for caveats around how `max_num_batched_tokens` is populated on that backend, and see [Priority Scheduling](priority-scheduling.md) for how router priority differs from backend engine priority. - `--router-queue-policy`: Scheduling policy for the router queue (default: `fcfs`). - `--router-policy-config`: Startup-only policy-family and cache-bucket YAML path. When omitted, `--router-queue-threshold` and `--router-queue-policy` define one synthetic policy class. The equivalent environment variable is `DYN_ROUTER_POLICY_CONFIG`. +- `--router-conditional-disagg`: **Experimental.** Enables conditional disaggregation in frontend-embedded disaggregated serving. The prefill router can bypass the remote prefill worker and route directly to a decode worker when the configured policy says local prefill+decode is preferable. Requires `--router-mode kv`, separate prefill/decode worker pools, and decode-worker KV event publishing. See [Conditional Disaggregation](router-disaggregated-serving.md#conditional-disaggregation) for backend requirements, policy flags, and threshold flags. For how queue backpressure differs from candidate filtering and busy-threshold overload handling, see [Router Filtering](router-filtering.md). diff --git a/docs/components/router/router-disaggregated-serving.md b/docs/components/router/router-disaggregated-serving.md index 96328723393f..a8d82f4b639f 100644 --- a/docs/components/router/router-disaggregated-serving.md +++ b/docs/components/router/router-disaggregated-serving.md @@ -96,3 +96,61 @@ graph TD ``` When topology-aware KV transfer is enabled, the prefill router also derives decode `RoutingConstraints` from the selected prefill worker's runtime topology metadata before the request enters the decode router. + +## Conditional Disaggregation + +> [!WARNING] +> **Experimental.** Validate/tune conditional disaggregation against your workload before using it in production. + +Conditional disaggregation is a feature that enables a hybrid of aggregated and disaggregated request routing. The router may serve a request `prefill worker -> decode worker`, or it may send the request directly to a decode worker and the backend runs local prefill plus decode there. + +For workloads with a high degree of KV reusage on long-ISL requests, e.g. multi-turn / agentic conversation scenarios, conditional disaggregation can help your deployment maintain predictable SLA. Compared to unconditional disaggregation, it reduces memory pressure / TTFT on prefill workers by optimizing reuse of *decode-worker* KV cache; compared to unconditionally aggregated deployments, it avoids the heavy ITL penalty incurred by co-scheduling heavy prefill workload onto decode workers. + +Enable conditional disaggregation with `--router-conditional-disagg` on the frontend: + +```bash +python -m dynamo.frontend \ + --router-mode kv \ + --router-conditional-disagg \ + --router-conditional-disagg-policy isl_bounding +``` + +### Backend Requirements + +Conditional disaggregation requires decode-worker KV visibility. The router uses decode-side KV events to estimate effective ISL and decide whether local prefill+decode is cheaper than remote prefill. + +Configure workers as follows: + +| Backend | Requirement | +| --- | --- | +| vLLM | On decode workers, pass `--enable-conditional-disagg`. This opts decode workers into KV event publication and bypass handling. | +| TensorRT-LLM | Pass `--publish-kv-events` on prefill AND decode workers to opt them into KV-aware routing. | +| SGLang | Not supported yet. Dynamo rejects SGLang conditional-disagg setups until the backend can run bypassed decode requests as local prefill+decode. | + +If decode workers do not publish KV events, the router cannot accurately assess bypass conditions. + +Append these additional flags to tune the conditional disaggregation policy: + +> [!NOTE] +> We recommend tuning these values against your workload and deployment configuration. + +For ISL-based policies, `effective ISL` is the request prompt length after subtracting the selected decode worker's cached prefix overlap. The absolute threshold limits the number of prompt tokens the decode worker may need to compute locally. The ratio threshold limits that local work as a fraction of the raw prompt length. These thresholds measure both "how much compute does this request require" (absolute) and "how compute/memory-bound is this request, due to the ratio of its computed / cached KV cache" (ratio), respectively. + +As a tuning starting point, we recommend choosing thresholds based on your workload's expected effective-ISL (and effective ISL / raw ISL ratio) distribution. For example, with `isl_bounding`, setting the absolute threshold to p25 of the workload's effective ISL would make the absolute-threshold check pass for roughly 25% of requests. + +| Flag | Default | Use | +| ----------------------------------------------------- | -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--router-conditional-disagg` | Disabled | Enables conditional disaggregation. Requires `--router-mode kv` and separate prefill/decode worker pools. | +| `--router-conditional-disagg-policy` | `isl_bounding` | Selects the bypass policy: `isl_bounding`, `prefill_load`, or `isl_or_load`. | +| `--router-conditional-disagg-eff-isl-threshold` | `2048` | For `isl_bounding` and the ISL arm of `isl_or_load`, require effective ISL to be below this many tokens. | +| `--router-conditional-disagg-eff-isl-ratio-threshold` | `0.7` | For `isl_bounding` and the ISL arm of `isl_or_load`, require `effective ISL / raw ISL` to be below this value. Must be in `[0.0, 1.0]`. | +| `--router-conditional-disagg-prefill-busy-threshold` | Unset | Sets the prefill busy threshold for `prefill_load` and `isl_or_load`. When unset, those policies inherit `--router-queue-threshold` if it is set. | +| `--router-conditional-disagg-decode-busy-threshold` | Unset | Decode-busy guard. When unset, the guard is disabled. When set, conditional-disagg bypass is disabled if the selected decode worker's projected decode load exceeds this fraction of KV capacity. | + +The available policies are: + +| Policy | Bypass Condition | +| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `isl_bounding` | Effective ISL is below `--router-conditional-disagg-eff-isl-threshold` AND the effective/raw ISL ratio is below `--router-conditional-disagg-eff-isl-ratio-threshold`. | +| `prefill_load` | The selected prefill worker is above the configured prefill busy threshold. | +| `isl_or_load` | Either the `isl_bounding` condition OR the `prefill_load` condition is true. | diff --git a/docs/reference/glossary.md b/docs/reference/glossary.md index 9cdebef72d8e..770c96300a3b 100644 --- a/docs/reference/glossary.md +++ b/docs/reference/glossary.md @@ -11,7 +11,7 @@ subtitle: Definitions of core Dynamo terms covering disaggregated serving, KV ca ## C **Component** - The fundamental deployable unit in Dynamo. A discoverable service entity that can host multiple endpoints and typically maps to a Docker container (such as VllmWorker, Router, Processor). -**Conditional Disaggregation** - Dynamo's intelligent decision-making process within disaggregated serving that determines whether a request is processed locally or sent to a remote prefill engine based on prefill length and queue status. +**Conditional Disaggregation** - Dynamo's router policy within disaggregated serving that decides whether a request should use remote prefill or run local prefill+decode on a decode worker based on decode-side KV overlap, effective ISL, and prefill/decode load signals. ## D **Decode Phase** - The second phase of LLM inference that generates output tokens one at a time. diff --git a/lib/bindings/c/src/lib.rs b/lib/bindings/c/src/lib.rs index 9783e74a7ad0..198e6fc89627 100644 --- a/lib/bindings/c/src/lib.rs +++ b/lib/bindings/c/src/lib.rs @@ -833,6 +833,7 @@ pub unsafe extern "C" fn create_routers( Some(prefill_config), None, None, + None, model_name.clone(), actual_namespace.clone(), enable_eagle, diff --git a/lib/bindings/python/rust/llm/entrypoint.rs b/lib/bindings/python/rust/llm/entrypoint.rs index d41ef73e4964..868ee3b6f134 100644 --- a/lib/bindings/python/rust/llm/entrypoint.rs +++ b/lib/bindings/python/rust/llm/entrypoint.rs @@ -14,6 +14,7 @@ use pyo3::{ use pyo3_async_runtimes::TaskLocals; use dynamo_kv_router::config::{ + ConditionalDisaggPolicyKind as RsConditionalDisaggPolicyKind, KvRouterConfig as RsKvRouterConfig, RouterPrefillLoadModel as RsRouterPrefillLoadModel, apply_deprecated_overlap_score_weight_override, }; @@ -237,7 +238,7 @@ impl AicPerfConfig { #[pymethods] impl KvRouterConfig { #[new] - #[pyo3(signature = (overlap_score_weight=None, host_cache_hit_weight=0.75, disk_cache_hit_weight=0.25, router_temperature=0.0, use_kv_events=true, durable_kv_events=false, router_replica_sync=false, router_track_active_blocks=true, router_track_output_blocks=false, router_assume_kv_reuse=true, router_track_prefill_tokens=true, router_prefill_load_model="none", router_snapshot_threshold=1000000, router_reset_states=false, router_ttl_secs=120.0, router_queue_threshold=None, router_event_threads=4, router_queue_policy="fcfs", use_remote_indexer=false, serve_indexer=false, shared_cache_multiplier=0.0, shared_cache_type="none", router_predicted_ttl_secs=None, *, overlap_score_credit=1.0, overlap_score_credit_decay=0.0, prefill_load_scale=1.0, router_policy_config=None))] + #[pyo3(signature = (overlap_score_weight=None, host_cache_hit_weight=0.75, disk_cache_hit_weight=0.25, router_temperature=0.0, use_kv_events=true, durable_kv_events=false, router_replica_sync=false, router_track_active_blocks=true, router_track_output_blocks=false, router_assume_kv_reuse=true, router_track_prefill_tokens=true, router_prefill_load_model="none", router_snapshot_threshold=1000000, router_reset_states=false, router_ttl_secs=120.0, router_queue_threshold=None, router_event_threads=4, router_queue_policy="fcfs", use_remote_indexer=false, serve_indexer=false, shared_cache_multiplier=0.0, shared_cache_type="none", router_predicted_ttl_secs=None, *, conditional_disagg_enabled=false, conditional_disagg_policy="isl_bounding", conditional_disagg_eff_isl_threshold=2048, conditional_disagg_eff_isl_ratio_threshold=0.7, conditional_disagg_prefill_busy_threshold=None, conditional_disagg_decode_busy_threshold=None, overlap_score_credit=1.0, overlap_score_credit_decay=0.0, prefill_load_scale=1.0, router_policy_config=None))] #[allow(clippy::too_many_arguments)] fn new( overlap_score_weight: Option, @@ -263,6 +264,12 @@ impl KvRouterConfig { shared_cache_multiplier: f64, shared_cache_type: &str, router_predicted_ttl_secs: Option, + conditional_disagg_enabled: bool, + conditional_disagg_policy: &str, + conditional_disagg_eff_isl_threshold: usize, + conditional_disagg_eff_isl_ratio_threshold: f64, + conditional_disagg_prefill_busy_threshold: Option, + conditional_disagg_decode_busy_threshold: Option, mut overlap_score_credit: f64, overlap_score_credit_decay: f64, mut prefill_load_scale: f64, @@ -307,6 +314,14 @@ impl KvRouterConfig { serve_indexer, shared_cache_multiplier, shared_cache_type: shared_cache_type.parse().map_err(PyValueError::new_err)?, + conditional_disagg_enabled, + conditional_disagg_policy: conditional_disagg_policy + .parse::() + .map_err(PyValueError::new_err)?, + conditional_disagg_eff_isl_threshold, + conditional_disagg_eff_isl_ratio_threshold, + conditional_disagg_prefill_busy_threshold, + conditional_disagg_decode_busy_threshold, router_predicted_ttl_secs, }; validate_kv_router_config(&inner)?; diff --git a/lib/bindings/python/src/dynamo/_core.pyi b/lib/bindings/python/src/dynamo/_core.pyi index 1aee835ad579..9a3f79a6270c 100644 --- a/lib/bindings/python/src/dynamo/_core.pyi +++ b/lib/bindings/python/src/dynamo/_core.pyi @@ -1866,6 +1866,12 @@ class KvRouterConfig: serve_indexer: bool = False, shared_cache_multiplier: float = 0.0, shared_cache_type: str = "none", + conditional_disagg_enabled: bool = False, + conditional_disagg_policy: str = "isl_bounding", + conditional_disagg_eff_isl_threshold: int = 2048, + conditional_disagg_eff_isl_ratio_threshold: float = 0.7, + conditional_disagg_prefill_busy_threshold: Optional[float] = None, + conditional_disagg_decode_busy_threshold: Optional[float] = None, router_predicted_ttl_secs: Optional[float] = None, *, overlap_score_credit: float = 1.0, @@ -1920,6 +1926,12 @@ class KvRouterConfig: serve_indexer: Serve this router's local indexer from the worker component (default: False). shared_cache_multiplier: Credit multiplier for shared cache hits beyond the device prefix (default: 0.0). shared_cache_type: External shared KV cache type, "none" or "hicache" (default: "none"). + conditional_disagg_enabled: Enable conditional-disagg bypass from prefill to decode (default: False). + conditional_disagg_policy: Conditional-disagg policy, one of "isl_bounding", "prefill_load", or "isl_or_load" (default: "isl_bounding"). + conditional_disagg_eff_isl_threshold: For "isl_bounding" and the ISL arm of "isl_or_load", require effective ISL to be below this many tokens (default: 2048). + conditional_disagg_eff_isl_ratio_threshold: For "isl_bounding" and the ISL arm of "isl_or_load", require effective ISL / raw ISL to be below this value (default: 0.7). + conditional_disagg_prefill_busy_threshold: Prefill busy threshold for load-aware conditional-disagg policies. When omitted, inherits router_queue_threshold when available. + conditional_disagg_decode_busy_threshold: Decode-busy guard threshold that disables bypass when the selected decode worker's projected decode load exceeds this fraction of KV capacity (default: None). router_predicted_ttl_secs: Enables predict-on-route when set. This TTL applies to entries in the local side indexer and requires use_kv_events=True. Set to None to disable. Independent of diff --git a/lib/kv-router/src/conditional_disagg.rs b/lib/kv-router/src/conditional_disagg.rs new file mode 100644 index 000000000000..4264b12f2b69 --- /dev/null +++ b/lib/kv-router/src/conditional_disagg.rs @@ -0,0 +1,579 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Conditional-disagg bypass policy. +//! +//! Decides whether a request should skip remote prefill and run prefill +//! locally on the chosen decode worker. The trait operates over summary +//! signals so future policies can plug in without changing the router call site. + +use async_trait::async_trait; + +use crate::config::{ConditionalDisaggPolicyKind, KvRouterConfig}; + +/// Default effective-ISL absolute threshold (tokens). A request bypasses to +/// AGG only if its net-new prefill stays under this cap. +pub const DEFAULT_CONDITIONAL_DISAGG_EFF_ISL_THRESHOLD: usize = 2048; + +/// Default effective-ISL ratio threshold. A request bypasses to AGG only if +/// `eff_isl / prompt_tokens` stays under this fraction. +pub const DEFAULT_CONDITIONAL_DISAGG_EFF_ISL_RATIO_THRESHOLD: f64 = 0.7; + +/// Inputs passed to a `ConditionalDisaggPolicy` when deciding whether to +/// bypass remote prefill. +#[derive(Debug, Clone, Copy, PartialEq)] +#[non_exhaustive] +pub struct ConditionalDisaggDecisionInput { + /// Total prompt token count. + pub prompt_tokens: usize, + + /// Effective cache credit on the chosen decode worker, in weighted tokens. + pub decode_chosen_cached_tokens: usize, + + /// Whether the prefill worker the router would pick for this request is + /// over the prefill-busy line. `None` means the signal is unavailable. + pub prefill_chosen_worker_busy: Option, + + /// Whether the decode worker the router would pick for this request is + /// over the decode-busy line. `None` means the gate is disabled or the + /// signal is unavailable. + pub decode_chosen_worker_busy: Option, +} + +impl ConditionalDisaggDecisionInput { + pub fn new(prompt_tokens: usize, decode_chosen_cached_tokens: usize) -> Self { + Self { + prompt_tokens, + decode_chosen_cached_tokens, + prefill_chosen_worker_busy: None, + decode_chosen_worker_busy: None, + } + } + + pub fn with_prefill_chosen_worker_busy(mut self, busy: Option) -> Self { + self.prefill_chosen_worker_busy = busy; + self + } + + pub fn with_decode_chosen_worker_busy(mut self, busy: Option) -> Self { + self.decode_chosen_worker_busy = busy; + self + } + + /// Effective net-new prefill in tokens after the decode-side weighted + /// cache hit is subtracted. + pub fn net_new_tokens(self) -> usize { + self.prompt_tokens + .saturating_sub(self.decode_chosen_cached_tokens.min(self.prompt_tokens)) + } +} + +#[async_trait] +pub trait ConditionalDisaggPolicy: Send + Sync { + fn is_enabled(&self) -> bool; + + /// Decide whether the request should skip remote prefill. + async fn should_bypass_remote_prefill(&self, input: ConditionalDisaggDecisionInput) -> bool; + + /// True iff this policy consumes + /// [`ConditionalDisaggDecisionInput::prefill_chosen_worker_busy`]. + fn needs_prefill_worker_busy(&self) -> bool { + false + } +} + +/// Build the configured conditional-disagg policy. Returns a disabled policy +/// when conditional disagg is not enabled. +pub fn make_conditional_disagg_policy( + config: Option<&KvRouterConfig>, +) -> Box { + let Some(config) = config else { + return Box::new(IslBoundingPolicy::disabled()); + }; + match config.conditional_disagg_policy { + ConditionalDisaggPolicyKind::IslBounding => { + Box::new(IslBoundingPolicy::from_config(config)) + } + ConditionalDisaggPolicyKind::PrefillLoad => { + Box::new(PrefillLoadPolicy::from_config(config)) + } + ConditionalDisaggPolicyKind::IslOrLoad => Box::new(IslOrLoadPolicy::from_config(config)), + } +} + +pub fn policy_needs_prefill_worker_busy(config: Option<&KvRouterConfig>) -> bool { + make_conditional_disagg_policy(config).needs_prefill_worker_busy() +} + +/// v1 conditional-disagg policy. Bypasses to AGG when the request is both +/// small in absolute net-new prefill and mostly cached on the decode worker. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct IslBoundingPolicy { + enabled: bool, + eff_isl_threshold: usize, + eff_isl_ratio_threshold: f64, +} + +impl IslBoundingPolicy { + pub fn new(enabled: bool, eff_isl_threshold: usize, eff_isl_ratio_threshold: f64) -> Self { + Self { + enabled, + eff_isl_threshold, + eff_isl_ratio_threshold, + } + } + + pub fn from_config(config: &KvRouterConfig) -> Self { + Self { + enabled: config.conditional_disagg_enabled, + eff_isl_threshold: config.conditional_disagg_eff_isl_threshold, + eff_isl_ratio_threshold: config.conditional_disagg_eff_isl_ratio_threshold, + } + } + + pub fn disabled() -> Self { + Self { + enabled: false, + eff_isl_threshold: DEFAULT_CONDITIONAL_DISAGG_EFF_ISL_THRESHOLD, + eff_isl_ratio_threshold: DEFAULT_CONDITIONAL_DISAGG_EFF_ISL_RATIO_THRESHOLD, + } + } +} + +impl Default for IslBoundingPolicy { + fn default() -> Self { + Self::disabled() + } +} + +#[async_trait] +impl ConditionalDisaggPolicy for IslBoundingPolicy { + fn is_enabled(&self) -> bool { + self.enabled + } + + async fn should_bypass_remote_prefill(&self, input: ConditionalDisaggDecisionInput) -> bool { + if !self.enabled { + return false; + } + let eff_isl = input.net_new_tokens(); + if eff_isl >= self.eff_isl_threshold { + return false; + } + let denom = input.prompt_tokens.max(1) as f64; + let ratio = eff_isl as f64 / denom; + ratio < self.eff_isl_ratio_threshold + } +} + +/// v1.5 conditional-disagg policy. Bypasses to AGG when the prefill worker +/// the router would pick for this request is already over the prefill-busy line. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PrefillLoadPolicy { + enabled: bool, +} + +impl PrefillLoadPolicy { + pub fn new(enabled: bool) -> Self { + Self { enabled } + } + + pub fn from_config(config: &KvRouterConfig) -> Self { + Self { + enabled: config.conditional_disagg_enabled, + } + } + + pub fn disabled() -> Self { + Self { enabled: false } + } +} + +impl Default for PrefillLoadPolicy { + fn default() -> Self { + Self::disabled() + } +} + +#[async_trait] +impl ConditionalDisaggPolicy for PrefillLoadPolicy { + fn is_enabled(&self) -> bool { + self.enabled + } + + async fn should_bypass_remote_prefill(&self, input: ConditionalDisaggDecisionInput) -> bool { + if !self.enabled { + return false; + } + input.prefill_chosen_worker_busy.unwrap_or(false) + } + + fn needs_prefill_worker_busy(&self) -> bool { + self.enabled + } +} + +/// v1.5 composition policy: bypass when either the ISL policy or the load +/// policy wants bypass. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct IslOrLoadPolicy { + isl: IslBoundingPolicy, + load: PrefillLoadPolicy, +} + +impl IslOrLoadPolicy { + pub fn new(isl: IslBoundingPolicy, load: PrefillLoadPolicy) -> Self { + Self { isl, load } + } + + pub fn from_config(config: &KvRouterConfig) -> Self { + Self { + isl: IslBoundingPolicy::from_config(config), + load: PrefillLoadPolicy::from_config(config), + } + } + + pub fn disabled() -> Self { + Self { + isl: IslBoundingPolicy::disabled(), + load: PrefillLoadPolicy::disabled(), + } + } +} + +impl Default for IslOrLoadPolicy { + fn default() -> Self { + Self::disabled() + } +} + +#[async_trait] +impl ConditionalDisaggPolicy for IslOrLoadPolicy { + fn is_enabled(&self) -> bool { + self.isl.is_enabled() || self.load.is_enabled() + } + + async fn should_bypass_remote_prefill(&self, input: ConditionalDisaggDecisionInput) -> bool { + self.isl.should_bypass_remote_prefill(input).await + || self.load.should_bypass_remote_prefill(input).await + } + + fn needs_prefill_worker_busy(&self) -> bool { + self.isl.needs_prefill_worker_busy() || self.load.needs_prefill_worker_busy() + } +} + +#[cfg(test)] +#[derive(Debug, Clone, Copy)] +pub struct RandomBypassConditionalDisaggPolicy { + enabled: bool, + bypass_probability: f64, +} + +#[cfg(test)] +impl RandomBypassConditionalDisaggPolicy { + pub fn new(enabled: bool, bypass_probability: f64) -> Self { + Self { + enabled, + bypass_probability: bypass_probability.clamp(0.0, 1.0), + } + } +} + +#[cfg(test)] +#[async_trait] +impl ConditionalDisaggPolicy for RandomBypassConditionalDisaggPolicy { + fn is_enabled(&self) -> bool { + self.enabled + } + + async fn should_bypass_remote_prefill(&self, _input: ConditionalDisaggDecisionInput) -> bool { + if !self.enabled { + return false; + } + rand::random::() < self.bypass_probability + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn input(prompt_tokens: usize, cached_tokens: usize) -> ConditionalDisaggDecisionInput { + ConditionalDisaggDecisionInput::new(prompt_tokens, cached_tokens) + } + + fn input_from_blocks( + prompt_tokens: usize, + overlap_blocks: u32, + block_size: usize, + ) -> ConditionalDisaggDecisionInput { + input( + prompt_tokens, + (overlap_blocks as usize).saturating_mul(block_size), + ) + } + + #[tokio::test] + async fn disabled_never_bypasses() { + let policy = IslBoundingPolicy::new(false, 2048, 0.7); + assert!( + !policy + .should_bypass_remote_prefill(input_from_blocks(100, 0, 64)) + .await + ); + } + + #[tokio::test] + async fn small_and_mostly_cached_bypasses() { + let policy = IslBoundingPolicy::new(true, 2048, 0.7); + assert!( + policy + .should_bypass_remote_prefill(input_from_blocks(1000, 14, 64)) + .await + ); + } + + #[tokio::test] + async fn large_eff_isl_does_not_bypass_even_if_ratio_low() { + let policy = IslBoundingPolicy::new(true, 2048, 0.99); + assert!( + !policy + .should_bypass_remote_prefill(input_from_blocks(100_000, 1000, 64)) + .await + ); + } + + #[tokio::test] + async fn small_eff_isl_but_ratio_at_or_above_threshold_does_not_bypass() { + let policy = IslBoundingPolicy::new(true, 2048, 0.7); + assert!( + !policy + .should_bypass_remote_prefill(input_from_blocks(200, 0, 64)) + .await + ); + } + + #[tokio::test] + async fn boundary_eff_isl_equals_threshold_does_not_bypass() { + let policy = IslBoundingPolicy::new(true, 2048, 0.99); + assert!( + !policy + .should_bypass_remote_prefill(input_from_blocks(2048, 0, 64)) + .await + ); + } + + #[tokio::test] + async fn zero_prompt_tokens_does_not_panic() { + let policy = IslBoundingPolicy::new(true, 2048, 0.7); + assert!( + policy + .should_bypass_remote_prefill(input_from_blocks(0, 0, 64)) + .await + ); + } + + #[tokio::test] + async fn overlap_exceeding_prompt_clamps_to_zero_eff_isl() { + let policy = IslBoundingPolicy::new(true, 2048, 0.7); + assert!( + policy + .should_bypass_remote_prefill(input_from_blocks(500, 10, 64)) + .await + ); + } + + #[tokio::test] + async fn weighted_cached_tokens_do_not_round_up_to_full_block() { + let policy = IslBoundingPolicy::new(true, 2048, 0.99); + assert!(!policy.should_bypass_remote_prefill(input(2100, 48)).await); + } + + #[tokio::test] + async fn random_bypass_when_disabled_never_bypasses() { + let policy = RandomBypassConditionalDisaggPolicy::new(false, 1.0); + assert!( + !policy + .should_bypass_remote_prefill(input_from_blocks(100, 0, 64)) + .await + ); + } + + #[tokio::test] + async fn random_bypass_zero_probability_never_bypasses() { + let policy = RandomBypassConditionalDisaggPolicy::new(true, 0.0); + for _ in 0..50 { + assert!( + !policy + .should_bypass_remote_prefill(input_from_blocks(100, 0, 64)) + .await + ); + } + } + + #[tokio::test] + async fn random_bypass_one_probability_always_bypasses() { + let policy = RandomBypassConditionalDisaggPolicy::new(true, 1.0); + for _ in 0..50 { + assert!( + policy + .should_bypass_remote_prefill(input_from_blocks(100, 0, 64)) + .await + ); + } + } + + fn input_with_busy( + prompt_tokens: usize, + overlap_blocks: u32, + block_size: usize, + busy: Option, + ) -> ConditionalDisaggDecisionInput { + input_from_blocks(prompt_tokens, overlap_blocks, block_size) + .with_prefill_chosen_worker_busy(busy) + } + + #[tokio::test] + async fn prefill_load_disabled_never_bypasses() { + let policy = PrefillLoadPolicy::new(false); + assert!( + !policy + .should_bypass_remote_prefill(input_with_busy(1000, 0, 64, Some(true))) + .await + ); + } + + #[tokio::test] + async fn prefill_load_signal_none_does_not_bypass() { + let policy = PrefillLoadPolicy::new(true); + assert!( + !policy + .should_bypass_remote_prefill(input_with_busy(1000, 0, 64, None)) + .await + ); + } + + #[tokio::test] + async fn prefill_load_worker_calm_does_not_bypass() { + let policy = PrefillLoadPolicy::new(true); + assert!( + !policy + .should_bypass_remote_prefill(input_with_busy(1000, 0, 64, Some(false))) + .await + ); + } + + #[tokio::test] + async fn prefill_load_worker_busy_bypasses() { + let policy = PrefillLoadPolicy::new(true); + assert!( + policy + .should_bypass_remote_prefill(input_with_busy(1000, 0, 64, Some(true))) + .await + ); + } + + #[tokio::test] + async fn prefill_load_independent_of_isl_fields() { + let policy = PrefillLoadPolicy::new(true); + assert!( + policy + .should_bypass_remote_prefill(input_with_busy(100_000, 0, 64, Some(true))) + .await + ); + assert!( + !policy + .should_bypass_remote_prefill(input_with_busy(100_000, 0, 64, Some(false))) + .await + ); + } + + fn small_input(busy: Option) -> ConditionalDisaggDecisionInput { + input_with_busy(1000, 14, 64, busy) + } + + fn large_input(busy: Option) -> ConditionalDisaggDecisionInput { + input_with_busy(100_000, 0, 64, busy) + } + + fn enabled_or_policy() -> IslOrLoadPolicy { + IslOrLoadPolicy::new( + IslBoundingPolicy::new(true, 2048, 0.7), + PrefillLoadPolicy::new(true), + ) + } + + #[tokio::test] + async fn isl_or_load_small_and_calm_bypasses_via_isl() { + assert!( + enabled_or_policy() + .should_bypass_remote_prefill(small_input(Some(false))) + .await + ); + } + + #[tokio::test] + async fn isl_or_load_small_and_busy_bypasses() { + assert!( + enabled_or_policy() + .should_bypass_remote_prefill(small_input(Some(true))) + .await + ); + } + + #[tokio::test] + async fn isl_or_load_large_and_calm_does_not_bypass() { + assert!( + !enabled_or_policy() + .should_bypass_remote_prefill(large_input(Some(false))) + .await + ); + } + + #[tokio::test] + async fn isl_or_load_large_and_busy_bypasses_via_load() { + assert!( + enabled_or_policy() + .should_bypass_remote_prefill(large_input(Some(true))) + .await + ); + } + + #[tokio::test] + async fn isl_or_load_disabled_never_bypasses() { + let policy = IslOrLoadPolicy::disabled(); + assert!( + !policy + .should_bypass_remote_prefill(small_input(Some(true))) + .await + ); + assert!( + !policy + .should_bypass_remote_prefill(large_input(Some(true))) + .await + ); + } + + #[tokio::test] + async fn isl_or_load_signal_none_falls_back_to_isl_only() { + let policy = enabled_or_policy(); + assert!(policy.should_bypass_remote_prefill(small_input(None)).await); + assert!(!policy.should_bypass_remote_prefill(large_input(None)).await); + } + + #[tokio::test] + async fn decision_input_new_defaults_busy_to_none() { + let input = ConditionalDisaggDecisionInput::new(1000, 0); + assert_eq!(input.prefill_chosen_worker_busy, None); + assert_eq!(input.decode_chosen_worker_busy, None); + } + + #[tokio::test] + async fn decision_input_with_decode_chosen_worker_busy_round_trips() { + let input = + ConditionalDisaggDecisionInput::new(1000, 0).with_decode_chosen_worker_busy(Some(true)); + assert_eq!(input.decode_chosen_worker_busy, Some(true)); + assert_eq!(input.prefill_chosen_worker_busy, None); + } +} diff --git a/lib/kv-router/src/lib.rs b/lib/kv-router/src/lib.rs index c6574dc3a6ed..0e9dbc62e9ac 100644 --- a/lib/kv-router/src/lib.rs +++ b/lib/kv-router/src/lib.rs @@ -8,6 +8,7 @@ mod active_set; pub(crate) mod cleanup; +pub mod conditional_disagg; mod lookup_update; pub mod indexer; @@ -44,8 +45,8 @@ pub use self::sequences::{PrefillTokenDeltas, WorkerLoadProjection}; pub use concurrent_radix_tree::ConcurrentRadixTree; pub use concurrent_radix_tree_compressed::ConcurrentRadixTreeCompressed; pub use config::{ - KvRouterConfig, RouterConfigOverride, RouterPrefillLoadModel, RouterQueuePolicy, - SharedCacheType, + ConditionalDisaggPolicyKind, KvRouterConfig, RouterConfigOverride, RouterPrefillLoadModel, + RouterQueuePolicy, SharedCacheType, }; #[allow(deprecated)] pub use indexer::{ diff --git a/lib/kv-router/src/protocols.rs b/lib/kv-router/src/protocols.rs index 9f2fbd79777b..9f01268c9ef1 100644 --- a/lib/kv-router/src/protocols.rs +++ b/lib/kv-router/src/protocols.rs @@ -547,6 +547,10 @@ pub struct WorkerSelectionResult { /// Approximate cached-token count derived from the weighted cache hit. pub cached_tokens: usize, + + /// Selected worker's projected decode load after adding this request's + /// prompt blocks, in scheduler-tracked block units. + pub potential_decode_blocks: usize, } /// Active load metrics for a worker, used for overload detection. diff --git a/lib/kv-router/src/scheduling/config.rs b/lib/kv-router/src/scheduling/config.rs index 8a2c41006640..2643021187b6 100644 --- a/lib/kv-router/src/scheduling/config.rs +++ b/lib/kv-router/src/scheduling/config.rs @@ -106,6 +106,12 @@ pub fn kv_router_config_from_dynamo_env() -> KvRouterConfig { router_track_prefill_tokens = config.router_track_prefill_tokens, router_queue_threshold = ?config.router_queue_threshold, router_policy_config = ?config.router_policy_config, + conditional_disagg_enabled = config.conditional_disagg_enabled, + conditional_disagg_policy = ?config.conditional_disagg_policy, + conditional_disagg_eff_isl_threshold = config.conditional_disagg_eff_isl_threshold, + conditional_disagg_eff_isl_ratio_threshold = config.conditional_disagg_eff_isl_ratio_threshold, + conditional_disagg_prefill_busy_threshold = ?config.conditional_disagg_prefill_busy_threshold, + conditional_disagg_decode_busy_threshold = ?config.conditional_disagg_decode_busy_threshold, router_predicted_ttl_secs = ?config.router_predicted_ttl_secs, "KvRouterConfig initialized (DYN_* env overrides applied)" ); @@ -117,6 +123,10 @@ fn kv_router_config_from_lookup(get_env: impl Fn(&str) -> Option) -> KvR get_env(key).and_then(|value| value.parse().ok()) } + fn parse_usize(get_env: &impl Fn(&str) -> Option, key: &str) -> Option { + get_env(key).and_then(|value| value.parse().ok()) + } + fn parse_bool(get_env: &impl Fn(&str) -> Option, key: &str) -> Option { get_env(key).and_then(|value| match value.to_ascii_lowercase().as_str() { "true" | "1" | "yes" | "on" => Some(true), @@ -174,6 +184,35 @@ fn kv_router_config_from_lookup(get_env: impl Fn(&str) -> Option) -> KvR if let Some(value) = get_env("DYN_ROUTER_POLICY_CONFIG") { config.router_policy_config = Some(value); } + if let Some(value) = parse_bool(&get_env, "DYN_ROUTER_CONDITIONAL_DISAGG") { + config.conditional_disagg_enabled = value; + } + if let Some(value) = get_env("DYN_ROUTER_CONDITIONAL_DISAGG_POLICY") + && let Ok(policy) = value.parse() + { + config.conditional_disagg_policy = policy; + } + if let Some(value) = parse_usize(&get_env, "DYN_ROUTER_CONDITIONAL_DISAGG_EFF_ISL_THRESHOLD") { + config.conditional_disagg_eff_isl_threshold = value; + } + if let Some(value) = parse_f64( + &get_env, + "DYN_ROUTER_CONDITIONAL_DISAGG_EFF_ISL_RATIO_THRESHOLD", + ) { + config.conditional_disagg_eff_isl_ratio_threshold = value; + } + if let Some(value) = parse_f64( + &get_env, + "DYN_ROUTER_CONDITIONAL_DISAGG_PREFILL_BUSY_THRESHOLD", + ) { + config.conditional_disagg_prefill_busy_threshold = Some(value); + } + if let Some(value) = parse_f64( + &get_env, + "DYN_ROUTER_CONDITIONAL_DISAGG_DECODE_BUSY_THRESHOLD", + ) { + config.conditional_disagg_decode_busy_threshold = Some(value); + } if let Some(value) = parse_f64(&get_env, "DYN_ROUTER_PREDICTED_TTL_SECS") { config.router_predicted_ttl_secs = Some(value); } @@ -294,6 +333,44 @@ impl RouterPrefillLoadModel { } } +/// Which conditional-disagg bypass policy to run. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ConditionalDisaggPolicyKind { + /// Bypass when effective ISL is below both the absolute and ratio thresholds. + #[default] + IslBounding, + /// Bypass when the chosen prefill worker is over the prefill-busy line. + PrefillLoad, + /// Bypass when either `isl_bounding` or `prefill_load` would bypass. + IslOrLoad, +} + +impl fmt::Display for ConditionalDisaggPolicyKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::IslBounding => f.write_str("isl_bounding"), + Self::PrefillLoad => f.write_str("prefill_load"), + Self::IslOrLoad => f.write_str("isl_or_load"), + } + } +} + +impl FromStr for ConditionalDisaggPolicyKind { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "isl_bounding" => Ok(Self::IslBounding), + "prefill_load" => Ok(Self::PrefillLoad), + "isl_or_load" => Ok(Self::IslOrLoad), + _ => Err(format!( + "unknown conditional_disagg_policy: {s:?}, expected 'isl_bounding', 'prefill_load', or 'isl_or_load'" + )), + } + } +} + impl FromStr for RouterQueuePolicy { type Err = String; @@ -410,6 +487,14 @@ struct KvRouterConfigSerde { shared_cache_multiplier: f64, shared_cache_type: SharedCacheType, router_predicted_ttl_secs: Option, + conditional_disagg_enabled: bool, + conditional_disagg_policy: ConditionalDisaggPolicyKind, + conditional_disagg_eff_isl_threshold: usize, + conditional_disagg_eff_isl_ratio_threshold: f64, + #[serde(default)] + conditional_disagg_prefill_busy_threshold: Option, + #[serde(default)] + conditional_disagg_decode_busy_threshold: Option, } impl Default for KvRouterConfigSerde { @@ -444,6 +529,15 @@ impl Default for KvRouterConfigSerde { shared_cache_multiplier: config.shared_cache_multiplier, shared_cache_type: config.shared_cache_type, router_predicted_ttl_secs: config.router_predicted_ttl_secs, + conditional_disagg_enabled: config.conditional_disagg_enabled, + conditional_disagg_policy: config.conditional_disagg_policy, + conditional_disagg_eff_isl_threshold: config.conditional_disagg_eff_isl_threshold, + conditional_disagg_eff_isl_ratio_threshold: config + .conditional_disagg_eff_isl_ratio_threshold, + conditional_disagg_prefill_busy_threshold: config + .conditional_disagg_prefill_busy_threshold, + conditional_disagg_decode_busy_threshold: config + .conditional_disagg_decode_busy_threshold, } } } @@ -586,6 +680,44 @@ pub struct KvRouterConfig { #[serde(default)] #[validate(range(min = 0.0))] pub router_predicted_ttl_secs: Option, + + /// Enable conditional-disagg bypass. When true, the `PrefillRouter` + /// may short-circuit selected requests to prefill+decode on a decode worker. + #[serde(default)] + pub conditional_disagg_enabled: bool, + + /// Which conditional-disagg policy to run. + #[serde(default)] + pub conditional_disagg_policy: ConditionalDisaggPolicyKind, + + /// `IslBoundingPolicy` absolute effective-ISL cutoff in tokens. + #[serde(default = "default_conditional_disagg_eff_isl_threshold")] + pub conditional_disagg_eff_isl_threshold: usize, + + /// `IslBoundingPolicy` effective-ISL/prompt-token ratio cutoff. + #[serde(default = "default_conditional_disagg_eff_isl_ratio_threshold")] + #[validate(range(min = 0.0, max = 1.0))] + pub conditional_disagg_eff_isl_ratio_threshold: f64, + + /// `PrefillLoadPolicy` busy-line fraction for the chosen prefill worker. + /// When unset, the prefill-load condition falls back to `router_queue_threshold`. + #[serde(default)] + #[validate(range(min = 0.0))] + pub conditional_disagg_prefill_busy_threshold: Option, + + /// Decode-busy guard fraction for the chosen decode worker. When unset, + /// the guard is disabled. + #[serde(default)] + #[validate(range(min = 0.0))] + pub conditional_disagg_decode_busy_threshold: Option, +} + +fn default_conditional_disagg_eff_isl_threshold() -> usize { + crate::conditional_disagg::DEFAULT_CONDITIONAL_DISAGG_EFF_ISL_THRESHOLD +} + +fn default_conditional_disagg_eff_isl_ratio_threshold() -> f64 { + crate::conditional_disagg::DEFAULT_CONDITIONAL_DISAGG_EFF_ISL_RATIO_THRESHOLD } impl Default for KvRouterConfig { @@ -620,6 +752,13 @@ impl Default for KvRouterConfig { shared_cache_multiplier: 0.0, shared_cache_type: SharedCacheType::default(), router_predicted_ttl_secs: None, + conditional_disagg_enabled: false, + conditional_disagg_policy: ConditionalDisaggPolicyKind::default(), + conditional_disagg_eff_isl_threshold: default_conditional_disagg_eff_isl_threshold(), + conditional_disagg_eff_isl_ratio_threshold: + default_conditional_disagg_eff_isl_ratio_threshold(), + conditional_disagg_prefill_busy_threshold: None, + conditional_disagg_decode_busy_threshold: None, } } } @@ -669,6 +808,15 @@ impl TryFrom for KvRouterConfig { shared_cache_multiplier: compat.shared_cache_multiplier, shared_cache_type: compat.shared_cache_type, router_predicted_ttl_secs: compat.router_predicted_ttl_secs, + conditional_disagg_enabled: compat.conditional_disagg_enabled, + conditional_disagg_policy: compat.conditional_disagg_policy, + conditional_disagg_eff_isl_threshold: compat.conditional_disagg_eff_isl_threshold, + conditional_disagg_eff_isl_ratio_threshold: compat + .conditional_disagg_eff_isl_ratio_threshold, + conditional_disagg_prefill_busy_threshold: compat + .conditional_disagg_prefill_busy_threshold, + conditional_disagg_decode_busy_threshold: compat + .conditional_disagg_decode_busy_threshold, }) } } @@ -710,6 +858,44 @@ fn validate_kv_router_config(config: &KvRouterConfig) -> Result<(), ValidationEr "router_predicted_ttl_secs requires use_kv_events=true", )); } + if config.conditional_disagg_enabled + && matches!( + config.conditional_disagg_policy, + ConditionalDisaggPolicyKind::PrefillLoad | ConditionalDisaggPolicyKind::IslOrLoad, + ) + { + match ( + config.conditional_disagg_prefill_busy_threshold, + config.router_queue_threshold, + ) { + (Some(threshold), _) => { + tracing::info!( + busy_threshold = threshold, + "conditional_disagg prefill-load condition using --router-conditional-disagg-prefill-busy-threshold" + ); + } + (None, Some(threshold)) => { + tracing::info!( + inherited_threshold = threshold, + "conditional_disagg prefill-load condition using --router-queue-threshold because --router-conditional-disagg-prefill-busy-threshold is unset" + ); + } + (None, None) => { + tracing::warn!( + policy = ?config.conditional_disagg_policy, + "conditional_disagg prefill-load condition disabled: set --router-conditional-disagg-prefill-busy-threshold or --router-queue-threshold, or use policy=isl_bounding" + ); + } + } + } + if config.conditional_disagg_enabled + && let Some(threshold) = config.conditional_disagg_decode_busy_threshold + { + tracing::info!( + decode_busy_threshold = threshold, + "conditional_disagg decode-busy guard enabled: bypass is disabled when the selected decode worker's projected decode load exceeds this fraction of KV capacity" + ); + } if let Err(error) = config.loaded_policy_config() { let mut validation_error = ValidationError::new("router_policy_config"); validation_error.message = Some(error.to_string().into()); diff --git a/lib/kv-router/src/scheduling/local.rs b/lib/kv-router/src/scheduling/local.rs index f715c1537f46..b8be56792202 100644 --- a/lib/kv-router/src/scheduling/local.rs +++ b/lib/kv-router/src/scheduling/local.rs @@ -51,6 +51,58 @@ where Sel: WorkerSelector + Send + Sync + 'static, RF: OverlapScoresRefresh + 'static, { + fn make_scheduling_request( + &self, + request: ScheduleRequest, + resp_tx: Option>>, + ) -> (SchedulingRequest, Option>) { + let track_prefill_tokens = request + .router_config_override + .as_ref() + .and_then(|cfg| cfg.track_prefill_tokens) + .unwrap_or(self.track_prefill_tokens_default); + let ScheduleRequest { + mode, + token_seq, + block_hashes, + isl_tokens, + lora_name, + expected_output_tokens, + pinned_worker, + allowed_worker_ids, + routing_constraints, + router_config_override, + priority_jump, + strict_priority, + policy_class, + session_id, + overlap, + shared_cache_hits, + } = request; + let request = SchedulingRequest { + mode, + token_seq, + isl_tokens, + lora_name, + expected_output_tokens, + pinned_worker, + allowed_worker_ids, + routing_constraints, + router_config_override, + track_prefill_tokens, + priority_jump, + strict_priority, + policy_class, + session_id, + overlap, + shared_cache_hits, + worker_loads: FxHashMap::default(), + resp_tx, + }; + + (request, block_hashes) + } + fn worker_dp_ranges(workers: &HashMap) -> Vec { workers .iter() @@ -235,49 +287,7 @@ where request: ScheduleRequest, ) -> Result { let (resp_tx, resp_rx) = tokio::sync::oneshot::channel(); - let track_prefill_tokens = request - .router_config_override - .as_ref() - .and_then(|cfg| cfg.track_prefill_tokens) - .unwrap_or(self.track_prefill_tokens_default); - let ScheduleRequest { - mode, - token_seq, - block_hashes, - isl_tokens, - lora_name, - expected_output_tokens, - pinned_worker, - allowed_worker_ids, - routing_constraints, - router_config_override, - priority_jump, - strict_priority, - policy_class, - session_id, - overlap, - shared_cache_hits, - } = request; - let request = SchedulingRequest { - mode, - token_seq, - isl_tokens, - lora_name, - expected_output_tokens, - pinned_worker, - allowed_worker_ids, - routing_constraints, - router_config_override, - track_prefill_tokens, - priority_jump, - strict_priority, - policy_class, - session_id, - overlap, - shared_cache_hits, - worker_loads: FxHashMap::default(), - resp_tx: Some(resp_tx), - }; + let (request, block_hashes) = self.make_scheduling_request(request, Some(resp_tx)); self.queue .enqueue_with_block_hashes(request, block_hashes) @@ -288,6 +298,15 @@ where .map_err(|_| KvSchedulerError::SubscriberShutdown)? } + /// Select a worker from current scheduler state without queue admission or booking. + pub async fn select_without_admission( + &self, + request: ScheduleRequest, + ) -> Result { + let (request, _block_hashes) = self.make_scheduling_request(request, None); + self.queue.select_without_admission(request).await + } + #[expect(clippy::too_many_arguments)] pub async fn schedule( &self, @@ -490,6 +509,30 @@ where self.queue_updates.subscribe() } + pub fn worker_is_prefill_busy( + &self, + worker: WorkerWithDpRank, + decay_now: Instant, + threshold: f64, + ) -> Option { + self.queue + .worker_is_prefill_busy(worker, decay_now, threshold) + } + + pub fn worker_is_decode_busy(&self, worker: WorkerWithDpRank, threshold: f64) -> Option { + self.queue.worker_is_decode_busy(worker, threshold) + } + + pub fn projected_decode_load_exceeds( + &self, + worker: WorkerWithDpRank, + projected_blocks: usize, + threshold: f64, + ) -> Option { + self.queue + .projected_decode_load_exceeds(worker, projected_blocks, threshold) + } + pub fn add_output_block( &self, request_id: &str, diff --git a/lib/kv-router/src/scheduling/queue.rs b/lib/kv-router/src/scheduling/queue.rs index 2c3016045d34..a3512e0f611d 100644 --- a/lib/kv-router/src/scheduling/queue.rs +++ b/lib/kv-router/src/scheduling/queue.rs @@ -61,6 +61,10 @@ enum AdmissionCommand { block_hashes: Option>, ack_tx: oneshot::Sender<()>, }, + SelectWithoutAdmission { + request: SchedulingRequest, + resp_tx: oneshot::Sender>, + }, Update { worker: Option, ack_tx: oneshot::Sender<()>, @@ -379,6 +383,27 @@ impl< } } + /// Select a worker from current scheduler state without entering admission. + /// + /// This is for advisory policy probes that must not wait in the router + /// queue and must not book active scheduler state. + pub async fn select_without_admission( + &self, + request: SchedulingRequest, + ) -> Result { + request.eligibility().validate_pinned_worker_allowed()?; + + let (resp_tx, resp_rx) = oneshot::channel(); + let command = AdmissionCommand::SelectWithoutAdmission { request, resp_tx }; + if self.admission_tx.send(command).await.is_err() { + return Err(KvSchedulerError::SubscriberShutdown); + } + + resp_rx + .await + .map_err(|_| KvSchedulerError::SubscriberShutdown)? + } + /// Called on prefill_complete/free. Drains pending requests while workers have capacity. /// Each scheduled request updates active_tokens via add_request, so the prefill-busy check /// sees fresh state on the next iteration. @@ -416,6 +441,48 @@ impl< self.pending_isl_tokens.load(AtomicOrdering::Relaxed) } + /// Per-worker prefill-busy check used by conditional-disagg load policies. + /// Returns `None` when the worker config is unavailable. + pub fn worker_is_prefill_busy( + &self, + worker: WorkerWithDpRank, + decay_now: Instant, + threshold: f64, + ) -> Option { + let configs = self.workers_with_configs.borrow(); + let config = configs.get(&worker.worker_id)?; + let max_batched = config + .max_num_batched_tokens() + .unwrap_or(DEFAULT_MAX_BATCHED_TOKENS); + let active_tokens = self.slots.active_tokens(decay_now); + let tokens = active_tokens.get(&worker).copied().unwrap_or(0); + Some((tokens as f64) > threshold * (max_batched as f64)) + } + + /// Per-worker decode-busy check used by the conditional-disagg decode-busy + /// guard. Returns `None` when the worker config or total KV capacity is + /// unavailable. + pub fn worker_is_decode_busy(&self, worker: WorkerWithDpRank, threshold: f64) -> Option { + let configs = self.workers_with_configs.borrow(); + let config = configs.get(&worker.worker_id)?; + let total_kv_blocks = config.total_kv_blocks()?; + let active_blocks = self.slots.active_blocks(); + let blocks = active_blocks.get(&worker).copied().unwrap_or(0); + Some((blocks as f64) > threshold * (total_kv_blocks as f64)) + } + + pub fn projected_decode_load_exceeds( + &self, + worker: WorkerWithDpRank, + projected_blocks: usize, + threshold: f64, + ) -> Option { + let configs = self.workers_with_configs.borrow(); + let config = configs.get(&worker.worker_id)?; + let total_kv_blocks = config.total_kv_blocks()?; + Some((projected_blocks as f64) > threshold * (total_kv_blocks as f64)) + } + pub fn class_queue_stats(&self, class_index: usize) -> Option { let counters = self.class_counters.get(class_index)?; Some(ClassQueueStats { @@ -458,6 +525,13 @@ impl< self.handle_enqueue(request, block_hashes); let _ = ack_tx.send(()); } + AdmissionCommand::SelectWithoutAdmission { + mut request, + resp_tx, + } => { + let result = self.select_without_admission_inner(&mut request, Instant::now()); + let _ = resp_tx.send(result); + } AdmissionCommand::Update { worker, ack_tx } => { self.handle_update(worker).await; let _ = ack_tx.send(()); @@ -669,9 +743,11 @@ impl< } } - /// Run the full scheduling pipeline for a single request: - /// compute projected load -> select worker -> book tracked state -> respond. - fn admit_one(&self, mut request: SchedulingRequest, decay_now: Instant) { + fn select_without_admission_inner( + &self, + request: &mut SchedulingRequest, + decay_now: Instant, + ) -> Result { request.worker_loads = self .slots .project_worker_loads(request.token_seq.as_deref(), decay_now); @@ -684,7 +760,7 @@ impl< .and_then(|provider| provider()); let eligibility = request.eligibility_with_overloaded(overloaded_worker_ids.as_ref()); self.selector - .select_worker(&workers, &request, eligibility, self.block_size) + .select_worker(&workers, request, eligibility, self.block_size) .map(|selection| { let config = workers .get(&selection.worker.worker_id) @@ -696,20 +772,27 @@ impl< }) }; - let (selection, selected_worker_tiers) = match selection { - Ok(s) => s, - Err(e) => { - tracing::warn!("scheduling failed: {e}"); - request.respond(Err(e)); - return; - } - }; + let (selection, selected_worker_tiers) = selection?; - let response = SchedulingResponse { + Ok(SchedulingResponse { best_worker: selection.worker, effective_overlap_blocks: selection.effective_overlap_blocks, cached_tokens: selection.cached_tokens, selected_worker_tiers, + potential_decode_blocks: selection.potential_decode_blocks, + }) + } + + /// Run the full scheduling pipeline for a single request: + /// compute projected load -> select worker -> book tracked state -> respond. + fn admit_one(&self, mut request: SchedulingRequest, decay_now: Instant) { + let response = match self.select_without_admission_inner(&mut request, decay_now) { + Ok(response) => response, + Err(e) => { + tracing::warn!("scheduling failed: {e}"); + request.respond(Err(e)); + return; + } }; if !request.mode.is_tracked() { @@ -725,7 +808,7 @@ impl< let prefill_load_hint = self.prefill_load_hint_for( request.isl_tokens, - selection.cached_tokens, + response.cached_tokens, request.track_prefill_tokens, ); @@ -735,7 +818,7 @@ impl< track_prefill_tokens: request.track_prefill_tokens, expected_output_tokens: request.expected_output_tokens, prefill_load_hint, - worker: selection.worker, + worker: response.best_worker, lora_name: request.lora_name.take(), }; self.book_and_respond(request, sequence_request, response); @@ -1023,6 +1106,7 @@ mod tests { required_blocks: request.request_blocks(block_size), effective_overlap_blocks: request.effective_overlap_blocks_for(worker), cached_tokens: request.effective_cached_tokens_for(worker), + potential_decode_blocks: request.worker_load_for(worker).potential_decode_blocks(), }) } } diff --git a/lib/kv-router/src/scheduling/selector.rs b/lib/kv-router/src/scheduling/selector.rs index a5827434702a..18964f5973f7 100644 --- a/lib/kv-router/src/scheduling/selector.rs +++ b/lib/kv-router/src/scheduling/selector.rs @@ -202,10 +202,30 @@ impl DefaultWorkerSelector { + self.kv_router_config.host_cache_hit_weight * host_overlap_blocks + self.kv_router_config.disk_cache_hit_weight * disk_overlap_blocks + shared_overlap_blocks; - let adjusted_prefill_blocks = (raw_prefill_blocks - overlap_credit_blocks).max(0.0); - let prefill_cost_blocks = weights.prefill_load_scale * adjusted_prefill_blocks; let worker_load = worker_load.unwrap_or_default(); let decode_cost_blocks = worker_load.potential_decode_blocks() as f64; + + // Decode routers normally force `overlap_score_credit=0` through the + // per-request override, which preserves load-only disagg routing. When + // conditional disagg leaves a positive overlap credit in place, prefer + // cache-hot decode workers while still charging decode backlog. + if self.worker_type == "decode" + && !request.track_prefill_tokens + && weights.overlap_score_credit > 0.0 + { + let logit = decode_cost_blocks - overlap_credit_blocks; + tracing::debug!( + "{formula_name} for worker_id={} dp_rank={:?} with {effective_overlap_blocks:.2} effective cached blocks: {logit:.3} \ + = decode_blocks - overlap_credit_blocks \ + = {decode_cost_blocks:.3} - {overlap_credit_blocks:.3}", + worker.worker_id, + worker.dp_rank, + ); + return logit; + } + + let adjusted_prefill_blocks = (raw_prefill_blocks - overlap_credit_blocks).max(0.0); + let prefill_cost_blocks = weights.prefill_load_scale * adjusted_prefill_blocks; let logit = prefill_cost_blocks + decode_cost_blocks; if shared_beyond > 0 { @@ -327,6 +347,7 @@ impl WorkerSelector for DefaultWorkerSelector { required_blocks: request_blocks, effective_overlap_blocks, cached_tokens, + potential_decode_blocks: request.worker_load_for(worker).potential_decode_blocks(), }); } @@ -438,6 +459,9 @@ impl WorkerSelector for DefaultWorkerSelector { required_blocks: request_blocks, effective_overlap_blocks, cached_tokens, + potential_decode_blocks: request + .worker_load_for(best_worker) + .potential_decode_blocks(), }); } @@ -466,6 +490,9 @@ impl WorkerSelector for DefaultWorkerSelector { required_blocks: request_blocks, effective_overlap_blocks: best_overlap, cached_tokens: best_cached_tokens, + potential_decode_blocks: request + .worker_load_for(best_worker) + .potential_decode_blocks(), }) } } @@ -1288,6 +1315,33 @@ mod tests { ); } + #[test] + fn test_decode_worker_logit_credits_overlap_without_prefill_tracking() { + let worker = WorkerWithDpRank::from_worker_id(0); + let mut request = base_request(64); + request.track_prefill_tokens = false; + request.overlap.tier_overlap_blocks.device.insert(worker, 3); + request.worker_loads.insert( + worker, + crate::sequences::WorkerLoadProjection { + active_decode_blocks: 10, + ..Default::default() + }, + ); + let selector = DefaultWorkerSelector::new(Some(KvRouterConfig::default()), "decode"); + let weights = LogitWeights { + overlap_score_credit: 1.0, + overlap_score_credit_decay: 0.0, + prefill_load_scale: 1.0, + shared_cache_multiplier: 0.0, + }; + + assert_eq!( + selector.worker_logit(&request, worker, 16, 0, weights, "test"), + 7.0 + ); + } + #[test] fn test_overlap_credit_decay_can_prefer_less_loaded_cold_worker() { use crate::test_utils::SimpleWorkerConfig; diff --git a/lib/kv-router/src/scheduling/types.rs b/lib/kv-router/src/scheduling/types.rs index c6fe9678592f..7ff30703ca35 100644 --- a/lib/kv-router/src/scheduling/types.rs +++ b/lib/kv-router/src/scheduling/types.rs @@ -75,6 +75,7 @@ pub struct SchedulingResponse { pub effective_overlap_blocks: f64, pub cached_tokens: usize, pub selected_worker_tiers: SelectedWorkerTierSnapshot, + pub potential_decode_blocks: usize, } #[derive(Debug, Clone)] diff --git a/lib/llm/src/discovery/watcher.rs b/lib/llm/src/discovery/watcher.rs index b67368d74537..c6111a9456f2 100644 --- a/lib/llm/src/discovery/watcher.rs +++ b/lib/llm/src/discovery/watcher.rs @@ -1623,6 +1623,7 @@ impl ModelWatcher { router_config.router_mode, card.kv_cache_block_size, Some(prefill_config), + kv_chooser.clone(), self.prefill_load_estimator.clone(), router_config.session_affinity_ttl_secs, model_name.clone(), diff --git a/lib/llm/src/entrypoint/input/common.rs b/lib/llm/src/entrypoint/input/common.rs index fa5b1a87dfa0..89f4b62542a8 100644 --- a/lib/llm/src/entrypoint/input/common.rs +++ b/lib/llm/src/entrypoint/input/common.rs @@ -26,7 +26,7 @@ use crate::{ preprocessor::MultimodalData, }, request_template::RequestTemplate, - session_affinity::SessionAffinityPushRouter, + session_affinity::{AffinityCoordinator, SessionAffinityPushRouter}, types::{ Annotated, openai::chat_completions::{ @@ -184,6 +184,7 @@ fn preprocessed_backend_engine( chooser: Option>, model_manager: &Arc, session_affinity_ttl: Option, + decode_session_affinity: Option, ) -> anyhow::Result, ManyOut>>> { // Reject LoRA + unsupported-mode combinations up front (single source of truth, shared with @@ -225,7 +226,11 @@ fn preprocessed_backend_engine( let Some(chooser) = chooser else { anyhow::bail!("RouterMode::KV requires KVRouter to not be null"); }; - Arc::new(KvPushRouter::new(router, chooser, session_affinity_ttl)?) + Arc::new(KvPushRouter::new_with_affinity( + router, + chooser, + decode_session_affinity, + )?) } }; @@ -297,12 +302,20 @@ pub async fn build_preprocessed_routing( }); let encoder_router = encoder_chooser.unwrap_or_else(EncoderRouter::disabled); + let session_affinity_ttl = session_affinity_ttl_secs.map(Duration::from_secs); + let decode_session_affinity = if router_mode.is_kv_routing() { + prefill_router.get_or_create_decode_session_affinity(session_affinity_ttl)? + } else { + None + }; + let backend_engine = preprocessed_backend_engine( router, router_mode, chooser, &model_manager, - session_affinity_ttl_secs.map(Duration::from_secs), + session_affinity_ttl, + decode_session_affinity, )?; Ok(PreprocessedRouting { backend_engine, diff --git a/lib/llm/src/kv_router.rs b/lib/llm/src/kv_router.rs index 4881999964d6..522048dbb718 100644 --- a/lib/llm/src/kv_router.rs +++ b/lib/llm/src/kv_router.rs @@ -78,6 +78,7 @@ pub enum FindBestMatchOutcome { overlap_blocks: u32, effective_overlap_blocks: f64, cached_tokens: usize, + potential_decode_blocks: usize, routing_hashes: Option, }, QueueRejected { @@ -85,6 +86,12 @@ pub enum FindBestMatchOutcome { }, } +#[derive(Clone, Copy)] +enum FindBestMatchAdmission { + WithAdmission, + WithoutAdmission, +} + #[derive(Debug, Clone, Copy, Default)] pub(crate) struct WorkerCacheHitEstimate { pub effective_overlap_blocks: f64, @@ -516,6 +523,45 @@ where .await } + #[allow(clippy::too_many_arguments)] + pub async fn find_best_match_details_without_admission( + &self, + context_id: Option<&str>, + tokens: &[u32], + block_mm_infos: Option<&[Option]>, + router_config_override: Option<&RouterConfigOverride>, + return_routing_hashes: bool, + lora_name: Option, + cache_namespace: Option, + priority_jump: f64, + strict_priority: u32, + expected_output_tokens: Option, + pinned_worker: Option, + allowed_worker_ids: Option>, + routing_constraints: RoutingConstraints, + ) -> anyhow::Result { + self.find_best_match_details_with_policy_class_impl( + context_id, + tokens, + block_mm_infos, + router_config_override, + false, + return_routing_hashes, + lora_name, + cache_namespace, + priority_jump, + strict_priority, + None, + None, + expected_output_tokens, + pinned_worker, + allowed_worker_ids, + routing_constraints, + FindBestMatchAdmission::WithoutAdmission, + ) + .await + } + #[allow(clippy::too_many_arguments)] pub async fn find_best_match_details_with_policy_class( &self, @@ -535,13 +581,57 @@ where pinned_worker: Option, allowed_worker_ids: Option>, routing_constraints: RoutingConstraints, + ) -> anyhow::Result { + self.find_best_match_details_with_policy_class_impl( + context_id, + tokens, + block_mm_infos, + router_config_override, + update_states, + return_routing_hashes, + lora_name, + cache_namespace, + priority_jump, + strict_priority, + policy_class, + session_id, + expected_output_tokens, + pinned_worker, + allowed_worker_ids, + routing_constraints, + FindBestMatchAdmission::WithAdmission, + ) + .await + } + + #[allow(clippy::too_many_arguments)] + async fn find_best_match_details_with_policy_class_impl( + &self, + context_id: Option<&str>, + tokens: &[u32], + block_mm_infos: Option<&[Option]>, + router_config_override: Option<&RouterConfigOverride>, + update_states: bool, + return_routing_hashes: bool, + lora_name: Option, + cache_namespace: Option, + priority_jump: f64, + strict_priority: u32, + policy_class: Option, + session_id: Option, + expected_output_tokens: Option, + pinned_worker: Option, + allowed_worker_ids: Option>, + routing_constraints: RoutingConstraints, + admission: FindBestMatchAdmission, ) -> anyhow::Result { let start = Instant::now(); - if update_states && context_id.is_none() { + let with_admission = matches!(admission, FindBestMatchAdmission::WithAdmission); + if with_admission && update_states && context_id.is_none() { anyhow::bail!("context_id must be provided if update_states is true"); } - let mode = if update_states { + let mode = if with_admission && update_states { ScheduleMode::Tracked { request_id: context_id.expect("validated above").to_string(), } @@ -575,7 +665,7 @@ where }); let seq_hash_elapsed = start.elapsed(); - let supports_overlap_refresh = self.scheduler.supports_overlap_refresh(); + let supports_overlap_refresh = with_admission && self.scheduler.supports_overlap_refresh(); let retain_block_hashes = supports_overlap_refresh || return_routing_hashes; let TieredLookupResult { @@ -626,29 +716,40 @@ where pinned_worker.as_ref(), ); - let response = match self - .scheduler - .schedule_request(ScheduleRequest { - mode, - token_seq: maybe_seq_hashes, - block_hashes: block_hashes_for_refresh, - isl_tokens, - overlap, - router_config_override: router_config_override.cloned(), - lora_name, - priority_jump, - strict_priority, - policy_class, - session_id, - expected_output_tokens, - pinned_worker, - allowed_worker_ids, - routing_constraints, - shared_cache_hits, - }) - .instrument(tracing::info_span!("kv_router.schedule")) - .await - { + let schedule_request = ScheduleRequest { + mode, + token_seq: maybe_seq_hashes, + block_hashes: block_hashes_for_refresh, + isl_tokens, + overlap, + router_config_override: router_config_override.cloned(), + lora_name, + priority_jump, + strict_priority, + policy_class, + session_id, + expected_output_tokens, + pinned_worker, + allowed_worker_ids, + routing_constraints, + shared_cache_hits, + }; + let schedule_span = tracing::info_span!("kv_router.schedule"); + let response_result = match admission { + FindBestMatchAdmission::WithAdmission => { + self.scheduler + .schedule_request(schedule_request) + .instrument(schedule_span) + .await + } + FindBestMatchAdmission::WithoutAdmission => { + self.scheduler + .select_without_admission(schedule_request) + .instrument(schedule_span) + .await + } + }; + let response = match response_result { Ok(response) => response, Err(KvSchedulerError::QueueRejected(rejection)) => { return Ok(FindBestMatchOutcome::QueueRejected { rejection }); @@ -697,6 +798,7 @@ where overlap_blocks: response.effective_overlap_blocks.round() as u32, effective_overlap_blocks: response.effective_overlap_blocks, cached_tokens: response.cached_tokens, + potential_decode_blocks: response.potential_decode_blocks, routing_hashes, }) } @@ -879,6 +981,30 @@ where self.scheduler.add_output_block(request_id, decay_fraction) } + pub fn worker_is_prefill_busy( + &self, + worker: WorkerWithDpRank, + decay_now: tokio::time::Instant, + threshold: f64, + ) -> Option { + self.scheduler + .worker_is_prefill_busy(worker, decay_now, threshold) + } + + pub fn worker_is_decode_busy(&self, worker: WorkerWithDpRank, threshold: f64) -> Option { + self.scheduler.worker_is_decode_busy(worker, threshold) + } + + pub fn projected_decode_load_exceeds( + &self, + worker: WorkerWithDpRank, + projected_blocks: usize, + threshold: f64, + ) -> Option { + self.scheduler + .projected_decode_load_exceeds(worker, projected_blocks, threshold) + } + pub fn block_size(&self) -> u32 { self.block_size } @@ -1303,6 +1429,9 @@ mod tests { required_blocks: request.isl_tokens.div_ceil(block_size as usize) as u64, effective_overlap_blocks: 0.0, cached_tokens: 0, + potential_decode_blocks: request + .worker_load_for(self.selected_worker) + .potential_decode_blocks(), }) } } diff --git a/lib/llm/src/kv_router/prefill_router/activation.rs b/lib/llm/src/kv_router/prefill_router/activation.rs index 7bf12649fb8c..cc0f7cfc0a1a 100644 --- a/lib/llm/src/kv_router/prefill_router/activation.rs +++ b/lib/llm/src/kv_router/prefill_router/activation.rs @@ -7,7 +7,10 @@ use std::sync::atomic::Ordering; use anyhow::Result; use tokio::sync::oneshot; -use dynamo_kv_router::{PrefillLoadEstimator, config::KvRouterConfig}; +use dynamo_kv_router::{ + PrefillLoadEstimator, conditional_disagg::make_conditional_disagg_policy, + config::KvRouterConfig, +}; use dynamo_runtime::{ component::{Client, Endpoint}, pipeline::{PushRouter, RouterMode}, @@ -17,7 +20,7 @@ use dynamo_runtime::{ use super::{InnerPrefillRouter, PrefillLifecycleState, PrefillRouter}; use crate::{ discovery::ModelManager, - kv_router::KvPushRouter, + kv_router::{KvPushRouter, KvRouter}, protocols::common::{ llm_backend::{LLMEngineOutput, PreprocessedRequest}, timing::WORKER_TYPE_PREFILL, @@ -33,11 +36,16 @@ impl PrefillRouter { ) -> Arc { Arc::new(Self { prefill_router: std::sync::OnceLock::new(), + decode_router: None, + decode_session_affinity: std::sync::OnceLock::new(), model_manager, endpoint_id: std::sync::OnceLock::new(), cancel_token: tokio_util::sync::CancellationToken::new(), router_mode, session_affinity_ttl: session_affinity_ttl_secs.map(std::time::Duration::from_secs), + conditional_disagg_policy: make_conditional_disagg_policy(None), + conditional_disagg_prefill_busy_threshold: None, + conditional_disagg_decode_busy_threshold: None, prefill_load_estimator: None, model_name: String::new(), // Not used for disabled router namespace: String::new(), // Not used for disabled router @@ -53,6 +61,7 @@ impl PrefillRouter { router_mode: RouterMode, kv_cache_block_size: u32, kv_router_config: Option, + decode_router: Option>, prefill_load_estimator: Option>, session_affinity_ttl_secs: Option, model_name: String, @@ -62,14 +71,27 @@ impl PrefillRouter { ) -> Arc { let prefill_router = std::sync::OnceLock::new(); let cancel_token = tokio_util::sync::CancellationToken::new(); + let conditional_disagg_policy = make_conditional_disagg_policy(kv_router_config.as_ref()); + let conditional_disagg_prefill_busy_threshold = kv_router_config.as_ref().and_then(|c| { + c.conditional_disagg_prefill_busy_threshold + .or(c.router_queue_threshold) + }); + let conditional_disagg_decode_busy_threshold = kv_router_config + .as_ref() + .and_then(|c| c.conditional_disagg_decode_busy_threshold); let router = Arc::new(Self { prefill_router, + decode_router, + decode_session_affinity: std::sync::OnceLock::new(), model_manager: model_manager.clone(), endpoint_id: std::sync::OnceLock::new(), cancel_token: cancel_token.clone(), router_mode, session_affinity_ttl: session_affinity_ttl_secs.map(std::time::Duration::from_secs), + conditional_disagg_policy, + conditional_disagg_prefill_busy_threshold, + conditional_disagg_decode_busy_threshold, prefill_load_estimator, model_name, namespace, diff --git a/lib/llm/src/kv_router/prefill_router/conditional_bypass.rs b/lib/llm/src/kv_router/prefill_router/conditional_bypass.rs new file mode 100644 index 000000000000..c60054b1aa77 --- /dev/null +++ b/lib/llm/src/kv_router/prefill_router/conditional_bypass.rs @@ -0,0 +1,334 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use anyhow::Result; +use dynamo_kv_router::conditional_disagg::ConditionalDisaggDecisionInput; +use dynamo_kv_router::protocols::WorkerWithDpRank; + +use super::{InnerPrefillRouter, PrefillRouter}; +use crate::protocols::common::llm_backend::PreprocessedRequest; +use crate::session_affinity::AffinityTarget; + +/// Conditional-disagg decision: which decode worker to pin the request to, +/// plus diagnostic counts for logging. +pub(super) struct ConditionalDisaggDecodeDecision { + pub worker: WorkerWithDpRank, + pub overlap_tokens: usize, + pub net_new_tokens: usize, +} + +fn decode_gate_allows_bypass(policy_says_bypass: bool, decode_busy: Option) -> bool { + policy_says_bypass && decode_busy != Some(true) +} + +impl PrefillRouter { + /// Peek the decode router to see which decode worker would be picked, then + /// consult the configured conditional-disagg policy. + pub(super) async fn select_decode_worker_for_conditional_disagg( + &self, + req: &PreprocessedRequest, + request_id: &str, + decode_affinity_target: Option, + ) -> Result> { + if !self.router_mode.is_kv_routing() { + return Ok(None); + } + + let has_explicit_prefill_pin = req + .routing + .as_ref() + .is_some_and(|routing| routing.prefill_worker_id.is_some()); + if has_explicit_prefill_pin { + tracing::debug!( + request_id, + "Skipping conditional disagg because request has a preselected prefill worker" + ); + return Ok(None); + } + + let Some(decode_router) = self.decode_router.as_ref() else { + tracing::debug!( + request_id, + "Skipping conditional disagg because decode router is unavailable" + ); + return Ok(None); + }; + + let (routing_token_ids, block_mm_infos) = req.block_mm_routing_info(); + if routing_token_ids.is_empty() { + return Ok(None); + } + + let lora_name = req + .routing + .as_ref() + .and_then(|routing| routing.lora_name.clone()); + let cache_namespace = req + .routing + .as_ref() + .and_then(|routing| routing.cache_namespace.clone()); + let priority_jump = req + .routing + .as_ref() + .and_then(|routing| routing.priority_jump) + .unwrap_or(0.0); + let strict_priority = req + .routing + .as_ref() + .and_then(|routing| routing.strict_priority) + .unwrap_or(0); + let expected_output_tokens = req + .routing + .as_ref() + .and_then(|routing| routing.expected_output_tokens); + let allowed_worker_ids = req + .routing + .as_ref() + .and_then(|routing| routing.allowed_worker_ids.clone()); + let affinity_pinned_worker = match decode_affinity_target { + Some(target) => { + let Some(dp_rank) = target + .dp_rank + .or_else(|| decode_router.unique_dp_rank_for_worker(target.worker_id)) + else { + tracing::debug!( + request_id, + worker_id = target.worker_id, + "Skipping conditional disagg because decode affinity target has no resolved DP rank" + ); + return Ok(None); + }; + Some(WorkerWithDpRank::new(target.worker_id, dp_rank)) + } + None => None, + }; + let request_pinned_worker = req.routing.as_ref().and_then(|routing| { + let worker_id = routing.decode_worker_id.or(routing.backend_instance_id)?; + let dp_rank = routing + .dp_rank + .or_else(|| decode_router.unique_dp_rank_for_worker(worker_id))?; + Some(WorkerWithDpRank::new(worker_id, dp_rank)) + }); + let pinned_worker = request_pinned_worker.or(affinity_pinned_worker); + let routing_constraints = req + .routing + .as_ref() + .and_then(|routing| routing.routing_constraints.clone()) + .unwrap_or_default(); + + let outcome = decode_router + .find_best_match_details_without_admission( + Some(request_id), + routing_token_ids, + block_mm_infos, + req.router_config_override.as_ref(), + false, + lora_name, + cache_namespace, + priority_jump, + strict_priority, + expected_output_tokens, + pinned_worker, + allowed_worker_ids, + routing_constraints, + ) + .await?; + let (worker, overlap_blocks, cached_tokens, potential_decode_blocks) = match outcome { + crate::kv_router::FindBestMatchOutcome::Routed { + worker, + overlap_blocks, + cached_tokens, + potential_decode_blocks, + .. + } => ( + worker, + overlap_blocks, + cached_tokens, + potential_decode_blocks, + ), + crate::kv_router::FindBestMatchOutcome::QueueRejected { .. } => { + return Ok(None); + } + }; + + let block_size = decode_router.block_size() as usize; + let prompt_tokens = routing_token_ids.len(); + + let mut input = ConditionalDisaggDecisionInput::new(prompt_tokens, cached_tokens); + if self.conditional_disagg_policy.needs_prefill_worker_busy() { + let busy = self.peek_prefill_chosen_worker_busy(req).await; + tracing::debug!( + request_id, + prefill_chosen_worker_busy = ?busy, + "Conditional disagg prefill-load condition inspected selected prefill worker" + ); + input = input.with_prefill_chosen_worker_busy(busy); + } + let net_new_tokens = input.net_new_tokens(); + let overlap_tokens = (overlap_blocks as usize) * block_size; + + let policy_says_bypass = self + .conditional_disagg_policy + .should_bypass_remote_prefill(input) + .await; + + // This gate is advisory, not a decode-capacity reservation. Normal + // decode routing books scheduler state before returning a routing + // decision; conditional-disagg is still deciding whether to enter that + // decode path, so booking here would double-count unless we added a + // reservation handoff to the downstream decode router. Use the selected + // worker's projected decode load, including this request, but allow for + // concurrent bypass decisions to race the same threshold. + let decode_busy = if policy_says_bypass { + self.conditional_disagg_decode_busy_threshold + .and_then(|threshold| { + decode_router.projected_decode_load_exceeds( + worker, + potential_decode_blocks, + threshold, + ) + }) + } else { + None + }; + input = input.with_decode_chosen_worker_busy(decode_busy); + + let bypass = decode_gate_allows_bypass(policy_says_bypass, decode_busy); + let decode_gate_decision = if !policy_says_bypass { + "bypass_declined_by_policy" + } else if self.conditional_disagg_decode_busy_threshold.is_none() { + "bypass_allowed_decode_gate_disabled" + } else if decode_busy.is_none() { + "bypass_allowed_decode_busy_unknown" + } else if decode_busy == Some(true) { + "bypass_denied_decode_busy" + } else { + "bypass_allowed_decode_not_busy" + }; + + tracing::debug!( + request_id, + worker_id = worker.worker_id, + dp_rank = worker.dp_rank, + prompt_tokens, + net_new_tokens, + overlap_tokens, + prefill_chosen_worker_busy = ?input.prefill_chosen_worker_busy, + decode_chosen_worker_busy = ?decode_busy, + cached_tokens, + potential_decode_blocks, + decode_busy_threshold = ?self.conditional_disagg_decode_busy_threshold, + decode_gate_decision, + bypass, + "Conditional disagg decision" + ); + + if bypass { + return Ok(Some(ConditionalDisaggDecodeDecision { + worker, + overlap_tokens, + net_new_tokens, + })); + } + + Ok(None) + } + + async fn peek_prefill_chosen_worker_busy(&self, req: &PreprocessedRequest) -> Option { + let threshold = self.conditional_disagg_prefill_busy_threshold?; + let prefill_router = self.prefill_router.get()?; + let router = match prefill_router { + InnerPrefillRouter::KvRouter(router) => router, + InnerPrefillRouter::SimpleRouter(_) => return None, + }; + + let lora_name = req + .routing + .as_ref() + .and_then(|routing| routing.lora_name.clone()); + let cache_namespace = req + .routing + .as_ref() + .and_then(|routing| routing.cache_namespace.clone()); + let priority_jump = req + .routing + .as_ref() + .and_then(|routing| routing.priority_jump) + .unwrap_or(0.0); + let strict_priority = req + .routing + .as_ref() + .and_then(|routing| routing.strict_priority) + .unwrap_or(0); + let expected_output_tokens = req + .routing + .as_ref() + .and_then(|routing| routing.expected_output_tokens); + let allowed_worker_ids = req + .routing + .as_ref() + .and_then(|routing| routing.allowed_worker_ids.clone()); + let routing_constraints = req + .routing + .as_ref() + .and_then(|routing| routing.routing_constraints.clone()) + .unwrap_or_default(); + let (routing_token_ids, block_mm_infos) = req.block_mm_routing_info(); + + let outcome = router + .chooser + .find_best_match_details_without_admission( + None, + routing_token_ids, + block_mm_infos, + req.router_config_override.as_ref(), + false, + lora_name, + cache_namespace, + priority_jump, + strict_priority, + expected_output_tokens, + None, + allowed_worker_ids, + routing_constraints, + ) + .await + .ok()?; + + let worker = match outcome { + crate::kv_router::FindBestMatchOutcome::Routed { worker, .. } => worker, + crate::kv_router::FindBestMatchOutcome::QueueRejected { .. } => return Some(true), + }; + + router + .chooser + .worker_is_prefill_busy(worker, tokio::time::Instant::now(), threshold) + } +} + +#[cfg(test)] +mod tests { + use super::decode_gate_allows_bypass; + + #[test] + fn decode_gate_calm_and_policy_bypass_allows_bypass() { + assert!(decode_gate_allows_bypass(true, Some(false))); + } + + #[test] + fn decode_gate_busy_vetoes_policy_bypass() { + assert!(!decode_gate_allows_bypass(true, Some(true))); + } + + #[test] + fn decode_gate_does_not_bypass_when_policy_declines() { + assert!(!decode_gate_allows_bypass(false, Some(false))); + assert!(!decode_gate_allows_bypass(false, Some(true))); + assert!(!decode_gate_allows_bypass(false, None)); + } + + #[test] + fn decode_gate_signal_unavailable_does_not_block_bypass() { + assert!(decode_gate_allows_bypass(true, None)); + } +} diff --git a/lib/llm/src/kv_router/prefill_router/mod.rs b/lib/llm/src/kv_router/prefill_router/mod.rs index f30c7b101b67..e778980bf02d 100644 --- a/lib/llm/src/kv_router/prefill_router/mod.rs +++ b/lib/llm/src/kv_router/prefill_router/mod.rs @@ -9,16 +9,17 @@ use tokio_util::sync::CancellationToken; use uuid::Uuid; use dynamo_kv_router::{ - PrefillLoadEstimator, config::RouterConfigOverride, protocols::RoutingConstraints, - scheduling::QueueRejection, + PrefillLoadEstimator, conditional_disagg::ConditionalDisaggPolicy, + config::RouterConfigOverride, protocols::RoutingConstraints, scheduling::QueueRejection, }; use dynamo_runtime::{ pipeline::{ - AsyncEngineContextProvider, Context, ManyOut, Operator, RouterMode, ServerStreamingEngine, - SingleIn, async_trait, + AsyncEngineContextProvider, Context, ManyOut, Operator, ResponseStream, RouterMode, + ServerStreamingEngine, SingleIn, async_trait, }, protocols::{EndpointId, annotated::Annotated}, }; +use futures::stream::{self, StreamExt}; use crate::{ discovery::ModelManager, @@ -28,11 +29,12 @@ use crate::{ preprocessor::{BootstrapInfo, PrefillResult, TraceLink}, timing::{RequestPhase, RequestTracker}, }, - session_affinity::AffinityTarget, + session_affinity::{AffinityCoordinator, AffinityTarget}, }; mod activation; mod admission; +mod conditional_bypass; mod query; use admission::InnerPrefillRouter; @@ -126,6 +128,10 @@ struct PrefillCompletion { worker_link: Option, } +/// Annotation marker set when conditional disagg routes a request directly to +/// a DECODE-mode worker to run prefill+decode locally. +pub(crate) const BYPASS_REMOTE_PREFILL_ANNOTATION: &str = "x-bypass-remote-prefill"; + /// PrefillRouter is a forward-only operator that sits between Migration and the decode router. /// It optionally calls a prefill worker before routing to decode, extracting disaggregated_params /// from the prefill response and injecting them into the decode request. @@ -136,11 +142,21 @@ struct PrefillCompletion { /// - Normal: Worker IDs determined by router based on KV cache state pub struct PrefillRouter { prefill_router: OnceLock, + /// Reference to the decode-side `KvRouter` so conditional disagg can peek + /// the cache-hot decode worker. `None` for non-KV routing and disabled routers. + decode_router: Option>, + decode_session_affinity: OnceLock, model_manager: Arc, endpoint_id: OnceLock, cancel_token: CancellationToken, router_mode: RouterMode, session_affinity_ttl: Option, + conditional_disagg_policy: Box, + /// Resolved once at construction: dedicated threshold if set, otherwise + /// `router_queue_threshold`. `None` means the prefill-load condition is disabled. + conditional_disagg_prefill_busy_threshold: Option, + /// Dedicated decode-busy guard threshold. `None` means disabled. + conditional_disagg_decode_busy_threshold: Option, prefill_load_estimator: Option>, /// Model name (used for logging / lifecycle messages). model_name: String, @@ -178,6 +194,12 @@ impl let metadata = context.metadata().clone(); let engine_ctx = context.context(); + // Conditional-disagg bypass is a router-owned decision. Drop any + // client-supplied marker before the policy runs so normal disagg + // requests cannot accidentally or maliciously skip remote prefill. + req.annotations + .retain(|annotation| annotation != BYPASS_REMOTE_PREFILL_ANNOTATION); + // Save original max_tokens for decode let original_max_tokens = req.stop_conditions.max_tokens; @@ -192,6 +214,62 @@ impl .get_optional::(SESSION_AFFINITY_CONTEXT_KEY) .map_err(|message| anyhow::anyhow!("invalid session affinity context: {message}"))?; + let decode_affinity_target = + self.decode_session_affinity_target(session_affinity.as_deref())?; + + if self.conditional_disagg_policy.is_enabled() { + match self + .select_decode_worker_for_conditional_disagg( + &req, + &request_id, + decode_affinity_target, + ) + .await + { + Ok(Some(decision)) => { + tracing::info!( + request_id = %request_id, + worker_id = decision.worker.worker_id, + dp_rank = decision.worker.dp_rank, + net_new_tokens = decision.net_new_tokens, + overlap_tokens = decision.overlap_tokens, + "Conditional disagg routing to decode worker" + ); + + if req.tracker.is_none() { + req.tracker = Some(Arc::new(RequestTracker::new())); + } + if let Some(ref tracker) = req.tracker { + let _decode_permit = tracker.set_phase(RequestPhase::Decode).await; + } + + let routing = req.routing_mut(); + routing.decode_worker_id = Some(decision.worker.worker_id); + routing.dp_rank = Some(decision.worker.dp_rank); + + req.annotations + .push(BYPASS_REMOTE_PREFILL_ANNOTATION.to_string()); + + let response_stream = next.generate(context.map(|_| req)).await?; + let ctx = response_stream.context(); + let annotation = Annotated::::from_annotation( + BYPASS_REMOTE_PREFILL_ANNOTATION, + &true, + )?; + let merged = stream::once(async move { annotation }).chain(response_stream); + return Ok(ResponseStream::new(Box::pin(merged), ctx)); + } + Ok(None) => {} + Err(error) => { + tracing::warn!( + request_id = %request_id, + error = %error, + "Conditional disagg decision failed; falling back to remote prefill" + ); + } + } + } + // Ensure tracker exists for routing decisions in disaggregated mode. // Create one if not provided by the upstream DeltaGenerator. if req.tracker.is_none() { @@ -330,15 +408,50 @@ impl decode_req.stop_conditions.max_tokens = original_max_tokens; - // Decode should not score prompt overlap or account prompt-side load. + // Decode should not account prompt-side load. Normal disagg also + // forces zero overlap credit so decode routing stays load-only. let existing_override = decode_req.router_config_override.take(); - decode_req.router_config_override = Some(build_decode_router_override(existing_override)); + decode_req.router_config_override = Some(build_decode_router_override( + existing_override, + self.conditional_disagg_policy.is_enabled(), + )); next.generate(context.map(|_| decode_req)).await } } impl PrefillRouter { + pub(crate) fn get_or_create_decode_session_affinity( + &self, + ttl: Option, + ) -> Result> { + let Some(ttl) = ttl else { + return Ok(None); + }; + if let Some(affinity) = self.decode_session_affinity.get() { + return Ok(Some(affinity.clone())); + } + + let affinity = AffinityCoordinator::new(ttl)?; + if self.decode_session_affinity.set(affinity.clone()).is_ok() { + return Ok(Some(affinity)); + } + Ok(self.decode_session_affinity.get().cloned()) + } + + fn decode_session_affinity_target( + &self, + session_affinity: Option<&SessionAffinityId>, + ) -> Result> { + let Some(session_affinity) = session_affinity else { + return Ok(None); + }; + let Some(affinity) = self.decode_session_affinity.get() else { + return Ok(None); + }; + affinity.query_target(session_affinity, None) + } + fn prepare_prefill_dispatch( &self, request: &mut PreprocessedRequest, @@ -413,13 +526,21 @@ fn compute_bootstrap_room(dp_rank: Option, dp_size: Option, random_roo fn build_decode_router_override( existing_override: Option, + allow_decode_overlap_affinity: bool, ) -> RouterConfigOverride { - RouterConfigOverride { - overlap_score_credit: Some(0.0), - assume_kv_reuse: Some(false), - track_prefill_tokens: Some(false), - ..existing_override.unwrap_or_default() + let mut override_config = existing_override.unwrap_or_default(); + + // Normal disagg keeps decode routing load-only by forcing zero overlap + // credit. Conditional disagg leaves this unset so the base router + // `overlap_score_credit` applies, unless the request already had an + // explicit override. + if !allow_decode_overlap_affinity { + override_config.overlap_score_credit = Some(0.0); } + override_config.assume_kv_reuse = Some(false); + override_config.track_prefill_tokens = Some(false); + + override_config } fn merge_decode_topology_constraints( @@ -454,10 +575,14 @@ mod tests { #[test] fn decode_router_override_disables_overlap_and_prefill_tracking() { - let override_config = build_decode_router_override(Some(RouterConfigOverride { - router_temperature: Some(0.7), - ..Default::default() - })); + let override_config = build_decode_router_override( + Some(RouterConfigOverride { + overlap_score_credit: Some(0.5), + router_temperature: Some(0.7), + ..Default::default() + }), + false, + ); assert_eq!(override_config.overlap_score_credit, Some(0.0)); assert_eq!(override_config.assume_kv_reuse, Some(false)); @@ -465,6 +590,32 @@ mod tests { assert_eq!(override_config.router_temperature, Some(0.7)); } + #[test] + fn decode_router_override_inherits_base_overlap_when_conditional_disagg_allows_it() { + let override_config = build_decode_router_override(None, true); + + assert_eq!(override_config.overlap_score_credit, None); + assert_eq!(override_config.assume_kv_reuse, Some(false)); + assert_eq!(override_config.track_prefill_tokens, Some(false)); + } + + #[test] + fn decode_router_override_preserves_request_overlap_when_conditional_disagg_allows_it() { + let override_config = build_decode_router_override( + Some(RouterConfigOverride { + overlap_score_credit: Some(0.25), + router_temperature: Some(0.7), + ..Default::default() + }), + true, + ); + + assert_eq!(override_config.overlap_score_credit, Some(0.25)); + assert_eq!(override_config.assume_kv_reuse, Some(false)); + assert_eq!(override_config.track_prefill_tokens, Some(false)); + assert_eq!(override_config.router_temperature, Some(0.7)); + } + #[test] fn bootstrap_room_falls_back_when_dp_unavailable() { assert_eq!(compute_bootstrap_room(None, None, 12345), 12345); diff --git a/lib/llm/src/kv_router/push_router.rs b/lib/llm/src/kv_router/push_router.rs index 5f16f05c2867..4dba0afbe068 100644 --- a/lib/llm/src/kv_router/push_router.rs +++ b/lib/llm/src/kv_router/push_router.rs @@ -17,7 +17,9 @@ use futures::stream::{self, StreamExt}; use tracing::Instrument; use crate::{ - kv_router::{KvRouter, metrics::RouterRequestMetrics}, + kv_router::{ + KvRouter, metrics::RouterRequestMetrics, prefill_router::BYPASS_REMOTE_PREFILL_ANNOTATION, + }, preprocessor::PreprocessedRequest, protocols::common::{ llm_backend::LLMEngineOutput, @@ -67,7 +69,14 @@ impl KvPushRouter { let affinity = session_affinity_ttl .map(AffinityCoordinator::new) .transpose()?; + Self::new_with_affinity(inner, chooser, affinity) + } + pub(crate) fn new_with_affinity( + inner: PushRouter>, + chooser: Arc, + affinity: Option, + ) -> Result { // Eagerly register router request metrics (as zeros) so they are // scrapeable before any requests arrive. Both the frontend pipeline // and the standalone router create KvPushRouter, so this covers both. @@ -301,6 +310,7 @@ impl KvPushRouter { let phase_label = phase.to_string(); guard.start_dispatch(&phase_label); self.warn_if_output_replay_annotation_ignored(&request, &selection); + let bypassed_remote_prefill = request.has_annotation(BYPASS_REMOTE_PREFILL_ANNOTATION); let (mut backend_input, context) = request.into_parts(); backend_input.routing_mut().dp_rank = Some(selection.dp_rank); @@ -328,6 +338,7 @@ impl KvPushRouter { dp_rank = selection.dp_rank, overlap_blocks = selection.overlap_amount, phase = ?phase, + bypassed = bypassed_remote_prefill, )), ) .await diff --git a/lib/llm/src/kv_router/push_router/selection.rs b/lib/llm/src/kv_router/push_router/selection.rs index 059a42c691f7..34ef4a8ea28f 100644 --- a/lib/llm/src/kv_router/push_router/selection.rs +++ b/lib/llm/src/kv_router/push_router/selection.rs @@ -101,6 +101,7 @@ impl KvPushRouter { overlap_blocks, effective_overlap_blocks, cached_tokens, + potential_decode_blocks: _, routing_hashes, } => Ok(WorkerSelection { instance_id: worker.worker_id, diff --git a/lib/llm/src/kv_router/scheduler.rs b/lib/llm/src/kv_router/scheduler.rs index 843bdbf254d3..1c90369c7078 100644 --- a/lib/llm/src/kv_router/scheduler.rs +++ b/lib/llm/src/kv_router/scheduler.rs @@ -321,6 +321,14 @@ where self.update_queue_metrics(); } + /// Select a worker from current scheduler state without queue admission or booking. + pub async fn select_without_admission( + &self, + request: ScheduleRequest, + ) -> Result { + self.inner.select_without_admission(request).await + } + pub fn register_workers(&self, worker_ids: &HashSet) { self.inner.register_workers(worker_ids); } @@ -349,6 +357,30 @@ where self.inner.pending_isl_tokens() } + pub fn worker_is_prefill_busy( + &self, + worker: WorkerWithDpRank, + decay_now: tokio::time::Instant, + threshold: f64, + ) -> Option { + self.inner + .worker_is_prefill_busy(worker, decay_now, threshold) + } + + pub fn worker_is_decode_busy(&self, worker: WorkerWithDpRank, threshold: f64) -> Option { + self.inner.worker_is_decode_busy(worker, threshold) + } + + pub fn projected_decode_load_exceeds( + &self, + worker: WorkerWithDpRank, + projected_blocks: usize, + threshold: f64, + ) -> Option { + self.inner + .projected_decode_load_exceeds(worker, projected_blocks, threshold) + } + pub fn worker_type(&self) -> &'static str { self.inner.worker_type() }