Skip to content
132 changes: 132 additions & 0 deletions components/src/dynamo/common/configuration/groups/kv_router_args.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
"""

import argparse
import logging
import os
import warnings
from typing import Optional
Expand Down Expand Up @@ -51,9 +52,24 @@
"serve_indexer",
"shared_cache_multiplier",
"shared_cache_type",
"conditional_disagg_enabled",
"conditional_disagg_policy",
"conditional_disagg_eff_isl_threshold",
"conditional_disagg_eff_isl_ratio_threshold",
"conditional_disagg_prefill_busy_threshold",
"conditional_disagg_decode_busy_threshold",
"router_predicted_ttl_secs",
)

CONDITIONAL_DISAGG_POLICY_CHOICES: tuple[str, ...] = (
"isl_bounding",
"prefill_load",
"isl_or_load",
)
LOAD_AWARE_CONDITIONAL_DISAGG_POLICIES: frozenset[str] = frozenset(
{"prefill_load", "isl_or_load"}
)

_DEPRECATED_OVERLAP_WEIGHT_MESSAGE = (
"router KV overlap score weight is deprecated; use "
"--router-prefill-load-scale or DYN_ROUTER_PREFILL_LOAD_SCALE for equivalent behavior"
Expand Down Expand Up @@ -105,6 +121,35 @@ def _default_prefill_load_scale() -> float:
return 1.0


def warn_conditional_disagg_prefill_busy_threshold_resolution(
*,
policy: str,
busy_threshold: Optional[float],
queue_threshold: Optional[float],
) -> None:
if policy not in LOAD_AWARE_CONDITIONAL_DISAGG_POLICIES:
return
if busy_threshold is not None:
logging.getLogger(__name__).info(
"conditional_disagg load gate using "
"--router-conditional-disagg-prefill-busy-threshold=%s",
busy_threshold,
)
elif queue_threshold is not None:
logging.getLogger(__name__).info(
"conditional_disagg load gate inheriting --router-queue-threshold=%s",
queue_threshold,
)
else:
warnings.warn(
f"--router-conditional-disagg-policy={policy!r} consumes the "
"prefill-worker busy signal, but neither "
"--router-conditional-disagg-prefill-busy-threshold nor "
"--router-queue-threshold is set; the load gate will be a no-op.",
stacklevel=3,
)


class KvRouterConfigBase(ConfigBase):
"""Mixin carrying the shared KvRouterConfig fields."""

Expand Down Expand Up @@ -134,6 +179,12 @@ class KvRouterConfigBase(ConfigBase):
serve_indexer: bool = False
shared_cache_multiplier: float = 0.0
shared_cache_type: str = "none"
conditional_disagg_enabled: bool = False
conditional_disagg_policy: str = "isl_bounding"
conditional_disagg_eff_isl_threshold: int = 2048
conditional_disagg_eff_isl_ratio_threshold: float = 0.7
conditional_disagg_prefill_busy_threshold: Optional[float] = None
conditional_disagg_decode_busy_threshold: Optional[float] = None
router_predicted_ttl_secs: Optional[float] = None
load_aware: bool = False

Expand Down Expand Up @@ -416,6 +467,87 @@ def add_arguments(self, parser) -> None:
),
arg_type=str,
)
add_negatable_bool_argument(
g,
flag_name="--router-conditional-disagg",
env_var="DYN_ROUTER_CONDITIONAL_DISAGG",
default=False,
help=(
"[EXPERIMENTAL] KV Router: Enable conditional disaggregation. "
"When enabled, the frontend may skip remote prefill and route "
"a request directly to the cache-hot decode worker when the "
"configured policy decides local prefill+decode is preferable."
),
dest="conditional_disagg_enabled",
)
add_argument(
g,
flag_name="--router-conditional-disagg-policy",
env_var="DYN_ROUTER_CONDITIONAL_DISAGG_POLICY",
default="isl_bounding",
help=(
"KV Router: Conditional-disagg bypass policy. "
"'isl_bounding': bypass when effective ISL is below both the "
"absolute and ratio thresholds. 'prefill_load': bypass when "
"the chosen prefill worker is busy. 'isl_or_load': bypass "
"when either policy would bypass."
),
arg_type=str,
choices=list(CONDITIONAL_DISAGG_POLICY_CHOICES),
dest="conditional_disagg_policy",
)
add_argument(
g,
flag_name="--router-conditional-disagg-eff-isl-threshold",
env_var="DYN_ROUTER_CONDITIONAL_DISAGG_EFF_ISL_THRESHOLD",
default=2048,
help=(
"KV Router: Effective-ISL token cutoff for the 'isl_bounding' "
"conditional-disagg policy."
),
arg_type=int,
dest="conditional_disagg_eff_isl_threshold",
)
add_argument(
g,
flag_name="--router-conditional-disagg-eff-isl-ratio-threshold",
env_var="DYN_ROUTER_CONDITIONAL_DISAGG_EFF_ISL_RATIO_THRESHOLD",
default=0.7,
help=(
"KV Router: Effective-ISL/prompt-token ratio cutoff for the "
"'isl_bounding' conditional-disagg policy."
),
arg_type=float,
dest="conditional_disagg_eff_isl_ratio_threshold",
)
add_argument(
g,
flag_name="--router-conditional-disagg-prefill-busy-threshold",
env_var="DYN_ROUTER_CONDITIONAL_DISAGG_PREFILL_BUSY_THRESHOLD",
default=None,
help=(
"KV Router: Dedicated prefill-worker busy-line fraction for "
"'prefill_load' and 'isl_or_load'. A worker is busy when "
"active_tokens(W) > threshold * max_num_batched_tokens(W). "
"If unset, falls back to --router-queue-threshold."
),
arg_type=float,
dest="conditional_disagg_prefill_busy_threshold",
)
add_argument(
g,
flag_name="--router-conditional-disagg-decode-busy-threshold",
env_var="DYN_ROUTER_CONDITIONAL_DISAGG_DECODE_BUSY_THRESHOLD",
default=None,
help=(
"KV Router: Decode-side circuit breaker for conditional disagg. "
"Denies bypass when active_decode_blocks(W) exceeds "
"threshold * total_kv_blocks(W) for the chosen decode worker. "
"If unset, the decode gate is disabled."
),
arg_type=float,
dest="conditional_disagg_decode_busy_threshold",
)
add_argument(
g,
flag_name="--router-event-threads",
Expand Down
25 changes: 25 additions & 0 deletions components/src/dynamo/frontend/frontend_args.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,10 @@
AicPerfConfigBase,
)
from dynamo.common.configuration.groups.kv_router_args import (
CONDITIONAL_DISAGG_POLICY_CHOICES,
KvRouterArgGroup,
KvRouterConfigBase,
warn_conditional_disagg_prefill_busy_threshold_resolution,
)
from dynamo.common.configuration.groups.router_args import (
RouterArgGroup,
Expand Down Expand Up @@ -157,6 +159,29 @@ def validate(self) -> None:
raise ValueError(
"--serve-indexer and --use-remote-indexer are mutually exclusive"
)
if self.conditional_disagg_policy not in CONDITIONAL_DISAGG_POLICY_CHOICES:
raise ValueError(
"--router-conditional-disagg-policy must be one of "
+ ", ".join(
f"'{choice}'" for choice in CONDITIONAL_DISAGG_POLICY_CHOICES
)
)
if self.conditional_disagg_eff_isl_threshold < 0:
raise ValueError(
"--router-conditional-disagg-eff-isl-threshold must be >= 0"
)
if not 0.0 <= self.conditional_disagg_eff_isl_ratio_threshold <= 1.0:
raise ValueError(
"--router-conditional-disagg-eff-isl-ratio-threshold must be in [0.0, 1.0]"
)
if self.conditional_disagg_enabled and self.router_mode != "kv":
raise ValueError("--router-conditional-disagg requires --router-mode=kv")
if self.conditional_disagg_enabled:
warn_conditional_disagg_prefill_busy_threshold_resolution(
policy=self.conditional_disagg_policy,
busy_threshold=self.conditional_disagg_prefill_busy_threshold,
queue_threshold=self.router_queue_threshold,
)
self.apply_admission_control()


Expand Down
23 changes: 23 additions & 0 deletions components/src/dynamo/router/args.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,10 @@
AicPerfConfigBase,
)
from dynamo.common.configuration.groups.kv_router_args import (
CONDITIONAL_DISAGG_POLICY_CHOICES,
KvRouterArgGroup,
KvRouterConfigBase,
warn_conditional_disagg_prefill_busy_threshold_resolution,
)
from dynamo.common.configuration.utils import add_argument, add_negatable_bool_argument
from dynamo.llm import AicPerfConfig, KvRouterConfig
Expand Down Expand Up @@ -66,6 +68,27 @@ def validate(self) -> None:
"--router-prefill-load-model=aic requires "
"--router-track-prefill-tokens"
)
if self.conditional_disagg_policy not in CONDITIONAL_DISAGG_POLICY_CHOICES:
Comment thread
karen-sy marked this conversation as resolved.
Resolved
raise ValueError(
"--router-conditional-disagg-policy must be one of "
+ ", ".join(
f"'{choice}'" for choice in CONDITIONAL_DISAGG_POLICY_CHOICES
)
)
if self.conditional_disagg_eff_isl_threshold < 0:
raise ValueError(
"--router-conditional-disagg-eff-isl-threshold must be >= 0"
)
if not 0.0 <= self.conditional_disagg_eff_isl_ratio_threshold <= 1.0:
raise ValueError(
"--router-conditional-disagg-eff-isl-ratio-threshold must be in [0.0, 1.0]"
)
if self.conditional_disagg_enabled:

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The standalone router accepts and forwards this flag, but its handler constructs only KvRouter; the bypass policy is consumed by PrefillRouter, which this path does not instantiate. As written, the flag succeeds but has no conditional-bypass behavior. Could standalone reject it explicitly or be wired through the two-stage router?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Removed all conditional disagg related code for standalone router. Looks like standalone router doesn't acknowledge the kv router flags it does not support (rather than reject those flags explicitly)?

warn_conditional_disagg_prefill_busy_threshold_resolution(
policy=self.conditional_disagg_policy,
busy_threshold=self.conditional_disagg_prefill_busy_threshold,
queue_threshold=self.router_queue_threshold,
)


class DynamoRouterArgGroup(ArgGroup):
Expand Down
31 changes: 26 additions & 5 deletions components/src/dynamo/trtllm/request_handlers/handler_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@

logger = logging.getLogger(__name__)

BYPASS_REMOTE_PREFILL_ANNOTATION = "x-bypass-remote-prefill"


class TRTLLMEnginePauseController:
"""Adapts TRT-LLM sleep/wake to the standard pause controller interface.
Expand Down Expand Up @@ -685,9 +687,15 @@ def _setup_disaggregated_params_for_mode(
disaggregated_params = None
epd_metadata: dict[str, Any] = {}

# Canary probe: use its pre-built disagg params (skip prefill_result decode
# and skip the mode-specific request_type overrides).
if request.get(HEALTH_CHECK_KEY) and request.get("disaggregated_params"):
use_request_disagg_params = request.get(HEALTH_CHECK_KEY) or (
self.disaggregation_mode == DisaggregationMode.DECODE
and BYPASS_REMOTE_PREFILL_ANNOTATION in (request.get("annotations") or [])
)

# Canary probes and conditional-disagg bypasses run a full
# context+generation request on a disagg-mode worker, so they use
# the pre-built params and skip the normal prefill-result handoff.
if use_request_disagg_params and request.get("disaggregated_params"):
return LlmDisaggregatedParams(**request["disaggregated_params"]), None, {}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved

# PREFILL mode: setup context_only params
Expand Down Expand Up @@ -966,6 +974,16 @@ async def _generate_locally_impl(

# Normalize OpenAI format to TRT-LLM internal format
self._normalize_request_format(request)
bypass_remote_prefill = (
self.disaggregation_mode == DisaggregationMode.DECODE
and BYPASS_REMOTE_PREFILL_ANNOTATION in (request.get("annotations") or [])
)
if bypass_remote_prefill:
logging.debug(
"DECODE: conditional-disagg bypass annotation present; "
"running request as AGG (prefill+decode on this worker)."
)
request["disaggregated_params"] = {"request_type": "context_and_generation"}

# Setup disaggregated params based on PREFILL/DECODE mode
(
Expand Down Expand Up @@ -995,6 +1013,7 @@ async def _generate_locally_impl(
if (
self.disaggregation_mode == DisaggregationMode.DECODE
and disaggregated_params is None
and not bypass_remote_prefill
):
logging.error("DECODE: disaggregated_params is None but required!")
logging.error(f"DECODE: Request keys: {list(request.keys())}")
Expand Down Expand Up @@ -1123,11 +1142,13 @@ async def _generate_locally_impl(
priority=priority,
)

# In disagg decode mode, wrap abort() to defer until first token
# (KV transfer complete).
# In disagg decode mode with remote prefill, wrap abort() to defer
# until the first token is received (KV transfer complete).
abort_guard = (
_DeferredAbort(generation_result)
if self.disaggregation_mode == DisaggregationMode.DECODE
and disaggregated_params is not None
and not bypass_remote_prefill
else None
)

Expand Down
24 changes: 23 additions & 1 deletion components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,10 @@
from dynamo.trtllm.constants import DisaggregationMode
from dynamo.trtllm.health_check import TrtllmHealthCheckPayload
from dynamo.trtllm.llm_engine import TrtllmLLMEngine
from dynamo.trtllm.request_handlers.handler_base import HandlerBase
from dynamo.trtllm.request_handlers.handler_base import (
BYPASS_REMOTE_PREFILL_ANNOTATION,
HandlerBase,
)

pytestmark = [
pytest.mark.unit,
Expand Down Expand Up @@ -575,6 +578,13 @@ def _make_prefill_handler(self, machine_id: int = 42) -> HandlerBase:
handler.disaggregation_mode = DisaggregationMode.PREFILL
return handler

def _make_decode_handler(self) -> HandlerBase:
config = MagicMock()
config.shutdown_event = None
handler = _ConcreteHandler(config)
handler.disaggregation_mode = DisaggregationMode.DECODE
return handler

def test_disagg_request_id_populated_in_prefill_mode(self):
"""When mode is PREFILL and no ep_disaggregated_params, disagg_request_id is set."""
handler = self._make_prefill_handler()
Expand Down Expand Up @@ -640,6 +650,18 @@ def test_different_machine_ids_produce_different_id_ranges(self):
)
assert params_a.disagg_request_id != params_b.disagg_request_id

def test_decode_conditional_bypass_uses_request_disagg_params(self):
"""Conditional-disagg bypass runs full context+generation on decode."""
handler = self._make_decode_handler()
params, _, _ = handler._setup_disaggregated_params_for_mode(
request={
"annotations": [BYPASS_REMOTE_PREFILL_ANNOTATION],
"disaggregated_params": {"request_type": "context_and_generation"},
},
ep_disaggregated_params=None,
)
assert params.request_type == "context_and_generation"


class TestHealthCheckPriority:
"""Verify generate_locally forwards the correct priority to generate_async.
Expand Down
7 changes: 5 additions & 2 deletions components/src/dynamo/vllm/args.py
Original file line number Diff line number Diff line change
Expand Up @@ -389,10 +389,13 @@ def create_kv_events_config(
dynamo_config: Config, engine_config: AsyncEngineArgs
) -> Optional[KVEventsConfig]:
"""Create KVEventsConfig for prefix caching if needed."""
if dynamo_config.disaggregation_mode == DisaggregationMode.DECODE:
if (
dynamo_config.disaggregation_mode == DisaggregationMode.DECODE
and not dynamo_config.enable_conditional_disagg
):
logger.info(
"Decode worker detected (disaggregation_mode=decode): "
"kv_events_config disabled (decode workers don't publish KV events)"
"kv_events_config disabled (pass --enable-conditional-disagg to opt in)"
)
return None

Expand Down
Loading
Loading