Skip to content
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
dd87bae
feat(router): add policy-class queues
PeaBrane Jun 16, 2026
e4e2909
docs(router): document policy-class queues
PeaBrane Jun 16, 2026
ce2933a
refactor(router): simplify overlap refresh dequeue
PeaBrane Jun 16, 2026
35c8460
perf(router): index policy-class queue metrics
PeaBrane Jun 16, 2026
264c181
chore(ci): sync nested Rust workspaces
PeaBrane Jun 16, 2026
94fe053
fix(bindings): handle policy queue rejection in CI paths
PeaBrane Jun 16, 2026
e35e4b3
fix(router): repair strict config runtime CI
PeaBrane Jun 16, 2026
747d468
fix(router): close policy queue review gaps
PeaBrane Jun 16, 2026
8652c68
fix(router): repair policy queue CI paths
PeaBrane Jun 16, 2026
91d269c
test(router): remove low-signal policy config tests
PeaBrane Jun 16, 2026
718e508
test(router): trim redundant config coverage
PeaBrane Jun 16, 2026
40a08c6
test(router): consolidate legacy config coverage
PeaBrane Jun 16, 2026
cf3d8f0
Merge main without dropping policy classes
PeaBrane Jun 16, 2026
e49d8dd
feat(router): map uncached ISL to policy classes
PeaBrane Jun 16, 2026
ce4ad9b
docs(router): document uncached ISL class mapping
PeaBrane Jun 16, 2026
3626769
test(router): remove low-signal policy assertions
PeaBrane Jun 16, 2026
417635e
feat(router): scale policy queue limits per worker
PeaBrane Jun 16, 2026
90537fe
Merge origin/main into codex/policy-class-queues
PeaBrane Jun 16, 2026
877af8c
feat(router): classify policy families by cache bucket
PeaBrane Jun 16, 2026
0fc7398
docs(router): clarify DRR credit flow
PeaBrane Jun 16, 2026
78c3e11
docs(router): explain policy-class DRR scheduling
PeaBrane Jun 16, 2026
4a93a00
Merge main into policy-class queues
PeaBrane Jun 18, 2026
ddb0565
Merge main into policy queues
PeaBrane Jun 23, 2026
bfae063
Fix policy queue clippy after merge
PeaBrane Jun 23, 2026
2429b50
Merge main into policy queues
PeaBrane Jun 23, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
"router_reset_states",
"router_ttl_secs",
"router_queue_threshold",
"router_queue_by_incoming_missing_isl",
"router_policy_config",
"router_event_threads",
"router_queue_policy",
"use_remote_indexer",
Expand Down Expand Up @@ -127,7 +127,7 @@ class KvRouterConfigBase(ConfigBase):
router_reset_states: bool
router_ttl_secs: float
router_queue_threshold: Optional[float]
router_queue_by_incoming_missing_isl: Optional[list[tuple[int, int]]] = None
router_policy_config: Optional[str] = None
router_event_threads: int
router_queue_policy: str
use_remote_indexer: bool = False
Expand Down Expand Up @@ -403,6 +403,18 @@ def add_arguments(self, parser) -> None:
),
arg_type=nullable_float,
)
add_argument(
g,
flag_name="--router-policy-config",
env_var="DYN_ROUTER_POLICY_CONFIG",
default=None,
help=(
"KV Router: Startup-only YAML policy-class configuration. "
"When omitted, router_queue_threshold and router_queue_policy define "
"the existing single default queue."
),
arg_type=str,
)
add_argument(
g,
flag_name="--router-event-threads",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
# SPDX-License-Identifier: Apache-2.0

import argparse
from pathlib import Path

import pytest

Expand Down Expand Up @@ -402,27 +403,43 @@ def test_load_aware_preserves_cache_hit_weights() -> None:
assert kwargs["disk_cache_hit_weight"] == 0.1


def test_kv_router_kwargs_preserves_explicit_queue_tiers() -> None:
def test_kv_router_kwargs_preserves_policy_config_path(tmp_path: Path) -> None:
parser = argparse.ArgumentParser()
KvRouterArgGroup().add_arguments(parser)

config = KvRouterConfigBase.from_cli_args(parser.parse_args([]))
config.router_queue_by_incoming_missing_isl = [(0, 2048), (2048, 512)]
policy_path = str(tmp_path / "router-policy.yaml")
config.router_policy_config = policy_path

kwargs = config.kv_router_kwargs()

assert kwargs["router_queue_by_incoming_missing_isl"] == [(0, 2048), (2048, 512)]
assert kwargs["router_policy_config"] == policy_path


def test_kv_router_kwargs_uses_unbounded_queue_cap_by_default() -> None:
def test_policy_config_cli_overrides_environment(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
env_policy_path = str(tmp_path / "env-policy.yaml")
explicit_policy_path = str(tmp_path / "explicit-policy.yaml")
monkeypatch.setenv("DYN_ROUTER_POLICY_CONFIG", env_policy_path)
parser = argparse.ArgumentParser()
KvRouterArgGroup().add_arguments(parser)

args = parser.parse_args(["--router-policy-config", explicit_policy_path])
config = KvRouterConfigBase.from_cli_args(args)

assert config.kv_router_kwargs()["router_policy_config"] == explicit_policy_path


def test_kv_router_kwargs_omits_policy_config_by_default() -> None:
parser = argparse.ArgumentParser()
KvRouterArgGroup().add_arguments(parser)

config = KvRouterConfigBase.from_cli_args(parser.parse_args([]))

kwargs = config.kv_router_kwargs()

assert kwargs["router_queue_by_incoming_missing_isl"] is None
assert kwargs["router_policy_config"] is None


def test_load_aware_clears_predicted_ttl() -> None:
Expand Down
2 changes: 2 additions & 0 deletions components/src/dynamo/mocker/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ def run_mocker_trace_replay(
trace_format="mooncake",
trace_shared_prefix_ratio=0.0,
trace_num_prefix_groups=0,
model_name=None,
):
return _run_mocker_trace_replay(
trace_file,
Expand All @@ -64,6 +65,7 @@ def run_mocker_trace_replay(
trace_format=trace_format,
trace_shared_prefix_ratio=trace_shared_prefix_ratio,
trace_num_prefix_groups=trace_num_prefix_groups,
model_name=model_name,
)


Expand Down
4 changes: 4 additions & 0 deletions components/src/dynamo/replay/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ def run_trace_replay(
trace_num_prefix_groups=0,
report_jsonl_path=None,
max_sim_time_ms=None,
model_name=None,
):
return _run_mocker_trace_replay(
trace_file,
Expand All @@ -49,6 +50,7 @@ def run_trace_replay(
trace_num_prefix_groups=trace_num_prefix_groups,
report_jsonl_path=report_jsonl_path,
max_sim_time_ms=max_sim_time_ms,
model_name=model_name,
)


Expand All @@ -74,6 +76,7 @@ def run_synthetic_trace_replay(
shared_prefix_ratio=0.0,
num_prefix_groups=0,
inter_turn_delay_ms=0.0,
model_name=None,
):
return _run_mocker_synthetic_trace_replay(
input_tokens,
Expand All @@ -96,4 +99,5 @@ def run_synthetic_trace_replay(
shared_prefix_ratio=shared_prefix_ratio,
num_prefix_groups=num_prefix_groups,
inter_turn_delay_ms=inter_turn_delay_ms,
model_name=model_name,
)
46 changes: 42 additions & 4 deletions components/src/dynamo/replay/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import argparse
import importlib
import json
import os
import sys
from collections.abc import Sequence
from pathlib import Path
Expand Down Expand Up @@ -38,6 +39,26 @@ class PlannerProfileDataResult(Protocol):
_DEFAULT_AIC_SYSTEM = "h200_sxm"
_DEFAULT_MAX_NUM_BATCHED_TOKENS = 8192
_DEFAULT_VLLM_BLOCK_SIZE = 64


def _load_router_config(
router_config_json: str | None,
router_policy_config: str | None,
):
if router_policy_config is None:
return (
KvRouterConfig.from_json(router_config_json)
if router_config_json is not None
else None
)

values = json.loads(router_config_json) if router_config_json is not None else {}
if not isinstance(values, dict):
raise ValueError("--router-config must contain a JSON object")
values["router_policy_config"] = router_policy_config
return KvRouterConfig.from_json(json.dumps(values))


_DEFAULT_SGLANG_BLOCK_SIZE = 1
_DEFAULT_TRTLLM_BLOCK_SIZE = 32

Expand Down Expand Up @@ -324,6 +345,7 @@ def _run_planner_replay(
arrival_speedup_ratio: float,
trace_block_size: int,
planner_config_arg: str,
model_name: str | None = None,
benchmark_granularity: int = 8,
):
"""Run an offline replay with planner-in-the-loop (agg or disagg).
Expand Down Expand Up @@ -353,6 +375,7 @@ def _run_planner_replay(
num_workers=num_workers,
router_mode=router_mode,
router_config=router_config,
model_name=model_name,
arrival_speedup_ratio=arrival_speedup_ratio,
trace_block_size=trace_block_size,
)
Expand All @@ -371,6 +394,7 @@ def _run_planner_replay(
num_decode_workers=num_decode_workers,
router_mode=router_mode,
router_config=router_config,
model_name=model_name,
arrival_speedup_ratio=arrival_speedup_ratio,
trace_block_size=trace_block_size,
)
Expand Down Expand Up @@ -518,6 +542,18 @@ def main(argv: Sequence[str] | None = None) -> int:
parser.add_argument("--prefill-engine-args")
parser.add_argument("--decode-engine-args")
parser.add_argument("--router-config")
parser.add_argument(
"--router-policy-config",
default=os.environ.get("DYN_ROUTER_POLICY_CONFIG"),
help=(
"startup-only policy-class YAML path; overrides router_policy_config "
"inside --router-config"
),
)
parser.add_argument(
"--model-name",
help="model profile to select from --router-policy-config",
)
parser.add_argument("--aic-backend")
parser.add_argument("--aic-system")
parser.add_argument("--aic-backend-version")
Expand Down Expand Up @@ -668,10 +704,9 @@ def main(argv: Sequence[str] | None = None) -> int:
extra_engine_args = _load_engine_args(args.extra_engine_args)
prefill_engine_args = _load_engine_args(args.prefill_engine_args)
decode_engine_args = _load_engine_args(args.decode_engine_args)
router_config = (
KvRouterConfig.from_json(args.router_config)
if args.router_config is not None
else None
router_config = _load_router_config(
args.router_config,
args.router_policy_config,
)
try:
aic_perf_config = _load_aic_perf_config(args)
Expand Down Expand Up @@ -700,6 +735,7 @@ def main(argv: Sequence[str] | None = None) -> int:
arrival_speedup_ratio=args.arrival_speedup_ratio,
trace_block_size=args.trace_block_size,
planner_config_arg=args.planner_config,
model_name=args.model_name,
benchmark_granularity=args.benchmark_granularity,
)
report = planner_report.trace_report
Expand Down Expand Up @@ -748,6 +784,7 @@ def main(argv: Sequence[str] | None = None) -> int:
trace_num_prefix_groups=args.trace_num_prefix_groups,
report_jsonl_path=args.report_jsonl,
max_sim_time_ms=max_sim_time_ms,
model_name=args.model_name,
)
else:
report = run_synthetic_trace_replay(
Expand All @@ -771,6 +808,7 @@ def main(argv: Sequence[str] | None = None) -> int:
shared_prefix_ratio=args.shared_prefix_ratio,
num_prefix_groups=args.num_prefix_groups,
inter_turn_delay_ms=args.inter_turn_delay_ms,
model_name=args.model_name,
)

report_path = write_report_json(report, args.report_json)
Expand Down
1 change: 1 addition & 0 deletions docs/components/frontend/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ The Rust HTTP server also reads these environment variables (not exposed as CLI
| `--router-event-threads` | `DYN_ROUTER_EVENT_THREADS` | `4` | KV indexer worker threads. >1 enables the concurrent radix tree, including with `--no-router-kv-events` |
| `--router-queue-threshold` | `DYN_ROUTER_QUEUE_THRESHOLD` | `16.0` | Queue threshold fraction of prefill capacity. Priority hints only affect requests waiting in this queue |
| `--router-queue-policy` | `DYN_ROUTER_QUEUE_POLICY` | `fcfs` | Queue scheduling policy: `fcfs` (tail TTFT), `wspt` (avg TTFT), or `lcfs` (comparison-only reverse ordering) |
| `--router-policy-config` | `DYN_ROUTER_POLICY_CONFIG` | — | Startup-only policy-class YAML. Falls back to the single queue configured above when omitted |
| `--decode-fallback` / `--no-decode-fallback` | `DYN_DECODE_FALLBACK` | `false` | Fall back to aggregated mode when prefill workers unavailable |

## AIC Prefill Load Model
Expand Down
39 changes: 39 additions & 0 deletions docs/components/router/router-configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ This page collects the main router flags for frontend-embedded and standalone de
- `--router-prefill-load-model`: Selects the router's prompt-side load model. `none` keeps the existing static prompt load accounting. `aic` predicts one expected prefill duration per admitted request and lazily decays only the oldest active prefill request on each worker.
- `--router-queue-threshold`: Queue threshold fraction for prefill token capacity (default: 16.0). The router holds incoming requests in a priority queue while all eligible workers exceed `threshold * max_num_batched_tokens`, releasing them when capacity frees up. This defers dispatch rather than rejecting work, so routing decisions use the freshest load metrics at the moment a request is actually sent to a worker. `nvext.agent_hints.strict_priority` selects an absolute pending-queue tier, while `nvext.agent_hints.priority` adjusts ordering within the configured policy. Must be greater than or equal to 0; use `0.0` for maximum queueing sensitivity. Set to `None` to disable queueing. See the SGLang note under [Tuning Guidelines](#tuning-guidelines) for caveats around how `max_num_batched_tokens` is populated on that backend, and see [Priority Scheduling](../../agents/priority-scheduling.md) for how router priority differs from backend engine priority.
- `--router-queue-policy`: Scheduling policy for the router queue (default: `fcfs`).
- `--router-policy-config`: Startup-only policy-class YAML path. When omitted, `--router-queue-threshold` and `--router-queue-policy` retain the single default queue. The equivalent environment variable is `DYN_ROUTER_POLICY_CONFIG`.

For how queue backpressure differs from candidate filtering and busy-threshold overload handling, see [Router Filtering](router-filtering.md).

Expand All @@ -31,6 +32,44 @@ For all three policies, the complete pending-queue key is
`(strict_priority, policy_key)`. Higher strict tiers always win; the selected
policy orders requests within a tier.

### Policy-Class Queues

Clients select a class with `x-dynamo-meta-policy-class`. Dynamo carries the
header as request context metadata; it does not infer cached versus uncached
work from the prompt. Missing and unknown class names use
`default_policy_class`.

Each class owns its FCFS or WSPT heap, busy thresholds, queue limits, quantum,
deficit, and counters. Absolute and fractional busy thresholds use OR
semantics. When neither is specified, the fractional threshold defaults to
`16.0`. A class queues only when every eligible worker is busy for that class,
but a new arrival cannot bypass an existing backlog in the same class.

Queue limits are checked against the current class-local usage before adding
the incoming request. The request that crosses a limit is accepted; the next
queued request is rejected with HTTP 503 and structured limit details. DRR
charges the uncached-token snapshot captured at enqueue, while raw, cached,
and uncached snapshots remain unchanged for limits, WSPT, counters, and later
dispatch.

Profiles resolve in this order: exact model profile, root profile, then the
synthetic single-class fallback. A model profile completely replaces the root
profile; fields and classes are not inherited. See the tested
[sample policy](../../../examples/router/policy-class-queues.yaml).

```bash
python -m dynamo.frontend \
--router-mode kv \
--router-policy-config examples/router/policy-class-queues.yaml
```

The previous missing-ISL tier configuration is removed. Mapping those tiers to
client-selected classes is intentionally not behavior preserving: external
cache-state identification must happen before the request reaches Dynamo, and
weighted DRR replaces the previous shared-queue credit-gap behavior. A
`cached` WSPT class and an `uncached` FCFS class, as shown in the sample, are a
starting point rather than an exact compatibility profile.

For `--router-mode device-aware-weighted`, set `DYN_ENCODER_CUDA_TO_CPU_RATIO` to the approximate throughput ratio of one non-CPU worker relative to one CPU worker. The default is `8`.

### AIC Prefill Load Model
Expand Down
9 changes: 9 additions & 0 deletions docs/dynosim/runs.md
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@ The trace file must be Mooncake-style JSONL. Each line should contain:
- `hash_ids`
- optional `priority` (signed soft-priority hint)
- optional `strict_priority` (unsigned queue tier; larger values run first)
- optional `policy_class` (client-selected router policy class)

Example:

Expand Down Expand Up @@ -246,6 +247,8 @@ The dedicated DynoSim CLI exposes:
- `--prefill-engine-args` (JSON string)
- `--decode-engine-args` (JSON string)
- `--router-config` (JSON string)
- `--router-policy-config` (policy-class YAML path; `DYN_ROUTER_POLICY_CONFIG` fallback)
- `--model-name` (selects an exact model profile from the policy YAML)
- `--aic-backend`
- `--aic-system`
- `--aic-backend-version`
Expand Down Expand Up @@ -294,6 +297,12 @@ as `block_size`, `engine_type`, `dp_size`, `speedup_ratio`, and `decode_speedup_
used only for trace-file runs. Unspecified fields fall back to the same defaults used by
`MockEngineArgs::default()` and `KvRouterConfig::default()`.

`--router-policy-config` sets the startup-only policy YAML path and overrides a
`router_policy_config` value embedded in `--router-config`. Requests may provide an optional
`policy_class` field in Mooncake JSONL rows. Missing and unknown classes use the selected
profile's `default_policy_class`; Dynamo does not infer a class from cache state. Use
`--model-name` to select an exact model profile, otherwise the YAML root profile is used.

DynoSim has two independent AIC surfaces:

- engine timing AIC via `--extra-engine-args` / staged engine JSON
Expand Down
7 changes: 5 additions & 2 deletions docs/observability/metrics.md
Original file line number Diff line number Diff line change
Expand Up @@ -401,13 +401,16 @@ Histograms (in milliseconds) tracking the time spent in each phase of the routin

#### Router Queue Metrics (`dynamo_frontend_router_queue_*`)

Gauge tracking the number of requests pending in the router's scheduler queue. Only registered when `--router-queue-threshold` is set. Labeled by `worker_type` to distinguish prefill vs. decode queues in disaggregated mode.
Gauges track pending work in each router policy class. They are registered by the frontend and are populated when queueing is enabled through either `--router-queue-threshold` or `--router-policy-config`.

| Metric | Type | Description |
|--------|------|-------------|
| `dynamo_frontend_router_queue_pending_requests` | Gauge | Requests pending in the router scheduler queue |
| `dynamo_frontend_router_queue_pending_isl_tokens` | Gauge | Raw input tokens pending in the router scheduler queue |
| `dynamo_frontend_router_queue_pending_cached_tokens` | Gauge | Cached-token estimate snapshotted when each request is enqueued |
| `dynamo_frontend_router_queue_backpressure_total` | Counter | Queue rejections by configured limit reason |

**Labels:** `worker_type` (`prefill` or `decode`)
**Labels:** `model`, `worker_type` (`prefill` or `decode`), and `policy_class`. The rejection counter also has `reason`.

#### KV Indexer Metrics

Expand Down
Loading
Loading