Skip to content
Merged
Show file tree
Hide file tree
Changes from 12 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
96d1ce5
feat(router): add opt-in session-aware prefix index
glamr-agent Aug 18, 2026
afbcbd5
fix(kv-router): bound session prefix index and reject cyclic grafts
glamr-agent Aug 18, 2026
1710eff
style(router): trim session index comments
MatejKosec Aug 20, 2026
6100b63
feat(router): update sessionprefixindexer lineage from G1+G2, injest …
karen-sy Aug 26, 2026
3e060bb
remove dead kvhints reference for session cleanup
karen-sy Aug 26, 2026
14816b8
Update lib/bindings/python/src/dynamo/_core.pyi
karen-sy Aug 28, 2026
f3eac1c
oops clippy
karen-sy Sep 1, 2026
1045532
more ci fix
karen-sy Sep 1, 2026
f562b09
get rid of unattributed session processing
karen-sy Sep 10, 2026
235ae05
reject remote indexer
karen-sy Sep 11, 2026
68515ed
get rid of lru bounding
karen-sy Sep 12, 2026
74a6e8e
major refactor, synchronize worker-scoped session frontiers with stor…
karen-sy Sep 14, 2026
359a051
order session lineage updates and recede removed frontiers; have matc…
karen-sy Sep 15, 2026
2dc646c
fix(router): update session frontier from final admitted route
karen-sy Sep 15, 2026
6ac8384
refactor(router): defer session lifecycle cleanup
karen-sy Sep 16, 2026
3f07fa6
docs(router): clarify session index lifecycle scope
karen-sy Sep 16, 2026
fee1e26
perf(router): optimize session prefix hot paths
karen-sy Sep 16, 2026
d60a6c9
fix(router): bound session prefix index retention
karen-sy Sep 16, 2026
b94fc2f
fix(router): reconcile session frontiers with residency changes
karen-sy Sep 16, 2026
049a8f4
harden eventual consistency guarantee (one more query if epoch mismatch)
karen-sy Sep 16, 2026
bec6273
Merge remote-tracking branch 'origin/main' into karenc/session-prefix…
karen-sy Sep 17, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
"router_queue_policy",
"use_remote_indexer",
"serve_indexer",
"enable_session_prefix_index",
"shared_cache_multiplier",
"shared_cache_type",
"conditional_disagg_enabled",
Expand Down Expand Up @@ -218,6 +219,7 @@ class KvRouterConfigBase(ConfigBase):
router_queue_policy: str
use_remote_indexer: bool = False
serve_indexer: bool = False
enable_session_prefix_index: bool = False
shared_cache_multiplier: float = 0.0
shared_cache_type: str = "none"
conditional_disagg_enabled: bool = False
Expand Down Expand Up @@ -657,6 +659,18 @@ def add_arguments(self, parser) -> None:
),
dest="use_remote_indexer",
)
add_negatable_bool_argument(
g,
flag_name="--enable-session-prefix-index",
env_var="DYN_ENABLE_SESSION_PREFIX_INDEX",
default=False,
help=(
"[EXPERIMENTAL] KV Router: Track per-session block lineage in a logical "
"prefix index that outlives engine cache eviction. Nothing consumes the "
"index for routing decisions yet."
),
dest="enable_session_prefix_index",
)
add_argument(
g,
flag_name="--shared-cache-multiplier",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,12 @@
)
from dynamo.frontend.frontend_args import FrontendArgGroup, FrontendConfig

# Skip collection when the optional pyo3 extension is unavailable.
KvRouterConfig = pytest.importorskip(
Comment thread
karen-sy marked this conversation as resolved.
Resolved
"dynamo._core",
reason="the compiled dynamo._core binding is not built in this environment",
).KvRouterConfig

pytestmark = [pytest.mark.pre_merge, pytest.mark.unit, pytest.mark.gpu_0]


Expand Down Expand Up @@ -191,6 +197,44 @@ def test_decode_active_request_weight_flows_to_binding_kwargs() -> None:
assert kwargs["decode_active_request_weight"] == 64.0


def test_session_prefix_index_is_opt_in_and_reaches_the_binding() -> None:
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
parser = argparse.ArgumentParser()
KvRouterArgGroup().add_arguments(parser)

default_kwargs = KvRouterConfigBase.from_cli_args(
parser.parse_args([])
).kv_router_kwargs()
assert default_kwargs["enable_session_prefix_index"] is False

enabled_kwargs = KvRouterConfigBase.from_cli_args(
parser.parse_args(["--enable-session-prefix-index"])
).kv_router_kwargs()
assert enabled_kwargs["enable_session_prefix_index"] is True

disabled_kwargs = KvRouterConfigBase.from_cli_args(
parser.parse_args(["--no-enable-session-prefix-index"])
).kv_router_kwargs()
assert disabled_kwargs["enable_session_prefix_index"] is False

KvRouterConfig(**enabled_kwargs)


def test_session_prefix_index_environment_flows_to_binding_kwargs(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("DYN_ENABLE_SESSION_PREFIX_INDEX", "true")
parser = argparse.ArgumentParser()
KvRouterArgGroup().add_arguments(parser)

kwargs = KvRouterConfigBase.from_cli_args(parser.parse_args([])).kv_router_kwargs()
assert kwargs["enable_session_prefix_index"] is True

overridden = KvRouterConfigBase.from_cli_args(
parser.parse_args(["--no-enable-session-prefix-index"])
).kv_router_kwargs()
assert overridden["enable_session_prefix_index"] is False


def test_load_aware_cli_applies_no_cache_load_balancing_preset() -> None:
parser = argparse.ArgumentParser()
KvRouterArgGroup().add_arguments(parser)
Expand Down
4 changes: 3 additions & 1 deletion lib/bindings/python/rust/llm/entrypoint.rs
Original file line number Diff line number Diff line change
Expand Up @@ -242,7 +242,7 @@ impl AicPerfConfig {
#[pymethods]
impl KvRouterConfig {
#[new]
#[pyo3(signature = (overlap_score_weight=None, host_cache_hit_weight=0.75, disk_cache_hit_weight=0.25, router_temperature=0.0, use_kv_events=true, *, router_replica_sync=false, router_track_active_blocks=true, router_track_output_blocks=false, router_assume_kv_reuse=true, router_track_prefill_tokens=true, router_prefill_load_model="none", router_ttl_secs=120.0, router_approximate_cache_policy="ttl", router_queue_threshold=None, router_event_threads=4, router_queue_policy="fcfs", use_remote_indexer=false, serve_indexer=false, shared_cache_multiplier=0.0, shared_cache_type="none", router_predicted_ttl_secs=None, conditional_disagg_enabled=false, conditional_disagg_policy="isl_bounding", conditional_disagg_eff_isl_threshold=2048, conditional_disagg_eff_isl_ratio_threshold=0.7, conditional_disagg_prefill_busy_threshold=None, conditional_disagg_decode_busy_threshold=None, overlap_score_credit=1.0, overlap_score_credit_decay=0.0, prefill_load_scale=1.0, decode_active_request_weight=0.0, router_policy_config=None, router_prefill_policy=None, router_decode_policy=None, router_tracking_hash="public-xxh3-v1", router_tracking_key_file=None, router_tracking_key_id=None))]
#[pyo3(signature = (overlap_score_weight=None, host_cache_hit_weight=0.75, disk_cache_hit_weight=0.25, router_temperature=0.0, use_kv_events=true, *, router_replica_sync=false, router_track_active_blocks=true, router_track_output_blocks=false, router_assume_kv_reuse=true, router_track_prefill_tokens=true, router_prefill_load_model="none", router_ttl_secs=120.0, router_approximate_cache_policy="ttl", router_queue_threshold=None, router_event_threads=4, router_queue_policy="fcfs", use_remote_indexer=false, serve_indexer=false, enable_session_prefix_index=false, shared_cache_multiplier=0.0, shared_cache_type="none", router_predicted_ttl_secs=None, conditional_disagg_enabled=false, conditional_disagg_policy="isl_bounding", conditional_disagg_eff_isl_threshold=2048, conditional_disagg_eff_isl_ratio_threshold=0.7, conditional_disagg_prefill_busy_threshold=None, conditional_disagg_decode_busy_threshold=None, overlap_score_credit=1.0, overlap_score_credit_decay=0.0, prefill_load_scale=1.0, decode_active_request_weight=0.0, router_policy_config=None, router_prefill_policy=None, router_decode_policy=None, router_tracking_hash="public-xxh3-v1", router_tracking_key_file=None, router_tracking_key_id=None))]
#[allow(clippy::too_many_arguments)]
fn new(
overlap_score_weight: Option<f64>,
Expand All @@ -263,6 +263,7 @@ impl KvRouterConfig {
router_queue_policy: &str,
use_remote_indexer: bool,
serve_indexer: bool,
enable_session_prefix_index: bool,
shared_cache_multiplier: f64,
shared_cache_type: &str,
router_predicted_ttl_secs: Option<f64>,
Expand Down Expand Up @@ -328,6 +329,7 @@ impl KvRouterConfig {
router_queue_policy: router_queue_policy.parse().map_err(PyValueError::new_err)?,
use_remote_indexer,
serve_indexer,
enable_session_prefix_index,
shared_cache_multiplier,
shared_cache_type: shared_cache_type.parse().map_err(PyValueError::new_err)?,
conditional_disagg_enabled,
Expand Down
5 changes: 5 additions & 0 deletions lib/bindings/python/src/dynamo/_core.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -1796,6 +1796,7 @@ class KvRouterConfig:
router_queue_policy: str = "fcfs",
use_remote_indexer: bool = False,
serve_indexer: bool = False,
enable_session_prefix_index: bool = False,
shared_cache_multiplier: float = 0.0,
shared_cache_type: str = "none",
router_predicted_ttl_secs: Optional[float] = None,
Expand Down Expand Up @@ -1870,6 +1871,10 @@ class KvRouterConfig:
"wspt": weighted shortest processing time (Smith's rule) — optimizes average TTFT.
use_remote_indexer: Query a remote KV indexer served from the worker component (default: False).
serve_indexer: Serve this router's local indexer from the worker component (default: False).
enable_session_prefix_index: Track per-session block lineage in a logical prefix index that outlives engine cache eviction (default: False).
Lineage is fed from two sources: routing-lookup matches, and stored-block KV events that carry a session ID. Stored blocks without a session ID do not update lineage.
The index neither holds nor restores KV cache, so a match is a routing hint rather than a guarantee that the blocks are still resident.
Retention is bounded: a session is released when a request marks it final, and failing that the least recently used session is evicted once the tracked-session cap is reached.
shared_cache_multiplier: Credit multiplier for shared cache hits beyond the device prefix (default: 0.0).
shared_cache_type: External shared KV cache type, "none" or "hicache" (default: "none").
conditional_disagg_enabled: Enable conditional-disagg bypass from prefill to decode (default: False).
Expand Down
3 changes: 2 additions & 1 deletion lib/kv-router/src/indexer/thread_pool.rs
Original file line number Diff line number Diff line change
Expand Up @@ -749,7 +749,8 @@ impl<T: SyncIndexer> ThreadPoolIndexer<T> {
/// NOTE: Rank-to-queue assignment is stable for the indexer's lifetime. Replacement reset
/// depends on removal and acknowledgement using that same FIFO lane before activation. This
/// proves queue progress only; ordinary event errors are logged by the worker.
async fn flush_worker_lane_and_wait(
#[doc(hidden)]
pub async fn flush_worker_lane_and_wait(
&self,
worker: WorkerWithDpRank,
) -> Result<(), KvRouterError> {
Expand Down
4 changes: 4 additions & 0 deletions lib/kv-router/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ pub mod recovery;
pub mod scheduling;
pub mod sequences;
pub mod services;
pub mod session_prefix_index;
pub mod tracking_hash;
pub mod worker_type;
pub mod zmq_wire;
Expand Down Expand Up @@ -80,6 +81,9 @@ pub use selector::{
WorkerInputView, WorkerInputs, WorkerLoadInput, WorkerPicker, WorkerScorer,
WorkerSelectionContext, WorkerSelectionInput, WorkerSelectionPolicy, WorkerSelector,
};
pub use session_prefix_index::{
LogicalNode, NodeId, SessionId, SessionPrefixIndexError, SessionPrefixIndexer,
};
pub use tracking_hash::{TrackingHashAlgorithm, TrackingHashContext, TrackingHashScope};
pub use worker_type::WorkerType;

Expand Down
10 changes: 0 additions & 10 deletions lib/kv-router/src/protocols.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1466,9 +1466,6 @@ pub enum KvCacheEventError {
UnsupportedResidencyDomain,
}

/// Reserved session key for events that predate session attribution.
pub const UNATTRIBUTED_SESSION_ID: &str = "__dynamo_unattributed__";

/// A [`KvCacheEvent`] on a specific LLM worker denoted by [`WorkerId`].
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct RouterEvent {
Expand Down Expand Up @@ -1559,13 +1556,6 @@ impl RouterEvent {
self
}

/// Return the reported session or the shared fallback for unattributed events.
pub fn session_id_or_unattributed(&self) -> &str {
self.session_id
.as_deref()
.unwrap_or(UNATTRIBUTED_SESSION_ID)
}

/// Resolve a storage mutation to its canonical logical domain.
pub fn resolved_residency_domain(&self) -> Result<ResidencyDomain, UnsupportedResidencyDomain> {
let domain = self.residency_domain.parse()?.unwrap_or_default();
Expand Down
36 changes: 36 additions & 0 deletions lib/kv-router/src/scheduling/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -682,6 +682,8 @@ struct KvRouterConfigSerde {
router_queue_policy: RouterQueuePolicy,
use_remote_indexer: bool,
serve_indexer: bool,
#[serde(default)]
enable_session_prefix_index: bool,
shared_cache_multiplier: f64,
shared_cache_type: SharedCacheType,
router_predicted_ttl_secs: Option<f64>,
Expand Down Expand Up @@ -726,6 +728,7 @@ impl Default for KvRouterConfigSerde {
router_queue_policy: config.router_queue_policy,
use_remote_indexer: config.use_remote_indexer,
serve_indexer: config.serve_indexer,
enable_session_prefix_index: config.enable_session_prefix_index,
shared_cache_multiplier: config.shared_cache_multiplier,
shared_cache_type: config.shared_cache_type,
router_predicted_ttl_secs: config.router_predicted_ttl_secs,
Expand Down Expand Up @@ -874,6 +877,10 @@ pub struct KvRouterConfig {
#[serde(default)]
pub serve_indexer: bool,

/// Enable bounded per-session logical prefix tracking.
#[serde(default, skip_serializing_if = "is_default")]
pub enable_session_prefix_index: bool,
Comment thread
karen-sy marked this conversation as resolved.
Resolved

/// Multiplier for shared cache hits when scoring workers (0.0 to 1.0).
/// Blocks available in the shared cache are less valuable than device-local blocks
/// because they need to be fetched. A value of 0.5 means each shared cache hit
Expand Down Expand Up @@ -976,6 +983,7 @@ impl Default for KvRouterConfig {
router_queue_policy: RouterQueuePolicy::default(),
use_remote_indexer: false,
serve_indexer: false,
enable_session_prefix_index: false,
shared_cache_multiplier: 0.0,
shared_cache_type: SharedCacheType::default(),
router_predicted_ttl_secs: None,
Expand Down Expand Up @@ -1040,6 +1048,7 @@ impl TryFrom<KvRouterConfigSerde> for KvRouterConfig {
router_queue_policy: compat.router_queue_policy,
use_remote_indexer: compat.use_remote_indexer,
serve_indexer: compat.serve_indexer,
enable_session_prefix_index: compat.enable_session_prefix_index,
shared_cache_multiplier: compat.shared_cache_multiplier,
shared_cache_type: compat.shared_cache_type,
router_predicted_ttl_secs: compat.router_predicted_ttl_secs,
Expand Down Expand Up @@ -1077,6 +1086,11 @@ fn validate_kv_router_config(config: &KvRouterConfig) -> Result<(), String> {
if config.use_remote_indexer && config.serve_indexer {
return Err("use_remote_indexer and serve_indexer are mutually exclusive".to_string());
}
if config.use_remote_indexer && config.enable_session_prefix_index {
Comment thread
karen-sy marked this conversation as resolved.
Resolved
return Err(
"enable_session_prefix_index is not supported with use_remote_indexer=true".to_string(),
);
}
if config.serve_indexer && config.overlap_score_credit == 0.0 {
return Err("serve_indexer requires overlap_score_credit > 0".to_string());
}
Expand Down Expand Up @@ -2204,6 +2218,27 @@ worker_selection:
}
}

#[test]
fn session_prefix_index_flag_defaults_off_and_survives_the_wire() {
assert!(!KvRouterConfig::default().enable_session_prefix_index);

let from_legacy: KvRouterConfig = serde_json::from_value(serde_json::json!({})).unwrap();
assert!(!from_legacy.enable_session_prefix_index);

let enabled = KvRouterConfig {
enable_session_prefix_index: true,
..Default::default()
};
let encoded = serde_json::to_value(&enabled).unwrap();
assert_eq!(
encoded.get("enable_session_prefix_index"),
Some(&serde_json::json!(true)),
"an explicitly enabled flag must be carried on the wire"
);
let decoded: KvRouterConfig = serde_json::from_value(encoded).unwrap();
assert!(decoded.enable_session_prefix_index);
}

#[test]
fn kv_router_config_preserves_v1_3_wire_compatibility() {
let _: KvRouterConfig = serde_json::from_value(serde_json::json!({
Expand All @@ -2225,6 +2260,7 @@ worker_selection:
"conditional_disagg_eff_isl_ratio_threshold",
"conditional_disagg_prefill_busy_threshold",
"conditional_disagg_decode_busy_threshold",
"enable_session_prefix_index",
] {
assert!(value.get(post_v1_3_field).is_none(), "{post_v1_3_field}");
}
Expand Down
Loading
Loading