Skip to content
4 changes: 2 additions & 2 deletions docs/fern/components/router/router-offloading.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ Legend: ✅ tier-aware routing · 🟡 router-visible, tier-agnostic · 🚧 Dyn

| Framework | Version gates | GPU | CPU RAM | Disk | Shared pool |
| --- | --- | --- | --- | --- | --- |
| [**vLLM**](#vllm) | vLLM v0.24.0+; Dynamo v1.3.0+ | ✅ KV events | ✅ `OffloadingConnector` + self-describing KV events (aggregated) | 🚧 vLLM main emits FS/OBJ events; Dynamo tier mapping is in progress | 🚧 vLLM locality events are merged; Dynamo shared-pool indexing is in progress |
| [**vLLM**](#vllm) | vLLM v0.24.0+; Dynamo v1.3.0+ | ✅ KV events | ✅ `OffloadingConnector` + self-describing KV events (aggregated) | ✅ `TieringOffloadingSpec` + self-describing STORAGE events (aggregated) | 🚧 vLLM locality events are merged; Dynamo shared-pool indexing is in progress |
| [**SGLang**](#sglang) | SGLang v0.5.11+; v0.5.13+ with Mooncake; Dynamo v1.2+ | ✅ KV events | ✅ HiCache + KV events | — no separate disk tier; HiCache's third tier is the shared pool (next column) | ✅ HiCache + Mooncake + `--shared-cache-type hicache` |
| [**TensorRT-LLM**](#tensorrt-llm) | Dynamo v1.3.0+ for the current event flag | 🟡 `--publish-kv-events`; merged GPU + RAM view | 🟡 native host cache shares one router view with GPU; per-tier weights do not apply | — no native disk tier | — |

Expand Down Expand Up @@ -61,7 +61,7 @@ PYTHONHASHSEED=0 python3 -m dynamo.vllm \
```

- Versions: vLLM v0.24.0 or later. Earlier versions publish placeholder CPU events that the router silently drops — offloading still works engine-side, but the router only sees the GPU tier.
- Disk and multi-tier offloading (`TieringOffloadingSpec`): vLLM main emits FS and OBJ events. Dynamo tier mapping is in progress.
- Disk and multi-tier offloading (`TieringOffloadingSpec`): vLLM emits a unified `STORAGE` medium (vLLM 0.27+; #48123) that Dynamo maps to its Disk lower tier. Legacy `FS` / `OBJ` media (vLLM 0.26.0) are unrecognized and dropped (fail closed), not misrouted.
- Shared pools: vLLM main publishes optional `LOCAL` / `REMOTE` locality metadata on FS and OBJ events. Dynamo shared-pool indexing is in progress.

See [Native KV Offloading](../../backends/vllm/vllm-native-kv-offloading.md) for the full support matrix (including disaggregated and tensor-parallel status), setup commands, verification, and troubleshooting.
Expand Down
6 changes: 0 additions & 6 deletions lib/kv-router/src/protocols.rs
Original file line number Diff line number Diff line change
Expand Up @@ -446,12 +446,6 @@ impl StorageTier {
}
}

pub fn from_kv_medium_or_default(medium: Option<&str>) -> Self {
medium
.and_then(Self::from_kv_medium)
.unwrap_or(Self::Device)
}

/// Canonical wire-format medium string. `None` for the default GPU tier so
/// existing consumers that omit the field continue to round-trip.
pub fn to_kv_medium(self) -> Option<&'static str> {
Expand Down
19 changes: 18 additions & 1 deletion lib/kv-router/src/zmq_wire/convert.rs
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,24 @@ pub fn convert_event(
return None;
}

let storage_tier = StorageTier::from_kv_medium_or_default(medium);
// Fail closed on unrecognized media instead of silently indexing them on
// the device (G1) primary tree. vLLM 0.26.0 ships `FS` / `OBJ` (pre-#48123
// wire); those and any future medium strings are dropped, not misfiled. The
// normalizer's preprocess step classifies these as filtered first (so no
// event id is burned); this guard is a defensive backstop for direct
// convert_event callers that bypass preprocess, mirroring the locality gate.
let storage_tier = match medium {
None => StorageTier::Device,
Some(medium) => match StorageTier::from_kv_medium(medium) {
Some(tier) => tier,
None => {
if warning_count.fetch_add(1, Ordering::Relaxed) < 3 {
tracing::warn!(event_id, medium, "Dropping KV event with unknown medium");
}
return None;
}
},
};

let dp_rank = worker.dp_rank;
let event = match raw {
Expand Down
33 changes: 21 additions & 12 deletions lib/kv-router/src/zmq_wire/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ struct KvCacheGroupMetadata {
pub enum ZmqEventFilterReason {
IgnoredEvent,
NonLocalLocality,
UnknownMedium,
Comment thread
Change72 marked this conversation as resolved.
Resolved
AmbiguousCacheNamespace,
NonMainAttentionKind,
UnknownKind,
Expand All @@ -78,6 +79,7 @@ impl ZmqEventFilterReason {
match self {
Self::IgnoredEvent => "ignored_event",
Self::NonLocalLocality => "non_local_locality",
Self::UnknownMedium => "unknown_medium",
Self::AmbiguousCacheNamespace => "ambiguous_cache_namespace",
Self::NonMainAttentionKind => "non_main_attention_kind",
Self::UnknownKind => "unknown_kind",
Expand Down Expand Up @@ -139,18 +141,25 @@ impl ZmqEventNormalizer {
return Err(ZmqEventFilterReason::NonLocalLocality);
}

// Hash-only lower-tier events (STORAGE -> Disk, and External) carry no
// extra_keys/cache_namespace and must not mutate per-group metadata or
// the salted-namespace propagation chain; they are also outside the
// SW/SSM group filter's semantics, so route them straight to conversion.
// Device and host-pinned events (CPU offload, #10368) stay on the
// normalizer path so their salted namespaces still propagate.
if raw
.medium()
.and_then(StorageTier::from_kv_medium)
.is_some_and(|tier| matches!(tier, StorageTier::Disk | StorageTier::External))
{
return Ok(raw);
// Classify by medium before touching normalizer state:
// - Device / HostPinned (GPU, CPU offload #10368) stay on the normalizer
// path so their salted namespaces still propagate.
// - Disk / External (STORAGE) are hash-only lower-tier events with no
// extra_keys/cache_namespace, so they must not mutate per-group
// metadata or the salted-namespace chain and are outside the SW/SSM
// group filter's semantics; bypass straight to conversion, which keeps
// them (no event id is wasted).
// - Unrecognized media (e.g. vLLM 0.26.0 FS/OBJ) fail closed here so the
// listener records an intentional filter. Bypassing to conversion,
// which drops them, would instead accept the event, burn a
// next_event_id, and leave an id gap the event processor mistakes for
// an engine drop -- the same trap the locality gate above avoids.
if let Some(m) = raw.medium() {
match StorageTier::from_kv_medium(m) {
Some(StorageTier::Device | StorageTier::HostPinned) => {}
Some(_) => return Ok(raw),
None => return Err(ZmqEventFilterReason::UnknownMedium),
}
Comment thread
Change72 marked this conversation as resolved.
Resolved
}

let metadata = raw.metadata();
Expand Down
95 changes: 95 additions & 0 deletions lib/kv-router/src/zmq_wire/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1103,6 +1103,44 @@ fn test_preprocess_rejects_non_local_locality_for_every_tier() {
}
}

/// Unrecognized media must be classified as filtered in `preprocess_with_reason`
/// (reason `UnknownMedium`), so the listener records an intentional filter rather
/// than accepting the event, burning a next_event_id, and dropping it only in
/// conversion — an id gap the event processor misreads as an engine drop. This is
/// the same trap the locality gate avoids. Recognized tiers are never filtered
/// here: Device/HostPinned stay on the normalizer path and Disk/External bypass.
#[test]
fn test_preprocess_rejects_unknown_medium() {
let worker = WorkerWithDpRank::new(3, 0);

for event_kind in [TestEventKind::BlockStored, TestEventKind::BlockRemoved] {
for medium in ["FS", "OBJ", "XYZ"] {
let mut normalizer = ZmqEventNormalizer::new(2);
let raw = raw_placement_event(event_kind, Some(medium), Some(Locality::Local));
assert_eq!(
normalizer.preprocess_with_reason(raw, worker).unwrap_err(),
ZmqEventFilterReason::UnknownMedium,
"medium={medium} must filter as unknown_medium"
);
}
}

// Recognized media are never rejected as unknown: GPU/CPU stay on the
// normalizer path and STORAGE takes the lower-tier bypass.
for medium in ["GPU", "CPU", "STORAGE"] {
let mut normalizer = ZmqEventNormalizer::new(2);
let raw = raw_placement_event(
TestEventKind::BlockStored,
Some(medium),
Some(Locality::Local),
);
assert!(
normalizer.preprocess_with_reason(raw, worker).is_ok(),
"recognized medium={medium} must pass preprocess"
);
}
}

/// Mixed-case locality (`"local"`, `"Remote"`) is not UPPERCASE, so it folds to
/// `Unknown` and the event is dropped — it never decodes to a local placement.
#[test]
Expand Down Expand Up @@ -1230,3 +1268,60 @@ fn test_lower_tier_events_do_not_pollute_cache_namespace_state() {
};
assert_eq!(cache_namespace.as_deref(), Some("tenant-a"));
}

/// Unrecognized media (e.g. vLLM 0.26.0 `FS` / `OBJ`, or any future string) must
/// be dropped in conversion, never silently indexed on the device (G1) primary
/// tree — for both store and remove, and regardless of locality.
#[test]
fn test_convert_event_rejects_unknown_medium_instead_of_defaulting_to_device() {
let worker = WorkerWithDpRank::new(3, 0);
for event_kind in [TestEventKind::BlockStored, TestEventKind::BlockRemoved] {
for medium in ["FS", "OBJ", "XYZ"] {
for locality in [None, Some(Locality::Local)] {
let raw = raw_placement_event(event_kind, Some(medium), locality);
assert!(
convert_placement(raw, worker).is_none(),
"unrecognized medium {medium} must be dropped (locality {locality:?})"
);
}
}
}
}

/// Unrecognized media (vLLM 0.26.0 `FS` / `OBJ`) fail closed in preprocess, so
/// they never reach — let alone mutate — the salted-namespace state: a salted GPU
/// hash stays `Namespaced` and later salted GPU children keep inheriting it.
#[test]
fn test_unrecognized_media_do_not_pollute_cache_namespace_state() {
let worker = WorkerWithDpRank::new(7, 0);
let mut normalizer = ZmqEventNormalizer::new(2);

let salted_gpu = namespaced_block_stored(1, None, Some("tenant-a"), "GPU");
assert!(normalizer.preprocess(salted_gpu, worker).is_some());

// An FS event reusing the same hash fails closed before any namespace work.
let fs_same_hash = namespaced_block_stored(1, None, None, "FS");
assert_eq!(
normalizer
.preprocess_with_reason(fs_same_hash, worker)
.unwrap_err(),
ZmqEventFilterReason::UnknownMedium
);
assert!(matches!(
&normalizer.cache_namespaces[&(worker, 1)],
CacheNamespaceState::Namespaced(ns) if ns.as_ref() == "tenant-a"
));

// The salted chain continuation still inherits the parent namespace.
let gpu_child = namespaced_block_stored(2, Some(1), None, "GPU");
let child = normalizer
.preprocess(gpu_child, worker)
.expect("unrecognized-media event must not make the parent ambiguous");
let RawKvEvent::BlockStored {
cache_namespace, ..
} = child
else {
panic!("expected BlockStored");
};
assert_eq!(cache_namespace.as_deref(), Some("tenant-a"));
}
19 changes: 12 additions & 7 deletions lib/kvbm-consolidator/src/ingress/zmq_subscriber.rs
Original file line number Diff line number Diff line change
Expand Up @@ -92,14 +92,14 @@ pub async fn spawn(

fn process_event(tracker: &mut Tracker, event: RawKvEvent, engine_source: EventSource) {
// G1-only ingress: this source is the engine's local device (G1) cache.
// Non-local events (REMOTE / unknown locality) and native lower-tier media
// (CPU offload, vLLM STORAGE) belong to other systems — KVBM offload arrives
// via its own source — so they must not be tracked as G1 here.
// Non-local events (REMOTE / unknown locality), native lower-tier media
// (CPU offload, vLLM STORAGE), and unrecognized media (fail-closed) belong
// to other systems — KVBM offload arrives via its own source — so they must
// not be tracked as G1 here.
if matches!(event.locality(), Some(Locality::Remote | Locality::Unknown))
|| event
.medium()
.and_then(StorageTier::from_kv_medium)
.is_some_and(|tier| !tier.is_gpu())
.is_some_and(|m| StorageTier::from_kv_medium(m) != Some(StorageTier::Device))
{
return;
}
Expand Down Expand Up @@ -223,8 +223,8 @@ mod tests {
}

/// G1-only ingress contract: only local device (G1) events reach the
/// tracker. Native lower-tier media (vLLM STORAGE, CPU offload) and non-local
/// (REMOTE / unknown locality) events are dropped.
/// tracker. Native lower-tier media (vLLM STORAGE, CPU offload), unrecognized
/// media (FS), and non-local (REMOTE / unknown locality) events are dropped.
#[test]
fn process_event_tracks_only_g1_device_events() {
let mut tracker = Tracker::new(None);
Expand All @@ -239,6 +239,11 @@ mod tests {
stored_event(Some("CPU"), None),
EventSource::Vllm,
);
process_event(
&mut tracker,
stored_event(Some("FS"), None),
EventSource::Vllm,
);
process_event(
&mut tracker,
stored_event(Some("GPU"), Some(Locality::Remote)),
Expand Down
24 changes: 15 additions & 9 deletions lib/llm/src/block_manager/kv_consolidator/subscriber.rs
Original file line number Diff line number Diff line change
Expand Up @@ -154,14 +154,14 @@ fn process_event(
engine_source: EventSource,
) {
// G1-only ingress: this source is the engine's local device (G1) cache.
// Non-local events (REMOTE / unknown locality) and native lower-tier media
// (CPU offload, vLLM STORAGE) belong to other systems — KVBM offload arrives
// via its own source — so they must not be tracked as G1 here.
// Non-local events (REMOTE / unknown locality), native lower-tier media
// (CPU offload, vLLM STORAGE), and unrecognized media (fail-closed) belong
// to other systems — KVBM offload arrives via its own source — so they must
// not be tracked as G1 here.
if matches!(event.locality(), Some(Locality::Remote | Locality::Unknown))
|| event
.medium()
.and_then(RouterStorageTier::from_kv_medium)
.is_some_and(|tier| !tier.is_gpu())
|| event.medium().is_some_and(|m| {
RouterStorageTier::from_kv_medium(m) != Some(RouterStorageTier::Device)
})
{
return;
}
Expand Down Expand Up @@ -294,8 +294,8 @@ mod tests {
}

/// G1-only ingress contract: only local device (G1) events reach the
/// tracker. Native lower-tier media (vLLM STORAGE, CPU offload) and non-local
/// (REMOTE / unknown locality) events are dropped.
/// tracker. Native lower-tier media (vLLM STORAGE, CPU offload), unrecognized
/// media (FS), and non-local (REMOTE / unknown locality) events are dropped.
#[test]
fn process_event_tracks_only_g1_device_events() {
let mut tracker = PassthroughCacheStatusTracker::new();
Expand All @@ -312,6 +312,12 @@ mod tests {
None,
EventSource::Vllm,
);
process_event(
&mut tracker,
stored_event(Some("FS"), None),
None,
EventSource::Vllm,
);
process_event(
&mut tracker,
stored_event(Some("GPU"), Some(Locality::Remote)),
Expand Down
Loading
Loading