Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
c796c28
add taints
michaelfeil May 14, 2026
5cd49a9
Merge remote-tracking branch 'upstream/main' into mf/taints
michaelfeil May 14, 2026
bcb124c
add taints
michaelfeil May 14, 2026
9bc8f64
add schema
michaelfeil May 14, 2026
c164501
make taints simplified
michaelfeil May 14, 2026
70c5c71
do not, commit the codexx file ...
michaelfeil May 14, 2026
304a5b6
fix the tests
michaelfeil May 14, 2026
848070a
fix the fmt
michaelfeil May 14, 2026
044328e
Merge branch 'main' into mf/taints
michaelfeil May 14, 2026
6e934bf
fix the fmt
michaelfeil May 14, 2026
bd2a681
fix the fmt
michaelfeil May 14, 2026
7d25ea9
adding a single good regression test and the complaint about the c bi…
michaelfeil May 14, 2026
7f731f8
added taints to queue, and push-router
michaelfeil May 14, 2026
da3bce6
added tests to make sure the queue is passing
michaelfeil May 14, 2026
dd566cc
rename from Taints to RoutingContraints
michaelfeil May 14, 2026
7c3911a
rename from Taints to RoutingContraints
michaelfeil May 14, 2026
04e78c8
rename required to required_taints
michaelfeil May 14, 2026
28e853f
add routing contraints to RouterRequest which is the whole reason we …
michaelfeil May 14, 2026
179203b
taints are now hashsets
michaelfeil May 14, 2026
5a4050d
python api should be hashset
michaelfeil May 14, 2026
250562b
python api should be hashset -> now also in core.pyi
michaelfeil May 14, 2026
bffdef8
pinned worker + required taints -> error instead of warning
michaelfeil May 14, 2026
ccd5d3b
pinned worker + required taints -> error instead of warning FMT
michaelfeil May 14, 2026
93f702e
add a prefered taint bias
michaelfeil May 14, 2026
3414556
add a prefered taint bias as hashmap, with individual scores
michaelfeil May 15, 2026
5c122a9
address comments
michaelfeil May 15, 2026
8b7487f
address comments
michaelfeil May 15, 2026
bf2571b
verify routing constraints with selector
michaelfeil May 15, 2026
7469e89
add a comment
michaelfeil May 15, 2026
32b141c
fmt
michaelfeil May 15, 2026
65596fe
add improved heuristics support around preferred routing targets
michaelfeil May 15, 2026
ce236a3
add eligibility helper to bundle checks with usually inline
michaelfeil May 15, 2026
d45ead4
add re-export and test utils import fix
michaelfeil May 15, 2026
0b263d5
Merge branch 'main' into mf/taints
michaelfeil May 15, 2026
b471aff
make api more clear to only multiply if useful
michaelfeil May 15, 2026
1922ae4
make api more clear to only multiply if useful
michaelfeil May 15, 2026
f802890
ove pinned worker in the api, scheduled context helper to make sure q…
michaelfeil May 15, 2026
64ccf6d
add comment why heuristic is choosen
michaelfeil May 15, 2026
6aca329
add comment why heuristic is choosen
michaelfeil May 15, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion lib/bindings/python/rust/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ use dynamo_llm::{self as llm_rs};

use crate::llm::entrypoint::RouterConfig as PyRouterConfig;

use crate::llm::local_model::ModelRuntimeConfig;
use crate::llm::local_model::{ModelRuntimeConfig, Taints};
use crate::llm::preprocessor::{MediaDecoder, MediaFetcher};

#[pyclass(eq, eq_int)]
Expand Down Expand Up @@ -183,6 +183,7 @@ fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<llm::kv::WorkerMetricsPublisher>()?;
m.add_class::<llm::model_card::ModelDeploymentCard>()?; // Internal: only in _internal, not public API
m.add_class::<llm::local_model::ModelRuntimeConfig>()?;
m.add_class::<Taints>()?;
m.add_class::<llm::preprocessor::MediaDecoder>()?;
m.add_class::<llm::preprocessor::MediaFetcher>()?;
m.add_class::<llm::kv::OverlapScores>()?;
Expand Down
11 changes: 8 additions & 3 deletions lib/bindings/python/rust/llm/kv.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ use std::sync::mpsc;
use tokio_stream::StreamExt;

use super::*;
use super::local_model::Taints;
use crate::Endpoint;
#[cfg(feature = "kv-indexer")]
use clap::Parser;
Expand Down Expand Up @@ -936,7 +937,7 @@ impl KvRouter {
}

#[allow(clippy::too_many_arguments)]
#[pyo3(signature = (token_ids, model, stop_conditions=None, sampling_options=None, output_options=None, router_config_override=None, worker_id=None, dp_rank=None, extra_args=None, block_mm_infos=None, multi_modal_data=None, mm_routing_info=None))]
#[pyo3(signature = (token_ids, model, stop_conditions=None, sampling_options=None, output_options=None, router_config_override=None, worker_id=None, dp_rank=None, extra_args=None, block_mm_infos=None, multi_modal_data=None, mm_routing_info=None, taints=None))]
fn generate<'p>(
&self,
py: Python<'p>,
Expand All @@ -952,6 +953,7 @@ impl KvRouter {
block_mm_infos: Option<PyObject>,
multi_modal_data: Option<PyObject>,
mm_routing_info: Option<PyObject>,
taints: Option<Taints>,
) -> PyResult<Bound<'p, PyAny>> {
// Depythonize the options with defaults
let stop_conditions: StopConditions = if let Some(obj) = stop_conditions {
Expand Down Expand Up @@ -1027,10 +1029,11 @@ impl KvRouter {
.tracker(Some(tracker.clone()));

// Set routing hints if worker_id or dp_rank is provided
if worker_id.is_some() || dp_rank.is_some() {
if worker_id.is_some() || dp_rank.is_some() || taints.is_some() {
let routing = llm_rs::protocols::common::preprocessor::RoutingHints {
backend_instance_id: worker_id,
dp_rank,
taints: taints.map(|t| t.inner),
..Default::default()
};
request_builder.routing(Some(routing));
Expand Down Expand Up @@ -1066,7 +1069,7 @@ impl KvRouter {
}

#[allow(clippy::too_many_arguments)]
#[pyo3(signature = (token_ids, router_config_override=None, request_id=None, update_indexer=false, block_mm_infos=None, lora_name=None))]
#[pyo3(signature = (token_ids, router_config_override=None, request_id=None, update_indexer=false, block_mm_infos=None, lora_name=None, taints=None))]
fn best_worker<'p>(
&self,
py: Python<'p>,
Expand All @@ -1076,6 +1079,7 @@ impl KvRouter {
update_indexer: bool,
block_mm_infos: Option<PyObject>,
lora_name: Option<String>,
taints: Option<Taints>,
) -> PyResult<Bound<'p, PyAny>> {
let router_config_override = if let Some(obj) = router_config_override {
let override_config: RouterConfigOverride =
Expand Down Expand Up @@ -1104,6 +1108,7 @@ impl KvRouter {
0.0,
None,
None, // allowed_worker_ids: pass via RoutingHints in PreprocessedRequest path
taints.map(|t| t.inner).unwrap_or_default(),
)
.await
.map_err(to_pyerr)?;
Expand Down
53 changes: 53 additions & 0 deletions lib/bindings/python/rust/llm/local_model.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,47 @@
use super::*;
use llm_rs::local_model::runtime_config::DisaggregatedEndpoint as RsDisaggregatedEndpoint;
use llm_rs::local_model::runtime_config::ModelRuntimeConfig as RsModelRuntimeConfig;
use llm_rs::protocols::Taints as RsTaints;
Comment thread
michaelfeil marked this conversation as resolved.
Resolved

#[pyclass]
#[derive(Clone, Debug, Default)]
pub struct Taints {
pub(crate) inner: RsTaints,
}

#[pymethods]
impl Taints {
#[new]
#[pyo3(signature = (required=None, preferred=None))]
fn new(required: Option<Vec<String>>, preferred: Option<Vec<String>>) -> Self {
Self {
inner: RsTaints {
required: required.unwrap_or_default(),
preferred: preferred.unwrap_or_default(),
},
}
}

#[getter]
fn required(&self) -> Vec<String> {
self.inner.required.clone()
}

#[setter]
fn set_required(&mut self, required: Vec<String>) {
self.inner.required = required;
}

#[getter]
fn preferred(&self) -> Vec<String> {
self.inner.preferred.clone()
}

#[setter]
fn set_preferred(&mut self, preferred: Vec<String>) {
self.inner.preferred = preferred;
}
}

#[pyclass]
#[derive(Clone, Debug, Default)]
Expand Down Expand Up @@ -73,6 +114,11 @@ impl ModelRuntimeConfig {
self.inner.enable_eagle = enable_eagle;
}

#[setter]
fn set_taints(&mut self, taints: &Taints) {
self.inner.taints = taints.inner.clone();
}

fn set_engine_specific(&mut self, key: &str, value: String) -> PyResult<()> {
let value: serde_json::Value = serde_json::from_str(&value).map_err(to_pyerr)?;
self.inner
Expand Down Expand Up @@ -182,4 +228,11 @@ impl ModelRuntimeConfig {
fn enable_eagle(&self) -> bool {
self.inner.enable_eagle
}

#[getter]
fn taints(&self) -> Taints {
Taints {
inner: self.inner.taints.clone(),
}
}
}
14 changes: 14 additions & 0 deletions lib/bindings/python/src/dynamo/_core.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -514,6 +514,7 @@ class ModelRuntimeConfig:
data_parallel_size: int
enable_local_indexer: bool
enable_eagle: bool
taints: Taints
runtime_data: dict[str, Any]
tensor_model_config: Any | None
bootstrap_host: str | None
Expand Down Expand Up @@ -545,6 +546,16 @@ class ModelRuntimeConfig:
"""Get the tensor model configuration."""
...

class Taints:
required: List[str]
preferred: List[str]

def __init__(
self,
required: Optional[List[str]] = None,
preferred: Optional[List[str]] = None,
) -> None: ...

class OverlapScores:
"""
A collection of prefix matching scores of workers for a given token ids.
Expand Down Expand Up @@ -1998,6 +2009,7 @@ class KvRouter:
block_mm_infos: Optional[List[Optional[Dict[str, Any]]]] = None,
multi_modal_data: Optional[JsonLike] = None,
mm_routing_info: Optional[JsonLike] = None,
taints: Optional[Taints] = None,
) -> AsyncIterator[JsonLike]:
"""
Generate text using the KV-aware router.
Expand Down Expand Up @@ -2026,6 +2038,7 @@ class KvRouter:
mm_routing_info: Optional structured routing-only multimodal payload
(e.g., {"routing_token_ids": [...], "block_mm_infos": [...]})
used by router selection without changing execution token_ids.
taints: Optional request taints used to constrain or prefer tainted workers.
Comment thread
michaelfeil marked this conversation as resolved.
Resolved

Returns:
An async iterator yielding generation responses
Expand Down Expand Up @@ -2059,6 +2072,7 @@ class KvRouter:
update_indexer: bool = False,
block_mm_infos: Optional[List[Optional[Dict[str, Any]]]] = None,
lora_name: Optional[str] = None,
taints: Optional[Taints] = None,
) -> Tuple[int, int, int]:
"""
Find the best matching worker for the given tokens.
Expand Down
1 change: 1 addition & 0 deletions lib/bindings/python/src/dynamo/llm/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
from dynamo._core import RouterConfig as RouterConfig
from dynamo._core import RouterMode as RouterMode
from dynamo._core import SglangArgs as SglangArgs
from dynamo._core import Taints as Taints
from dynamo._core import WorkerMetricsPublisher as WorkerMetricsPublisher
from dynamo._core import compute_block_hash_for_seq as compute_block_hash_for_seq
from dynamo._core import fetch_model as fetch_model
Expand Down
31 changes: 31 additions & 0 deletions lib/kv-router/src/protocols.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
use std::future::Future;
use std::ops::Range;
use std::time::Duration;
use std::sync::LazyLock;

use dynamo_tokens::{SequenceHash, Token, compute_hash_v2};
use rustc_hash::FxHashMap;
Expand Down Expand Up @@ -180,8 +181,38 @@ pub trait WorkerConfigLike {
fn data_parallel_size(&self) -> u32;
fn max_num_batched_tokens(&self) -> Option<u64>;
fn total_kv_blocks(&self) -> Option<u64>;
fn taints(&self) -> &Taints {
&EMPTY_TAINTS
}
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
pub struct Taints {
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub required: Vec<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub preferred: Vec<String>,
}

impl Taints {
pub fn is_empty(&self) -> bool {
self.required.is_empty() && self.preferred.is_empty()
}

pub fn is_compatible_with(&self, request: &Taints) -> bool {
request
.required
.iter()
.all(|taint| self.required.iter().any(|worker_taint| worker_taint == taint))
&& self
.required
.iter()
.all(|taint| request.required.iter().any(|request_taint| request_taint == taint))
}
}

static EMPTY_TAINTS: LazyLock<Taints> = LazyLock::new(Taints::default);

/// Transport abstraction for publishing batched router-visible KV cache events.
pub trait RouterEventSink: Send + Sync {
fn publish_event(&self, event: &RouterEvent)
Expand Down
3 changes: 3 additions & 0 deletions lib/kv-router/src/scheduling/local.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ use super::types::{
KvSchedulerError, PotentialLoad, SchedulingRequest, SchedulingResponse, TierOverlapBlocks,
};
use crate::protocols::{WorkerConfigLike, WorkerId, WorkerWithDpRank};
use crate::protocols::Taints;
use crate::sequences::{
ActiveSequencesMultiWorker, PrefillTokenDeltas, SequenceError, SequencePublisher,
SequenceRequest,
Expand Down Expand Up @@ -185,6 +186,7 @@ where
expected_output_tokens: Option<u32>,
pinned_worker: Option<WorkerWithDpRank>,
allowed_worker_ids: Option<HashSet<WorkerId>>,
taints: Taints,
shared_cache_hits: Option<crate::SharedCacheHits>,
Comment thread
michaelfeil marked this conversation as resolved.
Resolved
) -> Result<SchedulingResponse, KvSchedulerError> {
let (resp_tx, resp_rx) = tokio::sync::oneshot::channel();
Expand All @@ -201,6 +203,7 @@ where
decode_blocks: FxHashMap::default(),
prefill_tokens: FxHashMap::default(),
track_prefill_tokens,
taints,
router_config_override: router_config_override.cloned(),
update_states,
lora_name,
Expand Down
19 changes: 19 additions & 0 deletions lib/kv-router/src/scheduling/selector.rs
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,16 @@ impl<C: WorkerConfigLike> WorkerSelector<C> for DefaultWorkerSelector {
return Err(KvSchedulerError::NoEndpoints);
}

if pinned_worker.is_none()
&& !request.taints.is_empty()
&& workers
.iter()
.filter(|(worker_id, _)| request.is_worker_allowed(**worker_id))
.all(|(_, config)| !config.taints().is_compatible_with(&request.taints))
{
return Err(KvSchedulerError::NoEndpoints);
}

let request_blocks = request.request_blocks(block_size);

let weights = LogitWeights {
Expand All @@ -237,6 +247,12 @@ impl<C: WorkerConfigLike> WorkerSelector<C> for DefaultWorkerSelector {

if let Some(worker) = pinned_worker {
pinned_worker_config(workers, worker)?;
if workers
.get(&worker.worker_id)
.is_some_and(|config| !config.taints().is_compatible_with(&request.taints))
{
Comment thread
michaelfeil marked this conversation as resolved.
Resolved
return Err(KvSchedulerError::NoEndpoints);
}

let logit = self.worker_logit(request, worker, block_size, weights, "Pinned formula");
let effective_overlap_blocks = request.effective_overlap_blocks_for(worker);
Expand Down Expand Up @@ -272,6 +288,9 @@ impl<C: WorkerConfigLike> WorkerSelector<C> for DefaultWorkerSelector {
let worker_iter = workers
.iter()
.filter(move |(worker_id, _)| request.is_worker_allowed(**worker_id))
.filter(move |(_, config)| {
request.taints.is_empty() || config.taints().is_compatible_with(&request.taints)
})
.flat_map(|(worker_id, config)| {
let data_parallel_size = config.data_parallel_size();
let data_parallel_start_rank = config.data_parallel_start_rank();
Expand Down
3 changes: 2 additions & 1 deletion lib/kv-router/src/scheduling/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ use rustc_hash::FxHashMap;
use serde::{Deserialize, Serialize};

use super::config::RouterConfigOverride;
use crate::protocols::{DpRank, SharedCacheHits, WorkerConfigLike, WorkerId, WorkerWithDpRank};
use crate::protocols::{DpRank, SharedCacheHits, Taints, WorkerConfigLike, WorkerId, WorkerWithDpRank};
use crate::sequences::PrefillTokenDeltas;

#[derive(Debug, Clone, Default, Serialize, Deserialize)]
Expand Down Expand Up @@ -62,6 +62,7 @@ pub struct SchedulingRequest {
// Routing constraints and request-level config.
pub pinned_worker: Option<WorkerWithDpRank>,
pub allowed_worker_ids: Option<HashSet<WorkerId>>,
pub taints: Taints,
Comment thread
michaelfeil marked this conversation as resolved.
Resolved
pub router_config_override: Option<RouterConfigOverride>,
pub track_prefill_tokens: bool,
pub priority_jump: f64,
Expand Down
Loading
Loading