From 4890d4f608e8000191d6655489a85ba6a2c7d46b Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Tue, 15 Sep 2026 09:36:01 -0700 Subject: [PATCH 1/4] fix(router): reject unknown explicit worker targets Signed-off-by: jthomson04 --- lib/llm/src/http/service/openai.rs | 63 +++++ lib/llm/src/kv_router/routing_host.rs | 33 +++ lib/llm/src/kv_router/routing_host/builtin.rs | 1 + lib/llm/src/kv_router/routing_host/kv.rs | 2 + lib/llm/src/kv_router/routing_host/tests.rs | 236 ++++++++++++++++-- 5 files changed, 319 insertions(+), 16 deletions(-) diff --git a/lib/llm/src/http/service/openai.rs b/lib/llm/src/http/service/openai.rs index 50cb8cc56349..30251fd2fbae 100644 --- a/lib/llm/src/http/service/openai.rs +++ b/lib/llm/src/http/service/openai.rs @@ -6295,6 +6295,69 @@ mod tests { ); } + #[tokio::test] + async fn unknown_explicit_worker_response_is_bad_request() { + use crate::{ + kv_router::{RouterLoadSource, RoutingHost, RoutingLoadContext}, + preprocessor::PreprocessedRequest, + }; + use dynamo_runtime::{ + DistributedRuntime, Runtime, + distributed::DistributedConfig, + pipeline::{AsyncEngine, Context, PushRouter, RouterMode}, + }; + + let runtime = Runtime::from_current().unwrap(); + let distributed = + DistributedRuntime::new(runtime.clone(), DistributedConfig::process_local()) + .await + .unwrap(); + let endpoint = distributed + .namespace("unknown-worker-http-error") + .unwrap() + .component("workers") + .unwrap() + .endpoint("generate"); + let client = endpoint.client().await.unwrap(); + let load_context = RoutingLoadContext::start( + client.clone(), + RouterLoadSource::Aggregated, + crate::discovery::LoadThresholdHandle::new(Default::default()), + &distributed.child_token(), + None, + ) + .await + .unwrap(); + let inner = PushRouter::from_client(client, RouterMode::RoundRobin) + .await + .unwrap(); + let host = RoutingHost::::new_builtin( + inner, + load_context, + ) + .unwrap(); + let mut request = PreprocessedRequest::builder() + .model("test".into()) + .token_ids(vec![1]) + .stop_conditions(Default::default()) + .sampling_options(Default::default()) + .output_options(Default::default()) + .build() + .unwrap(); + request.routing_mut().decode_worker_id = Some(u64::MAX); + let error = host.generate(Context::new(request)).await.unwrap_err(); + let response = ErrorMessage::from_anyhow(error, BACKUP_ERROR_MESSAGE); + assert_eq!(response.0, StatusCode::BAD_REQUEST); + assert_eq!(response.1.code, 400); + assert_eq!(response.1.error_type, "Bad Request"); + assert_eq!( + response.1.message, + "nvext.decode_worker_id does not identify a known worker" + ); + drop(host); + runtime.shutdown(); + } + #[test] fn test_nested_invalid_argument_response_from_anyhow() { use dynamo_runtime::error::{DynamoError, ErrorType}; diff --git a/lib/llm/src/kv_router/routing_host.rs b/lib/llm/src/kv_router/routing_host.rs index 34c458faf3fc..fad76bb87002 100644 --- a/lib/llm/src/kv_router/routing_host.rs +++ b/lib/llm/src/kv_router/routing_host.rs @@ -576,6 +576,39 @@ where } } + /// Check request-supplied targets only at initial selection, not after a route preview + /// or when dispatching: a worker disappearing later is a service failure. + fn validate_explicit_worker( + &self, + request: &PreprocessedRequest, + phase: RequestPhase, + ) -> Result<(), Error> { + let Some(routing) = request.routing.as_ref() else { + return Ok(()); + }; + let target = match phase { + RequestPhase::Prefill => routing + .prefill_worker_id + .map(|id| (id, "prefill_worker_id")), + RequestPhase::Decode | RequestPhase::Aggregated => { + routing.decode_worker_id.map(|id| (id, "decode_worker_id")) + } + } + .or_else(|| { + routing + .backend_instance_id + .map(|id| (id, "backend_instance_id")) + }); + if let Some((worker_id, field)) = target + && !self.inner.client.is_instance_live(worker_id) + { + return Err(invalid_argument(format!( + "nvext.{field} does not identify a known worker" + ))); + } + Ok(()) + } + fn affinity_target_is_valid(&self, target: AffinityTarget) -> bool { if !self.inner.client.is_instance_discovered(target.worker_id) { return false; diff --git a/lib/llm/src/kv_router/routing_host/builtin.rs b/lib/llm/src/kv_router/routing_host/builtin.rs index 28b511f93c1c..25295e353160 100644 --- a/lib/llm/src/kv_router/routing_host/builtin.rs +++ b/lib/llm/src/kv_router/routing_host/builtin.rs @@ -288,6 +288,7 @@ where { let phase_label = phase.to_string(); let route_guard = StageGuard::new(STAGE_ROUTE, &phase_label); + self.validate_explicit_worker(request.content(), phase)?; let explicit = explicit_target(request.content(), phase)?; let has_affinity_session = self.affinity.is_some() && affinity_id(&request)?.is_some(); let is_direct = matches!(&self.policy, RoutingPolicy::Direct); diff --git a/lib/llm/src/kv_router/routing_host/kv.rs b/lib/llm/src/kv_router/routing_host/kv.rs index c6c3c505e5ab..1f6e0de6d68e 100644 --- a/lib/llm/src/kv_router/routing_host/kv.rs +++ b/lib/llm/src/kv_router/routing_host/kv.rs @@ -93,6 +93,7 @@ where is_query_only: bool, budget: &CleanupBudget, ) -> Result<(WorkerSelection, Option), Error> { + self.validate_explicit_worker(request.content(), phase)?; self.select_with_session_affinity(request, phase, is_query_only, budget, |target| { self.select_request(request, phase, is_query_only, target, budget) }) @@ -132,6 +133,7 @@ where if self.kv_router_if_enabled().is_none() { return Err(anyhow::anyhow!("KV route previews require KV routing")); } + self.validate_explicit_worker(request.content(), phase)?; let phase_label = phase.to_string(); let route_guard = StageGuard::new(STAGE_ROUTE, &phase_label); diff --git a/lib/llm/src/kv_router/routing_host/tests.rs b/lib/llm/src/kv_router/routing_host/tests.rs index b37ab458eea4..aa2e39ee7aa5 100644 --- a/lib/llm/src/kv_router/routing_host/tests.rs +++ b/lib/llm/src/kv_router/routing_host/tests.rs @@ -636,12 +636,13 @@ async fn builtin_direct_fallback_stays_disabled_for_affinity() { let mut standalone = request(); standalone.routing_mut().backend_instance_id = Some(stale_worker); - let mut stream = host.generate(Context::new(standalone)).await.unwrap(); - while stream.next().await.is_some() {} - assert_eq!( - dispatch.worker_ids.lock().unwrap().as_slice(), - &[real_worker] - ); + let error = host.generate(Context::new(standalone)).await.unwrap_err(); + assert!(match_error_chain( + error.as_ref(), + &[ErrorType::InvalidArgument], + &[] + )); + assert!(dispatch.worker_ids.lock().unwrap().is_empty()); let session_id = SessionAffinityId::new("direct-affinity"); let AffinityAcquire::Initialize(initializer) = @@ -654,15 +655,16 @@ async fn builtin_direct_fallback_stays_disabled_for_affinity() { .commit(AffinityTarget::worker(stale_worker)) .unwrap(), ); - assert!( - host.generate(affinity_request("direct-affinity", Some(stale_worker))) - .await - .is_err() - ); - assert_eq!( - dispatch.worker_ids.lock().unwrap().as_slice(), - &[real_worker] - ); + let error = host + .generate(affinity_request("direct-affinity", None)) + .await + .unwrap_err(); + assert!(!match_error_chain( + error.as_ref(), + &[ErrorType::InvalidArgument], + &[] + )); + assert!(dispatch.worker_ids.lock().unwrap().is_empty()); assert_eq!(affinity.query_target(&session_id, None).unwrap(), None); drop(host); @@ -1166,8 +1168,22 @@ async fn router_with_worker_configs( .component("workers".to_string()) .unwrap(); let endpoint = component.endpoint("generate"); - let client = endpoint.client().await.unwrap(); + let mut client = endpoint.client().await.unwrap(); let worker_ids = workers.keys().copied().collect::>(); + let instances = worker_ids + .iter() + .map(|&instance_id| Instance { + namespace: "affinity-selection-cancellation".into(), + component: "workers".into(), + endpoint: "generate".into(), + instance_id, + transport: dynamo_runtime::component::TransportType::Tcp("127.0.0.1:1".into()), + device_type: None, + request_plane_codec: None, + }) + .collect(); + let (_instances_tx, instances) = watch::channel(instances); + client.instance_source = Arc::new(instances); let (_tx, workers) = watch::channel(workers); let config = KvRouterConfig { skip_initial_worker_wait: true, @@ -2853,3 +2869,191 @@ async fn kv_stopped_decode_request_without_staged_kv_never_reaches_a_worker() { drop(router); runtime.shutdown(); } + +#[tokio::test] +async fn unknown_explicit_workers_are_rejected_before_builtin_dispatch() { + let runtime = Runtime::from_current().unwrap(); + let distributed = DistributedRuntime::new(runtime.clone(), DistributedConfig::process_local()) + .await + .unwrap(); + let component = distributed + .namespace("unknown-explicit-workers") + .unwrap() + .component("workers") + .unwrap(); + for (index, mode) in [ + RouterMode::RoundRobin, + RouterMode::Random, + RouterMode::PowerOfTwoChoices, + RouterMode::LeastLoaded, + RouterMode::DeviceAwareWeighted, + RouterMode::Direct, + ] + .into_iter() + .enumerate() + { + let endpoint = component.endpoint(format!("mode-{index}")); + let client = endpoint.client().await.unwrap(); + let load_context = test_load_context(&client).await; + let dispatch = Arc::new(CompletedBuiltinDispatch::default()); + let inner = PushRouter::from_client_with_dispatch( + client, + mode, + Arc::clone(&dispatch) as Arc>, + ) + .await + .unwrap(); + let host = RoutingHost::::new_builtin(inner, load_context).unwrap(); + for worker_id in [42, u64::MAX] { + for (field, phase) in [ + ("backend_instance_id", RequestPhase::Aggregated), + ("decode_worker_id", RequestPhase::Decode), + ("prefill_worker_id", RequestPhase::Prefill), + ] { + let mut content = request(); + content.routing = + Some(serde_json::from_value(serde_json::json!({field: worker_id})).unwrap()); + let error = host + .select_and_dispatch_builtin(Context::new(content), phase, |_, _| Ok(())) + .await + .unwrap_err(); + assert!(match_error_chain( + error.as_ref(), + &[ErrorType::InvalidArgument], + &[] + )); + assert_eq!( + error.downcast_ref::().unwrap().message(), + format!("nvext.{field} does not identify a known worker") + ); + } + } + assert!(dispatch.worker_ids.lock().unwrap().is_empty()); + } + runtime.shutdown(); +} + +#[tokio::test] +async fn unknown_explicit_workers_are_rejected_before_kv_admission() { + let (host, dispatch, worker_id, runtime) = + router_with_recorded_dispatch("unknown-kv-worker").await; + for unknown in [worker_id.wrapping_add(1), u64::MAX] { + assert!(!host.inner.client.is_instance_live(unknown)); + for (field, phase) in [ + ("backend_instance_id", RequestPhase::Aggregated), + ("decode_worker_id", RequestPhase::Decode), + ("prefill_worker_id", RequestPhase::Prefill), + ] { + let mut content = request(); + content.routing = + Some(serde_json::from_value(serde_json::json!({field: unknown})).unwrap()); + let request = Context::new(content); + let error = host + .select_with_affinity(&request, phase, false, &CleanupBudget::default()) + .await + .err() + .expect("unknown worker must fail before admission"); + assert!(match_error_chain( + error.as_ref(), + &[ErrorType::InvalidArgument], + &[] + )); + assert_eq!( + error.downcast_ref::().unwrap().message(), + format!("nvext.{field} does not identify a known worker") + ); + let error = host + .preview_kv_route(&request, phase) + .await + .err() + .expect("unknown preview target"); + assert!(match_error_chain( + error.as_ref(), + &[ErrorType::InvalidArgument], + &[] + )); + } + } + assert!(dispatch.worker_ids.lock().unwrap().is_empty()); + assert!( + host.kv_router() + .get_potential_loads(&[], None, None, None, None) + .await + .unwrap() + .iter() + .all(|load| load.active_requests == 0) + ); + drop(host); + runtime.shutdown(); +} + +#[tokio::test] +async fn explicit_worker_validation_preserves_precedence_and_discovery_membership() { + let (host, runtime) = router_with_workers(None, &[u64::MAX]).await; + let cases = [ + (serde_json::json!({}), RequestPhase::Aggregated), + ( + serde_json::json!({"backend_instance_id": null, "decode_worker_id": null, + "prefill_worker_id": null, "dp_rank": null}), + RequestPhase::Aggregated, + ), + ( + serde_json::json!({"backend_instance_id": u64::MAX}), + RequestPhase::Prefill, + ), + ( + serde_json::json!({"backend_instance_id": u64::MAX, "decode_worker_id": null}), + RequestPhase::Decode, + ), + ( + serde_json::json!({"backend_instance_id": 42, "decode_worker_id": u64::MAX, + "prefill_worker_id": 42}), + RequestPhase::Aggregated, + ), + ( + serde_json::json!({"backend_instance_id": 42, "decode_worker_id": 42, + "prefill_worker_id": u64::MAX}), + RequestPhase::Prefill, + ), + ]; + for (value, phase) in cases { + // Exercise the public nvext types as well as the routing hints. + let _: crate::protocols::common::extensions::NvExt = + serde_json::from_value(value.clone()).unwrap(); + let mut content = request(); + content.routing = Some(serde_json::from_value(value).unwrap()); + host.validate_explicit_worker(&content, phase).unwrap(); + host.inner.client.override_instance_avail(vec![]); + host.validate_explicit_worker(&content, phase).unwrap(); + } + drop(host); + runtime.shutdown(); +} + +#[tokio::test] +async fn explicit_worker_disappearing_after_preview_is_not_revalidated() { + let (mut host, runtime) = router(None).await; + let mut content = request(); + content.routing_mut().decode_worker_id = Some(7); + let request = Context::new(content); + let preview = host + .preview_kv_route(&request, RequestPhase::Decode) + .await + .unwrap(); + let (_tx, empty) = watch::channel(vec![]); + host.inner.client.instance_source = Arc::new(empty); + // Selection already succeeded. Planning must retain the admitted target; + // subsequent transport failure must keep its service-error classification. + let plan = host + .plan_kv_route_from_preview(&request, preview) + .await + .unwrap(); + let error = host.dispatch_kv_plan(request, plan).await.unwrap_err(); + assert!(!match_error_chain( + error.as_ref(), + &[ErrorType::InvalidArgument], + &[] + )); + drop(host); + runtime.shutdown(); +} From ad941cfa399aa7a5a512a744cbd24c1db8990e5c Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Tue, 15 Sep 2026 10:08:35 -0700 Subject: [PATCH 2/4] fix(router): preserve routing behavior during target validation Signed-off-by: jthomson04 --- lib/llm/src/http/service/openai.rs | 63 ------- lib/llm/src/kv_router/prefill_router/mod.rs | 79 ++++++++- lib/llm/src/kv_router/routing_host.rs | 5 +- lib/llm/src/kv_router/routing_host/builtin.rs | 2 +- lib/llm/src/kv_router/routing_host/tests.rs | 160 ++++++++++-------- 5 files changed, 167 insertions(+), 142 deletions(-) diff --git a/lib/llm/src/http/service/openai.rs b/lib/llm/src/http/service/openai.rs index 30251fd2fbae..50cb8cc56349 100644 --- a/lib/llm/src/http/service/openai.rs +++ b/lib/llm/src/http/service/openai.rs @@ -6295,69 +6295,6 @@ mod tests { ); } - #[tokio::test] - async fn unknown_explicit_worker_response_is_bad_request() { - use crate::{ - kv_router::{RouterLoadSource, RoutingHost, RoutingLoadContext}, - preprocessor::PreprocessedRequest, - }; - use dynamo_runtime::{ - DistributedRuntime, Runtime, - distributed::DistributedConfig, - pipeline::{AsyncEngine, Context, PushRouter, RouterMode}, - }; - - let runtime = Runtime::from_current().unwrap(); - let distributed = - DistributedRuntime::new(runtime.clone(), DistributedConfig::process_local()) - .await - .unwrap(); - let endpoint = distributed - .namespace("unknown-worker-http-error") - .unwrap() - .component("workers") - .unwrap() - .endpoint("generate"); - let client = endpoint.client().await.unwrap(); - let load_context = RoutingLoadContext::start( - client.clone(), - RouterLoadSource::Aggregated, - crate::discovery::LoadThresholdHandle::new(Default::default()), - &distributed.child_token(), - None, - ) - .await - .unwrap(); - let inner = PushRouter::from_client(client, RouterMode::RoundRobin) - .await - .unwrap(); - let host = RoutingHost::::new_builtin( - inner, - load_context, - ) - .unwrap(); - let mut request = PreprocessedRequest::builder() - .model("test".into()) - .token_ids(vec![1]) - .stop_conditions(Default::default()) - .sampling_options(Default::default()) - .output_options(Default::default()) - .build() - .unwrap(); - request.routing_mut().decode_worker_id = Some(u64::MAX); - let error = host.generate(Context::new(request)).await.unwrap_err(); - let response = ErrorMessage::from_anyhow(error, BACKUP_ERROR_MESSAGE); - assert_eq!(response.0, StatusCode::BAD_REQUEST); - assert_eq!(response.1.code, 400); - assert_eq!(response.1.error_type, "Bad Request"); - assert_eq!( - response.1.message, - "nvext.decode_worker_id does not identify a known worker" - ); - drop(host); - runtime.shutdown(); - } - #[test] fn test_nested_invalid_argument_response_from_anyhow() { use dynamo_runtime::error::{DynamoError, ErrorType}; diff --git a/lib/llm/src/kv_router/prefill_router/mod.rs b/lib/llm/src/kv_router/prefill_router/mod.rs index 53b2cdceee5b..416148eb4887 100644 --- a/lib/llm/src/kv_router/prefill_router/mod.rs +++ b/lib/llm/src/kv_router/prefill_router/mod.rs @@ -20,6 +20,7 @@ use dynamo_kv_router::{ selector::{DefaultWorkerSelector, WorkerSelector}, }; use dynamo_runtime::{ + error::{ErrorType, match_error_chain}, pipeline::{ AsyncEngineContextProvider, Context, ManyOut, Operator, ResponseStream, RouterMode, ServerStreamingEngine, SingleIn, async_trait, propagate_first_response_guard, @@ -401,7 +402,13 @@ where Ok(None) => { (req, context) = conditional_request.into_parts(); } - Err(error) if crate::kv_router::routing_host::is_cancelled(&error) => { + Err(error) + if match_error_chain( + error.as_ref(), + &[ErrorType::Cancelled, ErrorType::InvalidArgument], + &[], + ) => + { return Err(error); } Err(error) => { @@ -847,6 +854,76 @@ mod tests { } } + #[tokio::test] + async fn conditional_disagg_rejects_unknown_decode_target_before_fallback() { + use crate::kv_router::KvRouter; + use dynamo_runtime::{ + DistributedRuntime, Runtime, distributed::DistributedConfig, pipeline::PushRouter, + }; + + let runtime = Runtime::from_current().unwrap(); + let distributed = + DistributedRuntime::new(runtime.clone(), DistributedConfig::process_local()) + .await + .unwrap(); + let endpoint = distributed + .namespace("conditional-invalid-worker") + .unwrap() + .component("decode") + .unwrap() + .endpoint("generate"); + let client = endpoint.client().await.unwrap(); + let (_workers_tx, workers) = watch::channel(HashMap::::new()); + let config = KvRouterConfig { + skip_initial_worker_wait: true, + use_kv_events: false, + router_track_active_blocks: false, + ..Default::default() + }; + let chooser = KvRouter::new( + endpoint, + client.clone(), + workers, + None, + 16, + DefaultWorkerSelector::new(Some(config.clone()), "decode"), + Some(config), + None, + "decode", + None, + false, + None, + None, + ) + .await + .unwrap(); + let inner = PushRouter::from_client(client, RouterMode::KV) + .await + .unwrap(); + let host = Arc::new(RoutingHost::new(inner, Arc::new(chooser), None).unwrap()); + let router = active_conditional_router(); + assert!(router.decode_routing_host.set(host).is_ok()); + let downstream = Arc::new(QueryOnlyDecodeHost::default()); + let mut request = query_only_request(); + request.annotations.clear(); + request.routing_mut().decode_worker_id = Some(u64::MAX); + + // No prefill binding: the old fallback would continue to downstream instead + // of returning the preview's client error. + let error = router + .generate(SingleIn::new(request), downstream.clone()) + .await + .unwrap_err(); + assert!(match_error_chain( + error.as_ref(), + &[ErrorType::InvalidArgument], + &[] + )); + assert_eq!(downstream.requests.load(Ordering::Relaxed), 0); + drop(router); + runtime.shutdown(); + } + #[tokio::test] async fn conditional_disagg_query_only_forwards_to_decode_host() { let router = active_conditional_router(); diff --git a/lib/llm/src/kv_router/routing_host.rs b/lib/llm/src/kv_router/routing_host.rs index fad76bb87002..df3e075c2baa 100644 --- a/lib/llm/src/kv_router/routing_host.rs +++ b/lib/llm/src/kv_router/routing_host.rs @@ -599,11 +599,12 @@ where .backend_instance_id .map(|id| (id, "backend_instance_id")) }); + // Validate discovery membership, not health: unavailable workers are still known. if let Some((worker_id, field)) = target - && !self.inner.client.is_instance_live(worker_id) + && !self.inner.client.is_instance_discovered(worker_id) { return Err(invalid_argument(format!( - "nvext.{field} does not identify a known worker" + "nvext.{field}={worker_id} does not identify a known worker" ))); } Ok(()) diff --git a/lib/llm/src/kv_router/routing_host/builtin.rs b/lib/llm/src/kv_router/routing_host/builtin.rs index 25295e353160..168adc439dad 100644 --- a/lib/llm/src/kv_router/routing_host/builtin.rs +++ b/lib/llm/src/kv_router/routing_host/builtin.rs @@ -288,7 +288,6 @@ where { let phase_label = phase.to_string(); let route_guard = StageGuard::new(STAGE_ROUTE, &phase_label); - self.validate_explicit_worker(request.content(), phase)?; let explicit = explicit_target(request.content(), phase)?; let has_affinity_session = self.affinity.is_some() && affinity_id(&request)?.is_some(); let is_direct = matches!(&self.policy, RoutingPolicy::Direct); @@ -324,6 +323,7 @@ where None, ) } else { + self.validate_explicit_worker(request.content(), phase)?; self.select_with_session_affinity(&request, phase, is_query_only, &budget, |target| { let pinned_target = explicit.or(match self.session_affinity_mode { SessionAffinityMode::Hard => target, diff --git a/lib/llm/src/kv_router/routing_host/tests.rs b/lib/llm/src/kv_router/routing_host/tests.rs index aa2e39ee7aa5..3dd23e6351ab 100644 --- a/lib/llm/src/kv_router/routing_host/tests.rs +++ b/lib/llm/src/kv_router/routing_host/tests.rs @@ -455,6 +455,7 @@ async fn builtin_lora_keeps_separate_selection_and_cleanup() { let mut adapter_request = request(); adapter_request.routing_mut().lora_name = Some("adapter".to_string()); + adapter_request.routing_mut().backend_instance_id = Some(stale_worker); let mut adapter_stream = host.generate(Context::new(adapter_request)).await.unwrap(); assert_eq!(estimator.get_inflight_counts().get("adapter"), Some(&1)); while adapter_stream.next().await.is_some() {} @@ -600,7 +601,7 @@ async fn builtin_hard_affinity_ignores_overload_while_soft_affinity_falls_back() #[tokio::test] #[serial_test::serial] -async fn builtin_direct_fallback_stays_disabled_for_affinity() { +async fn builtin_direct_distinguishes_unknown_requests_from_stale_affinity() { let runtime = Runtime::from_current().unwrap(); let distributed = DistributedRuntime::new(runtime.clone(), DistributedConfig::process_local()) .await @@ -667,6 +668,15 @@ async fn builtin_direct_fallback_stays_disabled_for_affinity() { assert!(dispatch.worker_ids.lock().unwrap().is_empty()); assert_eq!(affinity.query_target(&session_id, None).unwrap(), None); + let mut valid = request(); + valid.routing_mut().backend_instance_id = Some(real_worker); + let mut stream = host.generate(Context::new(valid)).await.unwrap(); + while stream.next().await.is_some() {} + assert_eq!( + dispatch.worker_ids.lock().unwrap().as_slice(), + &[real_worker] + ); + drop(host); runtime.shutdown(); } @@ -1168,22 +1178,8 @@ async fn router_with_worker_configs( .component("workers".to_string()) .unwrap(); let endpoint = component.endpoint("generate"); - let mut client = endpoint.client().await.unwrap(); + let client = endpoint.client().await.unwrap(); let worker_ids = workers.keys().copied().collect::>(); - let instances = worker_ids - .iter() - .map(|&instance_id| Instance { - namespace: "affinity-selection-cancellation".into(), - component: "workers".into(), - endpoint: "generate".into(), - instance_id, - transport: dynamo_runtime::component::TransportType::Tcp("127.0.0.1:1".into()), - device_type: None, - request_plane_codec: None, - }) - .collect(); - let (_instances_tx, instances) = watch::channel(instances); - client.instance_source = Arc::new(instances); let (_tx, workers) = watch::channel(workers); let config = KvRouterConfig { skip_initial_worker_wait: true, @@ -2881,55 +2877,54 @@ async fn unknown_explicit_workers_are_rejected_before_builtin_dispatch() { .unwrap() .component("workers") .unwrap(); - for (index, mode) in [ + let endpoint = component.endpoint("generate"); + let client = endpoint.client().await.unwrap(); + let load_context = test_load_context(&client).await; + endpoint.register_endpoint_instance().await.unwrap(); + let live_worker = client.wait_for_instances().await.unwrap()[0].id(); + let dispatch = Arc::new(CompletedBuiltinDispatch::default()); + let inner = PushRouter::from_client_with_dispatch( + client, RouterMode::RoundRobin, - RouterMode::Random, - RouterMode::PowerOfTwoChoices, - RouterMode::LeastLoaded, - RouterMode::DeviceAwareWeighted, - RouterMode::Direct, - ] - .into_iter() - .enumerate() - { - let endpoint = component.endpoint(format!("mode-{index}")); - let client = endpoint.client().await.unwrap(); - let load_context = test_load_context(&client).await; - let dispatch = Arc::new(CompletedBuiltinDispatch::default()); - let inner = PushRouter::from_client_with_dispatch( - client, - mode, - Arc::clone(&dispatch) as Arc>, - ) - .await - .unwrap(); - let host = RoutingHost::::new_builtin(inner, load_context).unwrap(); - for worker_id in [42, u64::MAX] { - for (field, phase) in [ - ("backend_instance_id", RequestPhase::Aggregated), - ("decode_worker_id", RequestPhase::Decode), - ("prefill_worker_id", RequestPhase::Prefill), - ] { - let mut content = request(); - content.routing = - Some(serde_json::from_value(serde_json::json!({field: worker_id})).unwrap()); - let error = host - .select_and_dispatch_builtin(Context::new(content), phase, |_, _| Ok(())) - .await - .unwrap_err(); - assert!(match_error_chain( - error.as_ref(), - &[ErrorType::InvalidArgument], - &[] - )); - assert_eq!( - error.downcast_ref::().unwrap().message(), - format!("nvext.{field} does not identify a known worker") - ); - } + Arc::clone(&dispatch) as Arc>, + ) + .await + .unwrap(); + let host = RoutingHost::::new_builtin(inner, load_context).unwrap(); + for worker_id in [live_worker.wrapping_add(1), u64::MAX] { + for (field, phase) in [ + ("backend_instance_id", RequestPhase::Aggregated), + ("decode_worker_id", RequestPhase::Decode), + ("prefill_worker_id", RequestPhase::Prefill), + ] { + let mut content = request(); + content.routing = + Some(serde_json::from_value(serde_json::json!({field: worker_id})).unwrap()); + let error = host + .select_and_dispatch_builtin(Context::new(content), phase, |_, _| Ok(())) + .await + .unwrap_err(); + assert!(match_error_chain( + error.as_ref(), + &[ErrorType::InvalidArgument], + &[] + )); + assert_eq!( + error.downcast_ref::().unwrap().message(), + format!("nvext.{field}={worker_id} does not identify a known worker") + ); } - assert!(dispatch.worker_ids.lock().unwrap().is_empty()); } + assert!(dispatch.worker_ids.lock().unwrap().is_empty()); + let mut valid = request(); + valid.routing_mut().decode_worker_id = Some(live_worker); + let mut stream = host.generate(Context::new(valid)).await.unwrap(); + while stream.next().await.is_some() {} + assert_eq!( + dispatch.worker_ids.lock().unwrap().as_slice(), + &[live_worker] + ); + drop(host); runtime.shutdown(); } @@ -2960,7 +2955,7 @@ async fn unknown_explicit_workers_are_rejected_before_kv_admission() { )); assert_eq!( error.downcast_ref::().unwrap().message(), - format!("nvext.{field} does not identify a known worker") + format!("nvext.{field}={unknown} does not identify a known worker") ); let error = host .preview_kv_route(&request, phase) @@ -2988,6 +2983,7 @@ async fn unknown_explicit_workers_are_rejected_before_kv_admission() { } #[tokio::test] +#[serial_test::serial] async fn explicit_worker_validation_preserves_precedence_and_discovery_membership() { let (host, runtime) = router_with_workers(None, &[u64::MAX]).await; let cases = [ @@ -3016,6 +3012,7 @@ async fn explicit_worker_validation_preserves_precedence_and_discovery_membershi RequestPhase::Prefill, ), ]; + host.inner.client.override_instance_avail(vec![]); for (value, phase) in cases { // Exercise the public nvext types as well as the routing hints. let _: crate::protocols::common::extensions::NvExt = @@ -3023,16 +3020,32 @@ async fn explicit_worker_validation_preserves_precedence_and_discovery_membershi let mut content = request(); content.routing = Some(serde_json::from_value(value).unwrap()); host.validate_explicit_worker(&content, phase).unwrap(); - host.inner.client.override_instance_avail(vec![]); - host.validate_explicit_worker(&content, phase).unwrap(); + } + for (field, phase) in [ + ("decode_worker_id", RequestPhase::Aggregated), + ("prefill_worker_id", RequestPhase::Prefill), + ] { + let mut content = request(); + content.routing = Some( + serde_json::from_value(serde_json::json!({ + "backend_instance_id": u64::MAX, field: 42 + })) + .unwrap(), + ); + let error = host.validate_explicit_worker(&content, phase).unwrap_err(); + assert_eq!( + error.downcast_ref::().unwrap().message(), + format!("nvext.{field}=42 does not identify a known worker") + ); } drop(host); runtime.shutdown(); } #[tokio::test] +#[serial_test::serial] async fn explicit_worker_disappearing_after_preview_is_not_revalidated() { - let (mut host, runtime) = router(None).await; + let (host, runtime) = router(None).await; let mut content = request(); content.routing_mut().decode_worker_id = Some(7); let request = Context::new(content); @@ -3040,15 +3053,12 @@ async fn explicit_worker_disappearing_after_preview_is_not_revalidated() { .preview_kv_route(&request, RequestPhase::Decode) .await .unwrap(); - let (_tx, empty) = watch::channel(vec![]); - host.inner.client.instance_source = Arc::new(empty); - // Selection already succeeded. Planning must retain the admitted target; - // subsequent transport failure must keep its service-error classification. - let plan = host - .plan_kv_route_from_preview(&request, preview) - .await - .unwrap(); - let error = host.dispatch_kv_plan(request, plan).await.unwrap_err(); + host.inner.client.override_discovered_instances(vec![]); + // A failure after successful selection must keep its service-error classification. + let error = match host.plan_kv_route_from_preview(&request, preview).await { + Ok(plan) => host.dispatch_kv_plan(request, plan).await.unwrap_err(), + Err(error) => error, + }; assert!(!match_error_chain( error.as_ref(), &[ErrorType::InvalidArgument], From 61946ea21821c9e666445cfa97b438835ed6c100 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Tue, 15 Sep 2026 12:36:08 -0700 Subject: [PATCH 3/4] test(router): simplify unknown-worker rejection cases Signed-off-by: jthomson04 --- lib/llm/src/kv_router/routing_host/tests.rs | 117 ++++++++++---------- 1 file changed, 57 insertions(+), 60 deletions(-) diff --git a/lib/llm/src/kv_router/routing_host/tests.rs b/lib/llm/src/kv_router/routing_host/tests.rs index 3dd23e6351ab..24860b68ffb8 100644 --- a/lib/llm/src/kv_router/routing_host/tests.rs +++ b/lib/llm/src/kv_router/routing_host/tests.rs @@ -2891,29 +2891,28 @@ async fn unknown_explicit_workers_are_rejected_before_builtin_dispatch() { .await .unwrap(); let host = RoutingHost::::new_builtin(inner, load_context).unwrap(); - for worker_id in [live_worker.wrapping_add(1), u64::MAX] { - for (field, phase) in [ - ("backend_instance_id", RequestPhase::Aggregated), - ("decode_worker_id", RequestPhase::Decode), - ("prefill_worker_id", RequestPhase::Prefill), - ] { - let mut content = request(); - content.routing = - Some(serde_json::from_value(serde_json::json!({field: worker_id})).unwrap()); - let error = host - .select_and_dispatch_builtin(Context::new(content), phase, |_, _| Ok(())) - .await - .unwrap_err(); - assert!(match_error_chain( - error.as_ref(), - &[ErrorType::InvalidArgument], - &[] - )); - assert_eq!( - error.downcast_ref::().unwrap().message(), - format!("nvext.{field}={worker_id} does not identify a known worker") - ); - } + let worker_id = live_worker.wrapping_add(1); + for (field, phase) in [ + ("backend_instance_id", RequestPhase::Aggregated), + ("decode_worker_id", RequestPhase::Decode), + ("prefill_worker_id", RequestPhase::Prefill), + ] { + let mut content = request(); + content.routing = + Some(serde_json::from_value(serde_json::json!({field: worker_id})).unwrap()); + let error = host + .select_and_dispatch_builtin(Context::new(content), phase, |_, _| Ok(())) + .await + .unwrap_err(); + assert!(match_error_chain( + error.as_ref(), + &[ErrorType::InvalidArgument], + &[] + )); + assert_eq!( + error.downcast_ref::().unwrap().message(), + format!("nvext.{field}={worker_id} does not identify a known worker") + ); } assert!(dispatch.worker_ids.lock().unwrap().is_empty()); let mut valid = request(); @@ -2932,42 +2931,41 @@ async fn unknown_explicit_workers_are_rejected_before_builtin_dispatch() { async fn unknown_explicit_workers_are_rejected_before_kv_admission() { let (host, dispatch, worker_id, runtime) = router_with_recorded_dispatch("unknown-kv-worker").await; - for unknown in [worker_id.wrapping_add(1), u64::MAX] { - assert!(!host.inner.client.is_instance_live(unknown)); - for (field, phase) in [ - ("backend_instance_id", RequestPhase::Aggregated), - ("decode_worker_id", RequestPhase::Decode), - ("prefill_worker_id", RequestPhase::Prefill), - ] { - let mut content = request(); - content.routing = - Some(serde_json::from_value(serde_json::json!({field: unknown})).unwrap()); - let request = Context::new(content); - let error = host - .select_with_affinity(&request, phase, false, &CleanupBudget::default()) - .await - .err() - .expect("unknown worker must fail before admission"); - assert!(match_error_chain( - error.as_ref(), - &[ErrorType::InvalidArgument], - &[] - )); - assert_eq!( - error.downcast_ref::().unwrap().message(), - format!("nvext.{field}={unknown} does not identify a known worker") - ); - let error = host - .preview_kv_route(&request, phase) - .await - .err() - .expect("unknown preview target"); - assert!(match_error_chain( - error.as_ref(), - &[ErrorType::InvalidArgument], - &[] - )); - } + let unknown = worker_id.wrapping_add(1); + assert!(!host.inner.client.is_instance_live(unknown)); + for (field, phase) in [ + ("backend_instance_id", RequestPhase::Aggregated), + ("decode_worker_id", RequestPhase::Decode), + ("prefill_worker_id", RequestPhase::Prefill), + ] { + let mut content = request(); + content.routing = + Some(serde_json::from_value(serde_json::json!({field: unknown})).unwrap()); + let request = Context::new(content); + let error = host + .select_with_affinity(&request, phase, false, &CleanupBudget::default()) + .await + .err() + .expect("unknown worker must fail before admission"); + assert!(match_error_chain( + error.as_ref(), + &[ErrorType::InvalidArgument], + &[] + )); + assert_eq!( + error.downcast_ref::().unwrap().message(), + format!("nvext.{field}={unknown} does not identify a known worker") + ); + let error = host + .preview_kv_route(&request, phase) + .await + .err() + .expect("unknown preview target"); + assert!(match_error_chain( + error.as_ref(), + &[ErrorType::InvalidArgument], + &[] + )); } assert!(dispatch.worker_ids.lock().unwrap().is_empty()); assert!( @@ -3014,7 +3012,6 @@ async fn explicit_worker_validation_preserves_precedence_and_discovery_membershi ]; host.inner.client.override_instance_avail(vec![]); for (value, phase) in cases { - // Exercise the public nvext types as well as the routing hints. let _: crate::protocols::common::extensions::NvExt = serde_json::from_value(value.clone()).unwrap(); let mut content = request(); From e843aea5b5ecf291e4b05209e06a2e9f7190a106 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Tue, 15 Sep 2026 14:01:26 -0700 Subject: [PATCH 4/4] test(router): serialize worker validation dispatch test Signed-off-by: jthomson04 --- lib/llm/src/kv_router/routing_host/tests.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/llm/src/kv_router/routing_host/tests.rs b/lib/llm/src/kv_router/routing_host/tests.rs index 24860b68ffb8..1034d5228fa5 100644 --- a/lib/llm/src/kv_router/routing_host/tests.rs +++ b/lib/llm/src/kv_router/routing_host/tests.rs @@ -2867,6 +2867,7 @@ async fn kv_stopped_decode_request_without_staged_kv_never_reaches_a_worker() { } #[tokio::test] +#[serial_test::serial] async fn unknown_explicit_workers_are_rejected_before_builtin_dispatch() { let runtime = Runtime::from_current().unwrap(); let distributed = DistributedRuntime::new(runtime.clone(), DistributedConfig::process_local())