diff --git a/lib/llm/src/kv_router/prefill_router/mod.rs b/lib/llm/src/kv_router/prefill_router/mod.rs index 53b2cdceee5b..416148eb4887 100644 --- a/lib/llm/src/kv_router/prefill_router/mod.rs +++ b/lib/llm/src/kv_router/prefill_router/mod.rs @@ -20,6 +20,7 @@ use dynamo_kv_router::{ selector::{DefaultWorkerSelector, WorkerSelector}, }; use dynamo_runtime::{ + error::{ErrorType, match_error_chain}, pipeline::{ AsyncEngineContextProvider, Context, ManyOut, Operator, ResponseStream, RouterMode, ServerStreamingEngine, SingleIn, async_trait, propagate_first_response_guard, @@ -401,7 +402,13 @@ where Ok(None) => { (req, context) = conditional_request.into_parts(); } - Err(error) if crate::kv_router::routing_host::is_cancelled(&error) => { + Err(error) + if match_error_chain( + error.as_ref(), + &[ErrorType::Cancelled, ErrorType::InvalidArgument], + &[], + ) => + { return Err(error); } Err(error) => { @@ -847,6 +854,76 @@ mod tests { } } + #[tokio::test] + async fn conditional_disagg_rejects_unknown_decode_target_before_fallback() { + use crate::kv_router::KvRouter; + use dynamo_runtime::{ + DistributedRuntime, Runtime, distributed::DistributedConfig, pipeline::PushRouter, + }; + + let runtime = Runtime::from_current().unwrap(); + let distributed = + DistributedRuntime::new(runtime.clone(), DistributedConfig::process_local()) + .await + .unwrap(); + let endpoint = distributed + .namespace("conditional-invalid-worker") + .unwrap() + .component("decode") + .unwrap() + .endpoint("generate"); + let client = endpoint.client().await.unwrap(); + let (_workers_tx, workers) = watch::channel(HashMap::::new()); + let config = KvRouterConfig { + skip_initial_worker_wait: true, + use_kv_events: false, + router_track_active_blocks: false, + ..Default::default() + }; + let chooser = KvRouter::new( + endpoint, + client.clone(), + workers, + None, + 16, + DefaultWorkerSelector::new(Some(config.clone()), "decode"), + Some(config), + None, + "decode", + None, + false, + None, + None, + ) + .await + .unwrap(); + let inner = PushRouter::from_client(client, RouterMode::KV) + .await + .unwrap(); + let host = Arc::new(RoutingHost::new(inner, Arc::new(chooser), None).unwrap()); + let router = active_conditional_router(); + assert!(router.decode_routing_host.set(host).is_ok()); + let downstream = Arc::new(QueryOnlyDecodeHost::default()); + let mut request = query_only_request(); + request.annotations.clear(); + request.routing_mut().decode_worker_id = Some(u64::MAX); + + // No prefill binding: the old fallback would continue to downstream instead + // of returning the preview's client error. + let error = router + .generate(SingleIn::new(request), downstream.clone()) + .await + .unwrap_err(); + assert!(match_error_chain( + error.as_ref(), + &[ErrorType::InvalidArgument], + &[] + )); + assert_eq!(downstream.requests.load(Ordering::Relaxed), 0); + drop(router); + runtime.shutdown(); + } + #[tokio::test] async fn conditional_disagg_query_only_forwards_to_decode_host() { let router = active_conditional_router(); diff --git a/lib/llm/src/kv_router/routing_host.rs b/lib/llm/src/kv_router/routing_host.rs index 34c458faf3fc..df3e075c2baa 100644 --- a/lib/llm/src/kv_router/routing_host.rs +++ b/lib/llm/src/kv_router/routing_host.rs @@ -576,6 +576,40 @@ where } } + /// Check request-supplied targets only at initial selection, not after a route preview + /// or when dispatching: a worker disappearing later is a service failure. + fn validate_explicit_worker( + &self, + request: &PreprocessedRequest, + phase: RequestPhase, + ) -> Result<(), Error> { + let Some(routing) = request.routing.as_ref() else { + return Ok(()); + }; + let target = match phase { + RequestPhase::Prefill => routing + .prefill_worker_id + .map(|id| (id, "prefill_worker_id")), + RequestPhase::Decode | RequestPhase::Aggregated => { + routing.decode_worker_id.map(|id| (id, "decode_worker_id")) + } + } + .or_else(|| { + routing + .backend_instance_id + .map(|id| (id, "backend_instance_id")) + }); + // Validate discovery membership, not health: unavailable workers are still known. + if let Some((worker_id, field)) = target + && !self.inner.client.is_instance_discovered(worker_id) + { + return Err(invalid_argument(format!( + "nvext.{field}={worker_id} does not identify a known worker" + ))); + } + Ok(()) + } + fn affinity_target_is_valid(&self, target: AffinityTarget) -> bool { if !self.inner.client.is_instance_discovered(target.worker_id) { return false; diff --git a/lib/llm/src/kv_router/routing_host/builtin.rs b/lib/llm/src/kv_router/routing_host/builtin.rs index 28b511f93c1c..168adc439dad 100644 --- a/lib/llm/src/kv_router/routing_host/builtin.rs +++ b/lib/llm/src/kv_router/routing_host/builtin.rs @@ -323,6 +323,7 @@ where None, ) } else { + self.validate_explicit_worker(request.content(), phase)?; self.select_with_session_affinity(&request, phase, is_query_only, &budget, |target| { let pinned_target = explicit.or(match self.session_affinity_mode { SessionAffinityMode::Hard => target, diff --git a/lib/llm/src/kv_router/routing_host/kv.rs b/lib/llm/src/kv_router/routing_host/kv.rs index c6c3c505e5ab..1f6e0de6d68e 100644 --- a/lib/llm/src/kv_router/routing_host/kv.rs +++ b/lib/llm/src/kv_router/routing_host/kv.rs @@ -93,6 +93,7 @@ where is_query_only: bool, budget: &CleanupBudget, ) -> Result<(WorkerSelection, Option), Error> { + self.validate_explicit_worker(request.content(), phase)?; self.select_with_session_affinity(request, phase, is_query_only, budget, |target| { self.select_request(request, phase, is_query_only, target, budget) }) @@ -132,6 +133,7 @@ where if self.kv_router_if_enabled().is_none() { return Err(anyhow::anyhow!("KV route previews require KV routing")); } + self.validate_explicit_worker(request.content(), phase)?; let phase_label = phase.to_string(); let route_guard = StageGuard::new(STAGE_ROUTE, &phase_label); diff --git a/lib/llm/src/kv_router/routing_host/tests.rs b/lib/llm/src/kv_router/routing_host/tests.rs index b37ab458eea4..1034d5228fa5 100644 --- a/lib/llm/src/kv_router/routing_host/tests.rs +++ b/lib/llm/src/kv_router/routing_host/tests.rs @@ -455,6 +455,7 @@ async fn builtin_lora_keeps_separate_selection_and_cleanup() { let mut adapter_request = request(); adapter_request.routing_mut().lora_name = Some("adapter".to_string()); + adapter_request.routing_mut().backend_instance_id = Some(stale_worker); let mut adapter_stream = host.generate(Context::new(adapter_request)).await.unwrap(); assert_eq!(estimator.get_inflight_counts().get("adapter"), Some(&1)); while adapter_stream.next().await.is_some() {} @@ -600,7 +601,7 @@ async fn builtin_hard_affinity_ignores_overload_while_soft_affinity_falls_back() #[tokio::test] #[serial_test::serial] -async fn builtin_direct_fallback_stays_disabled_for_affinity() { +async fn builtin_direct_distinguishes_unknown_requests_from_stale_affinity() { let runtime = Runtime::from_current().unwrap(); let distributed = DistributedRuntime::new(runtime.clone(), DistributedConfig::process_local()) .await @@ -636,12 +637,13 @@ async fn builtin_direct_fallback_stays_disabled_for_affinity() { let mut standalone = request(); standalone.routing_mut().backend_instance_id = Some(stale_worker); - let mut stream = host.generate(Context::new(standalone)).await.unwrap(); - while stream.next().await.is_some() {} - assert_eq!( - dispatch.worker_ids.lock().unwrap().as_slice(), - &[real_worker] - ); + let error = host.generate(Context::new(standalone)).await.unwrap_err(); + assert!(match_error_chain( + error.as_ref(), + &[ErrorType::InvalidArgument], + &[] + )); + assert!(dispatch.worker_ids.lock().unwrap().is_empty()); let session_id = SessionAffinityId::new("direct-affinity"); let AffinityAcquire::Initialize(initializer) = @@ -654,16 +656,26 @@ async fn builtin_direct_fallback_stays_disabled_for_affinity() { .commit(AffinityTarget::worker(stale_worker)) .unwrap(), ); - assert!( - host.generate(affinity_request("direct-affinity", Some(stale_worker))) - .await - .is_err() - ); + let error = host + .generate(affinity_request("direct-affinity", None)) + .await + .unwrap_err(); + assert!(!match_error_chain( + error.as_ref(), + &[ErrorType::InvalidArgument], + &[] + )); + assert!(dispatch.worker_ids.lock().unwrap().is_empty()); + assert_eq!(affinity.query_target(&session_id, None).unwrap(), None); + + let mut valid = request(); + valid.routing_mut().backend_instance_id = Some(real_worker); + let mut stream = host.generate(Context::new(valid)).await.unwrap(); + while stream.next().await.is_some() {} assert_eq!( dispatch.worker_ids.lock().unwrap().as_slice(), &[real_worker] ); - assert_eq!(affinity.query_target(&session_id, None).unwrap(), None); drop(host); runtime.shutdown(); @@ -2853,3 +2865,203 @@ async fn kv_stopped_decode_request_without_staged_kv_never_reaches_a_worker() { drop(router); runtime.shutdown(); } + +#[tokio::test] +#[serial_test::serial] +async fn unknown_explicit_workers_are_rejected_before_builtin_dispatch() { + let runtime = Runtime::from_current().unwrap(); + let distributed = DistributedRuntime::new(runtime.clone(), DistributedConfig::process_local()) + .await + .unwrap(); + let component = distributed + .namespace("unknown-explicit-workers") + .unwrap() + .component("workers") + .unwrap(); + let endpoint = component.endpoint("generate"); + let client = endpoint.client().await.unwrap(); + let load_context = test_load_context(&client).await; + endpoint.register_endpoint_instance().await.unwrap(); + let live_worker = client.wait_for_instances().await.unwrap()[0].id(); + let dispatch = Arc::new(CompletedBuiltinDispatch::default()); + let inner = PushRouter::from_client_with_dispatch( + client, + RouterMode::RoundRobin, + Arc::clone(&dispatch) as Arc>, + ) + .await + .unwrap(); + let host = RoutingHost::::new_builtin(inner, load_context).unwrap(); + let worker_id = live_worker.wrapping_add(1); + for (field, phase) in [ + ("backend_instance_id", RequestPhase::Aggregated), + ("decode_worker_id", RequestPhase::Decode), + ("prefill_worker_id", RequestPhase::Prefill), + ] { + let mut content = request(); + content.routing = + Some(serde_json::from_value(serde_json::json!({field: worker_id})).unwrap()); + let error = host + .select_and_dispatch_builtin(Context::new(content), phase, |_, _| Ok(())) + .await + .unwrap_err(); + assert!(match_error_chain( + error.as_ref(), + &[ErrorType::InvalidArgument], + &[] + )); + assert_eq!( + error.downcast_ref::().unwrap().message(), + format!("nvext.{field}={worker_id} does not identify a known worker") + ); + } + assert!(dispatch.worker_ids.lock().unwrap().is_empty()); + let mut valid = request(); + valid.routing_mut().decode_worker_id = Some(live_worker); + let mut stream = host.generate(Context::new(valid)).await.unwrap(); + while stream.next().await.is_some() {} + assert_eq!( + dispatch.worker_ids.lock().unwrap().as_slice(), + &[live_worker] + ); + drop(host); + runtime.shutdown(); +} + +#[tokio::test] +async fn unknown_explicit_workers_are_rejected_before_kv_admission() { + let (host, dispatch, worker_id, runtime) = + router_with_recorded_dispatch("unknown-kv-worker").await; + let unknown = worker_id.wrapping_add(1); + assert!(!host.inner.client.is_instance_live(unknown)); + for (field, phase) in [ + ("backend_instance_id", RequestPhase::Aggregated), + ("decode_worker_id", RequestPhase::Decode), + ("prefill_worker_id", RequestPhase::Prefill), + ] { + let mut content = request(); + content.routing = + Some(serde_json::from_value(serde_json::json!({field: unknown})).unwrap()); + let request = Context::new(content); + let error = host + .select_with_affinity(&request, phase, false, &CleanupBudget::default()) + .await + .err() + .expect("unknown worker must fail before admission"); + assert!(match_error_chain( + error.as_ref(), + &[ErrorType::InvalidArgument], + &[] + )); + assert_eq!( + error.downcast_ref::().unwrap().message(), + format!("nvext.{field}={unknown} does not identify a known worker") + ); + let error = host + .preview_kv_route(&request, phase) + .await + .err() + .expect("unknown preview target"); + assert!(match_error_chain( + error.as_ref(), + &[ErrorType::InvalidArgument], + &[] + )); + } + assert!(dispatch.worker_ids.lock().unwrap().is_empty()); + assert!( + host.kv_router() + .get_potential_loads(&[], None, None, None, None) + .await + .unwrap() + .iter() + .all(|load| load.active_requests == 0) + ); + drop(host); + runtime.shutdown(); +} + +#[tokio::test] +#[serial_test::serial] +async fn explicit_worker_validation_preserves_precedence_and_discovery_membership() { + let (host, runtime) = router_with_workers(None, &[u64::MAX]).await; + let cases = [ + (serde_json::json!({}), RequestPhase::Aggregated), + ( + serde_json::json!({"backend_instance_id": null, "decode_worker_id": null, + "prefill_worker_id": null, "dp_rank": null}), + RequestPhase::Aggregated, + ), + ( + serde_json::json!({"backend_instance_id": u64::MAX}), + RequestPhase::Prefill, + ), + ( + serde_json::json!({"backend_instance_id": u64::MAX, "decode_worker_id": null}), + RequestPhase::Decode, + ), + ( + serde_json::json!({"backend_instance_id": 42, "decode_worker_id": u64::MAX, + "prefill_worker_id": 42}), + RequestPhase::Aggregated, + ), + ( + serde_json::json!({"backend_instance_id": 42, "decode_worker_id": 42, + "prefill_worker_id": u64::MAX}), + RequestPhase::Prefill, + ), + ]; + host.inner.client.override_instance_avail(vec![]); + for (value, phase) in cases { + let _: crate::protocols::common::extensions::NvExt = + serde_json::from_value(value.clone()).unwrap(); + let mut content = request(); + content.routing = Some(serde_json::from_value(value).unwrap()); + host.validate_explicit_worker(&content, phase).unwrap(); + } + for (field, phase) in [ + ("decode_worker_id", RequestPhase::Aggregated), + ("prefill_worker_id", RequestPhase::Prefill), + ] { + let mut content = request(); + content.routing = Some( + serde_json::from_value(serde_json::json!({ + "backend_instance_id": u64::MAX, field: 42 + })) + .unwrap(), + ); + let error = host.validate_explicit_worker(&content, phase).unwrap_err(); + assert_eq!( + error.downcast_ref::().unwrap().message(), + format!("nvext.{field}=42 does not identify a known worker") + ); + } + drop(host); + runtime.shutdown(); +} + +#[tokio::test] +#[serial_test::serial] +async fn explicit_worker_disappearing_after_preview_is_not_revalidated() { + let (host, runtime) = router(None).await; + let mut content = request(); + content.routing_mut().decode_worker_id = Some(7); + let request = Context::new(content); + let preview = host + .preview_kv_route(&request, RequestPhase::Decode) + .await + .unwrap(); + host.inner.client.override_discovered_instances(vec![]); + // A failure after successful selection must keep its service-error classification. + let error = match host.plan_kv_route_from_preview(&request, preview).await { + Ok(plan) => host.dispatch_kv_plan(request, plan).await.unwrap_err(), + Err(error) => error, + }; + assert!(!match_error_chain( + error.as_ref(), + &[ErrorType::InvalidArgument], + &[] + )); + drop(host); + runtime.shutdown(); +}