From 949e38e5d848b8f53e3f2fd2eb001160dda8aa40 Mon Sep 17 00:00:00 2001 From: "tzulingk@nvidia.com" Date: Wed, 10 Dec 2025 15:16:03 -0800 Subject: [PATCH 1/2] Fix health check race condition by waiting for watch stream discovery Signed-off-by: tzulingk@nvidia.com --- lib/runtime/src/health_check.rs | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/lib/runtime/src/health_check.rs b/lib/runtime/src/health_check.rs index a15cdda1bbbe..52d973da87ed 100644 --- a/lib/runtime/src/health_check.rs +++ b/lib/runtime/src/health_check.rs @@ -267,6 +267,37 @@ impl HealthCheckManager { .get_or_create_router(endpoint_subject, endpoint) .await?; + // Wait for watch stream to discover instances before checking + // This ensures the router's client has populated its instance list + // from etcd before we attempt to send the health check request. + // Without this, the first health check can fail due to a race condition + // where the watch stream hasn't completed its initial discovery yet. + match tokio::time::timeout( + Duration::from_secs(10), // 10 second timeout for discovery + router.client.wait_for_instances() + ).await { + Ok(Ok(instances)) => { + debug!( + "Health check for {}: watch stream ready, found {} instance(s)", + endpoint_subject, + instances.len() + ); + } + Ok(Err(e)) => { + return Err(anyhow::anyhow!( + "Failed to discover instances for {} during health check: {}", + endpoint_subject, + e + )); + } + Err(_) => { + return Err(anyhow::anyhow!( + "Timeout waiting for instance discovery for {} during health check", + endpoint_subject + )); + } + } + // Create the request context let request: SingleIn = Context::new(payload.clone()); From 1c6bac491fbd383d461551f90c9f127ebe51279d Mon Sep 17 00:00:00 2001 From: "tzulingk@nvidia.com" Date: Wed, 10 Dec 2025 15:23:53 -0800 Subject: [PATCH 2/2] format Signed-off-by: tzulingk@nvidia.com --- lib/runtime/src/health_check.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/runtime/src/health_check.rs b/lib/runtime/src/health_check.rs index 52d973da87ed..63a3f8ee040c 100644 --- a/lib/runtime/src/health_check.rs +++ b/lib/runtime/src/health_check.rs @@ -273,9 +273,11 @@ impl HealthCheckManager { // Without this, the first health check can fail due to a race condition // where the watch stream hasn't completed its initial discovery yet. match tokio::time::timeout( - Duration::from_secs(10), // 10 second timeout for discovery - router.client.wait_for_instances() - ).await { + Duration::from_secs(10), // 10 second timeout for discovery + router.client.wait_for_instances(), + ) + .await + { Ok(Ok(instances)) => { debug!( "Health check for {}: watch stream ready, found {} instance(s)",