Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 13 additions & 2 deletions bindings/golang/src/policy.rs
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,8 @@ impl Worker for GrpcWorker {
}

// FFI workers don't run the state machine — these counters are unused.
// The Go SDK manages worker health via direct set_healthy() calls.
// The Go SDK manages worker health via direct set_status() calls
// through the `sgl_multi_client_set_worker_health` FFI entry point.
Comment thread
slin1237 marked this conversation as resolved.
Resolved
fn consecutive_failures_increment(&self) -> usize {
0
}
Expand Down Expand Up @@ -444,7 +445,17 @@ pub unsafe extern "C" fn sgl_multi_client_set_worker_health(
if worker_index >= client.grpc_workers.len() {
return SglErrorCode::InvalidArgument;
}
client.grpc_workers[worker_index].set_healthy(healthy);
// The Go SDK is the source of truth for FFI worker health, so we
// map its boolean directly without the legacy `set_healthy` guard
// (which only demoted from `Ready`). FFI workers never run the
// local state machine, so a `Pending` start state can be flipped
// straight to `Ready` or `NotReady` here.
let status = if healthy {
WorkerStatus::Ready
} else {
WorkerStatus::NotReady
};
client.grpc_workers[worker_index].set_status(status);
SglErrorCode::Success
}

Expand Down
2 changes: 1 addition & 1 deletion model_gateway/benches/manual_policy_benchmark.rs
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ fn bench_failover(c: &mut Criterion) {
let policy = ManualPolicy::new();
let workers = create_workers(count);
let idx = select_with_key(&policy, &workers, "failover-test").unwrap();
workers[idx].set_healthy(false);
workers[idx].set_status(openai_protocol::worker::WorkerStatus::NotReady);
(policy, workers)
},
|(policy, workers)| {
Expand Down
11 changes: 6 additions & 5 deletions model_gateway/src/observability/metrics_ws/collectors.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,9 @@ pub struct CollectorConfig {
/// Interval for the metrics (Prometheus) collector.
pub metrics_interval: Duration,
/// Checkpoint interval for the worker collector.
/// Catches health changes that bypass the broadcast (e.g., `set_healthy()`
/// called directly by the health checker or circuit breaker).
/// Catches health changes that bypass the broadcast (e.g.,
/// `set_status()` called directly by FFI bindings, the registry
/// teardown path, or the mesh subscriber).
pub worker_checkpoint_interval: Duration,
}

Expand Down Expand Up @@ -80,9 +81,9 @@ pub fn start_collectors(
// ── Event-driven collector ──────────────────────────────────────────────

/// Listens on WorkerRegistry broadcast for instant push on register/remove.
/// Also polls on a checkpoint interval to catch health changes that bypass
/// the broadcast (e.g., `set_healthy()` called directly by health checker
/// or circuit breaker).
/// Also polls on a checkpoint interval to catch health changes that
/// bypass the broadcast (e.g., `set_status()` called directly by FFI
/// bindings, the registry teardown path, or the mesh subscriber).
fn spawn_worker_collector(
context: Arc<AppContext>,
registry: Arc<WatchRegistry>,
Expand Down
4 changes: 2 additions & 2 deletions model_gateway/src/policies/cache_aware.rs
Original file line number Diff line number Diff line change
Expand Up @@ -918,7 +918,7 @@ impl Default for CacheAwarePolicy {
#[cfg(test)]
mod tests {
use kv_index::{compute_content_hash, SequenceHash, StoredBlock, WorkerBlockMap};
use openai_protocol::worker::HealthCheckConfig;
use openai_protocol::worker::{HealthCheckConfig, WorkerStatus};

use super::*;
use crate::worker::{BasicWorkerBuilder, WorkerType};
Expand Down Expand Up @@ -1076,7 +1076,7 @@ mod tests {

// Remove a worker
policy.remove_worker_by_url("http://w1:8000");
workers[0].set_healthy(false);
workers[0].set_status(WorkerStatus::NotReady);

// All requests should now go to worker2
let idx = policy
Expand Down
12 changes: 6 additions & 6 deletions model_gateway/src/policies/consistent_hashing.rs
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@ impl LoadBalancingPolicy for ConsistentHashingPolicy {
mod tests {
use std::collections::HashMap;

use openai_protocol::worker::HealthCheckConfig;
use openai_protocol::worker::{HealthCheckConfig, WorkerStatus};

use super::*;
use crate::worker::{BasicWorkerBuilder, HashRing, WorkerType};
Expand Down Expand Up @@ -312,7 +312,7 @@ mod tests {
fn test_target_worker_miss_unhealthy() {
let policy = ConsistentHashingPolicy::new();
let workers = create_workers(&["http://w1:8000", "http://w2:8000"]);
workers[1].set_healthy(false);
workers[1].set_status(WorkerStatus::NotReady);

let headers = headers_with_target_worker(1);
let info = SelectWorkerInfo {
Expand Down Expand Up @@ -370,7 +370,7 @@ mod tests {
fn test_no_healthy_workers() {
let policy = ConsistentHashingPolicy::new();
let workers = create_workers(&["http://w1:8000"]);
workers[0].set_healthy(false);
workers[0].set_status(WorkerStatus::NotReady);

let headers = headers_with_routing_key("test");
let info = SelectWorkerInfo {
Expand Down Expand Up @@ -421,7 +421,7 @@ mod tests {
}

// Mark worker 1 as unhealthy
workers[1].set_healthy(false);
workers[1].set_status(WorkerStatus::NotReady);

// Record new routing and count how many keys moved
let mut moved_count = 0;
Expand Down Expand Up @@ -477,7 +477,7 @@ mod tests {
let original_idx = result.unwrap();

// Mark that worker unhealthy
workers[original_idx].set_healthy(false);
workers[original_idx].set_status(WorkerStatus::NotReady);

// Key should now route to a different healthy worker
let (failover_result, _) = policy.select_worker_impl(&workers, &info);
Expand All @@ -498,7 +498,7 @@ mod tests {
}

// Recover the original worker
workers[original_idx].set_healthy(true);
workers[original_idx].set_status(WorkerStatus::Ready);

// Key should route back to original worker
let (recovered_result, _) = policy.select_worker_impl(&workers, &info);
Expand Down
26 changes: 13 additions & 13 deletions model_gateway/src/policies/manual.rs
Original file line number Diff line number Diff line change
Expand Up @@ -299,7 +299,7 @@ fn min_group_select(workers: &[Arc<dyn Worker>], healthy_indices: &[usize]) -> u
mod tests {
use std::collections::HashMap;

use openai_protocol::worker::HealthCheckConfig;
use openai_protocol::worker::{HealthCheckConfig, WorkerStatus};

use super::*;
use crate::worker::{BasicWorkerBuilder, WorkerType};
Expand Down Expand Up @@ -402,7 +402,7 @@ mod tests {
let policy = ManualPolicy::new();
let workers = create_workers(&["http://w1:8000", "http://w2:8000"]);

workers[0].set_healthy(false);
workers[0].set_status(WorkerStatus::NotReady);

let headers = headers_with_routing_key("test-routing-id");
let info = SelectWorkerInfo {
Expand All @@ -426,7 +426,7 @@ mod tests {
let policy = ManualPolicy::new();
let workers = create_workers(&["http://w1:8000"]);

workers[0].set_healthy(false);
workers[0].set_status(WorkerStatus::NotReady);
let headers = headers_with_routing_key("test");
let info = SelectWorkerInfo {
headers: Some(&headers),
Expand Down Expand Up @@ -478,7 +478,7 @@ mod tests {
let first_idx = first_result.unwrap();
assert_eq!(branch, ExecutionBranch::Vacant);

workers[first_idx].set_healthy(false);
workers[first_idx].set_status(WorkerStatus::NotReady);

let (new_result, branch) = policy.select_worker_impl(&workers, &info);
let new_idx = new_result.unwrap();
Expand Down Expand Up @@ -547,14 +547,14 @@ mod tests {
let first_idx = first_result.unwrap();
assert_eq!(branch, ExecutionBranch::Vacant);

workers[first_idx].set_healthy(false);
workers[first_idx].set_status(WorkerStatus::NotReady);

let (second_result, branch) = policy.select_worker_impl(&workers, &info);
let second_idx = second_result.unwrap();
assert_ne!(second_idx, first_idx);
assert_eq!(branch, ExecutionBranch::OccupiedMiss);

workers[first_idx].set_healthy(true);
workers[first_idx].set_status(WorkerStatus::Ready);

let (after_recovery, branch) = policy.select_worker_impl(&workers, &info);
assert_eq!(
Expand All @@ -580,14 +580,14 @@ mod tests {
let first_idx = first_result.unwrap();
assert_eq!(branch, ExecutionBranch::Vacant);

workers[first_idx].set_healthy(false);
workers[first_idx].set_status(WorkerStatus::NotReady);

let (second_result, branch) = policy.select_worker_impl(&workers, &info);
let second_idx = second_result.unwrap();
assert_ne!(second_idx, first_idx);
assert_eq!(branch, ExecutionBranch::OccupiedMiss);

workers[second_idx].set_healthy(false);
workers[second_idx].set_status(WorkerStatus::NotReady);

let remaining_idx = (0..3).find(|&i| i != first_idx && i != second_idx).unwrap();
let (third_result, branch) = policy.select_worker_impl(&workers, &info);
Expand All @@ -598,7 +598,7 @@ mod tests {
);
assert_eq!(branch, ExecutionBranch::OccupiedMiss);

workers[first_idx].set_healthy(true);
workers[first_idx].set_status(WorkerStatus::Ready);

let (idx_after_restore, branch) = policy.select_worker_impl(&workers, &info);
assert_ne!(
Expand Down Expand Up @@ -654,17 +654,17 @@ mod tests {
"Should return first healthy worker in urls"
);

workers[0].set_healthy(false);
workers[0].set_status(WorkerStatus::NotReady);
let healthy_indices = vec![1, 2];
let result = find_healthy_worker(&urls, &workers, &healthy_indices);
assert_eq!(result, Some(1), "Should skip unhealthy and return next");

workers[1].set_healthy(false);
workers[1].set_status(WorkerStatus::NotReady);
let healthy_indices = vec![2];
let result = find_healthy_worker(&urls, &workers, &healthy_indices);
assert_eq!(result, Some(2), "Should return last healthy worker");

workers[2].set_healthy(false);
workers[2].set_status(WorkerStatus::NotReady);
let healthy_indices: Vec<usize> = vec![];
let result = find_healthy_worker(&urls, &workers, &healthy_indices);
assert_eq!(result, None, "Should return None when no healthy workers");
Expand Down Expand Up @@ -736,7 +736,7 @@ mod tests {
std::thread::sleep(std::time::Duration::from_millis(10));

// OccupiedMiss: worker becomes unhealthy
workers[first_idx].set_healthy(false);
workers[first_idx].set_status(WorkerStatus::NotReady);
let (_, branch) = policy.select_worker_impl(&workers, &info);
assert_eq!(branch, ExecutionBranch::OccupiedMiss);
let access_after_miss = policy.routing_map.get(&routing_id).unwrap().last_access;
Expand Down
4 changes: 2 additions & 2 deletions model_gateway/src/policies/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ pub struct SelectWorkerInfo<'a> {

#[cfg(test)]
mod tests {
use openai_protocol::worker::HealthCheckConfig;
use openai_protocol::worker::{HealthCheckConfig, WorkerStatus};

use super::*;
use crate::worker::{BasicWorkerBuilder, WorkerType};
Expand Down Expand Up @@ -225,7 +225,7 @@ mod tests {
assert_eq!(indices, vec![0, 1, 2]);

// Mark one unhealthy
workers[1].set_healthy(false);
workers[1].set_status(WorkerStatus::NotReady);
let indices = get_healthy_worker_indices(&workers);
assert_eq!(indices, vec![0, 2]);
}
Expand Down
4 changes: 2 additions & 2 deletions model_gateway/src/policies/prefix_hash.rs
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,7 @@ impl LoadBalancingPolicy for PrefixHashPolicy {

#[cfg(test)]
mod tests {
use openai_protocol::worker::HealthCheckConfig;
use openai_protocol::worker::{HealthCheckConfig, WorkerStatus};

use super::*;
use crate::worker::{BasicWorkerBuilder, HashRing, WorkerType};
Expand Down Expand Up @@ -375,7 +375,7 @@ mod tests {
fn test_no_healthy_workers() {
let policy = PrefixHashPolicy::with_defaults();
let workers = create_workers(&["http://w1:8000"]);
workers[0].set_healthy(false);
workers[0].set_status(WorkerStatus::NotReady);

let ring = Arc::new(HashRing::new(&workers));
let tokens: Vec<u32> = vec![1, 2, 3];
Expand Down
6 changes: 3 additions & 3 deletions model_gateway/src/policies/random.rs
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ impl LoadBalancingPolicy for RandomPolicy {
mod tests {
use std::collections::HashMap;

use openai_protocol::worker::HealthCheckConfig;
use openai_protocol::worker::{HealthCheckConfig, WorkerStatus};

use super::*;
use crate::worker::{BasicWorkerBuilder, WorkerType};
Expand Down Expand Up @@ -117,7 +117,7 @@ mod tests {
];

// Mark first worker as unhealthy
workers[0].set_healthy(false);
workers[0].set_status(WorkerStatus::NotReady);

// Should always select the healthy worker (index 1)
for _ in 0..10 {
Expand All @@ -138,7 +138,7 @@ mod tests {
.build(),
)];

workers[0].set_healthy(false);
workers[0].set_status(WorkerStatus::NotReady);
assert_eq!(
policy.select_worker(&workers, &SelectWorkerInfo::default()),
None
Expand Down
4 changes: 2 additions & 2 deletions model_gateway/src/policies/round_robin.rs
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ impl LoadBalancingPolicy for RoundRobinPolicy {

#[cfg(test)]
mod tests {
use openai_protocol::worker::HealthCheckConfig;
use openai_protocol::worker::{HealthCheckConfig, WorkerStatus};

use super::*;
use crate::worker::{BasicWorkerBuilder, WorkerType};
Expand Down Expand Up @@ -128,7 +128,7 @@ mod tests {
];

// Mark middle worker as unhealthy
workers[1].set_healthy(false);
workers[1].set_status(WorkerStatus::NotReady);

// Should skip unhealthy worker: 0, 2, 0, 2, ...
let info = SelectWorkerInfo::default();
Expand Down
7 changes: 6 additions & 1 deletion model_gateway/src/routers/http/pd_router.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1435,7 +1435,12 @@ mod tests {
let worker = BasicWorkerBuilder::new(url)
.worker_type(worker_type)
.build();
worker.set_healthy(healthy);
let status = if healthy {
openai_protocol::worker::WorkerStatus::Ready
} else {
openai_protocol::worker::WorkerStatus::NotReady
};
worker.set_status(status);
Box::new(worker)
}

Expand Down
2 changes: 1 addition & 1 deletion model_gateway/src/routers/http/router.rs
Original file line number Diff line number Diff line change
Expand Up @@ -823,7 +823,7 @@ mod tests {
fn create_test_unhealthy_router() -> Router {
let router = create_test_regular_router();
let workers = router.worker_registry.get_all();
workers[0].set_healthy(false);
workers[0].set_status(openai_protocol::worker::WorkerStatus::NotReady);
router
}

Expand Down
Loading
Loading