diff --git a/mesh/src/controller.rs b/mesh/src/controller.rs index eeb54e4d9a..909d3072e9 100644 --- a/mesh/src/controller.rs +++ b/mesh/src/controller.rs @@ -366,7 +366,6 @@ impl MeshController { let incremental_sender_handle = { use super::{ incremental::IncrementalUpdateCollector, service::gossip::IncrementalUpdate, - stores::StoreType as LocalStoreType, }; let collector = Arc::new(IncrementalUpdateCollector::new( @@ -390,23 +389,7 @@ impl MeshController { if !all_updates.is_empty() { for (store_type, updates) in all_updates { - let proto_store_type = match store_type { - LocalStoreType::Membership => { - super::service::gossip::StoreType::Membership as i32 - } - LocalStoreType::App => { - super::service::gossip::StoreType::App as i32 - } - LocalStoreType::Worker => { - super::service::gossip::StoreType::Worker as i32 - } - LocalStoreType::Policy => { - super::service::gossip::StoreType::Policy as i32 - } - LocalStoreType::RateLimit => { - super::service::gossip::StoreType::RateLimit as i32 - } - }; + let proto_store_type = store_type.to_proto(); let incremental_update = StreamMessage { message_type: StreamMessageType::IncrementalUpdate as i32, @@ -499,7 +482,6 @@ impl MeshController { if let Err(err) = stores.app.insert( app_state.key.clone(), app_state, - state_update.actor.clone(), ) { log::warn!(error = %err, "Failed to apply app state update"); } @@ -515,7 +497,6 @@ impl MeshController { if let Err(err) = stores.membership.insert( membership_state.name.clone(), membership_state, - state_update.actor.clone(), ) { log::warn!(error = %err, "Failed to apply membership state update"); } diff --git a/mesh/src/crdt_kv/crdt.rs b/mesh/src/crdt_kv/crdt.rs index 4b78a65e48..e29bf3b035 100644 --- a/mesh/src/crdt_kv/crdt.rs +++ b/mesh/src/crdt_kv/crdt.rs @@ -17,16 +17,14 @@ use super::{ /// Value metadata for CRDT OR-Map #[derive(Debug, Clone, PartialEq, Eq)] struct ValueMetadata { - value: Vec, timestamp: u64, replica_id: ReplicaId, is_tombstone: bool, // Marks if this version is a tombstone (deletion) } impl ValueMetadata { - fn new(value: Vec, timestamp: u64, replica_id: ReplicaId) -> Self { + fn new(timestamp: u64, replica_id: ReplicaId) -> Self { Self { - value, timestamp, replica_id, is_tombstone: false, @@ -35,7 +33,6 @@ impl ValueMetadata { fn tombstone(timestamp: u64, replica_id: ReplicaId) -> Self { Self { - value: Vec::new(), timestamp, replica_id, is_tombstone: true, @@ -119,7 +116,7 @@ impl CrdtOrMap { let key_guard = key_lock.lock(); let timestamp = self.clock.tick(); - let result = if self.record_insert_metadata(&key, &value, timestamp, self.replica_id) { + let result = if self.record_insert_metadata(&key, timestamp, self.replica_id) { let mut prev = None; let value_for_operation = value.clone(); let _ = self.store.upsert(key.clone(), |current| { @@ -158,22 +155,21 @@ impl CrdtOrMap { let updated_value = updater(current_value.as_deref()); let timestamp = self.clock.tick(); - let result = - if self.record_insert_metadata(&key, &updated_value, timestamp, self.replica_id) { - let operation = Operation::insert( - key.clone(), - updated_value.clone(), - timestamp, - self.replica_id, - ); + let result = if self.record_insert_metadata(&key, timestamp, self.replica_id) { + let operation = Operation::insert( + key.clone(), + updated_value.clone(), + timestamp, + self.replica_id, + ); - self.store.insert(key.clone(), updated_value.clone()); - self.operation_log.write().append(operation); + self.store.insert(key.clone(), updated_value.clone()); + self.operation_log.write().append(operation); - updated_value - } else { - self.store.get(&key).unwrap_or_default() - }; + updated_value + } else { + self.store.get(&key).unwrap_or_default() + }; drop(key_guard); self.try_cleanup_key_lock(&key, &key_lock); @@ -199,22 +195,21 @@ impl CrdtOrMap { }; let timestamp = self.clock.tick(); - let result = - if self.record_insert_metadata(&key, &updated_value, timestamp, self.replica_id) { - let operation = Operation::insert( - key.clone(), - updated_value.clone(), - timestamp, - self.replica_id, - ); + let result = if self.record_insert_metadata(&key, timestamp, self.replica_id) { + let operation = Operation::insert( + key.clone(), + updated_value.clone(), + timestamp, + self.replica_id, + ); - self.store.insert(key.clone(), updated_value.clone()); - self.operation_log.write().append(operation); + self.store.insert(key.clone(), updated_value.clone()); + self.operation_log.write().append(operation); - updated_value - } else { - self.store.get(&key).unwrap_or_default() - }; + updated_value + } else { + self.store.get(&key).unwrap_or_default() + }; drop(key_guard); self.try_cleanup_key_lock(&key, &key_lock); @@ -241,7 +236,7 @@ impl CrdtOrMap { let (result, changed) = if let Some(updated_value) = maybe_updated_value { let timestamp = self.clock.tick(); - if self.record_insert_metadata(&key, &updated_value, timestamp, self.replica_id) { + if self.record_insert_metadata(&key, timestamp, self.replica_id) { let operation = Operation::insert( key.clone(), updated_value.clone(), @@ -402,7 +397,7 @@ impl CrdtOrMap { let key_lock = self.key_lock_for(key); let key_guard = key_lock.lock(); - if self.record_insert_metadata(key, &value, timestamp, replica_id) { + if self.record_insert_metadata(key, timestamp, replica_id) { self.store.insert(key.to_string(), value); } @@ -421,14 +416,8 @@ impl CrdtOrMap { } } - fn record_insert_metadata( - &self, - key: &str, - value: &[u8], - timestamp: u64, - replica_id: ReplicaId, - ) -> bool { - let new_metadata = ValueMetadata::new(value.to_vec(), timestamp, replica_id); + fn record_insert_metadata(&self, key: &str, timestamp: u64, replica_id: ReplicaId) -> bool { + let new_metadata = ValueMetadata::new(timestamp, replica_id); match self.metadata.entry(key.to_string()) { MapEntry::Occupied(mut entry) => { diff --git a/mesh/src/crdt_kv/operation.rs b/mesh/src/crdt_kv/operation.rs index f4413e75a6..004308e9eb 100644 --- a/mesh/src/crdt_kv/operation.rs +++ b/mesh/src/crdt_kv/operation.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeMap, HashMap, HashSet}; +use std::collections::{HashMap, HashSet}; use serde::{Deserialize, Serialize}; @@ -111,16 +111,6 @@ impl OperationLog { &self.operations } - /// Serialize to JSON - pub fn to_json(&self) -> Result { - serde_json::to_string(self) - } - - /// Deserialize from JSON - pub fn from_json(json: &str) -> Result { - serde_json::from_str(json) - } - /// Serialize to JSON bytes pub fn to_bytes(&self) -> Result, serde_json::Error> { serde_json::to_vec(self) @@ -141,8 +131,8 @@ impl OperationLog { self.operations.is_empty() } - fn latest_operations_by_key(&self) -> BTreeMap { - let mut latest_by_key: BTreeMap = BTreeMap::new(); + fn latest_operations_by_key(&self) -> HashMap { + let mut latest_by_key: HashMap = HashMap::new(); for operation in &self.operations { let key = operation.key().to_string(); @@ -182,7 +172,7 @@ impl OperationLog { } /// Build a latest-state snapshot and clear the operation log. - pub fn snapshot_and_truncate(&mut self) -> BTreeMap { + pub fn snapshot_and_truncate(&mut self) -> HashMap { let snapshot = self.latest_operations_by_key(); self.operations.clear(); snapshot diff --git a/mesh/src/crdt_kv/replica.rs b/mesh/src/crdt_kv/replica.rs index 804cec1bac..55cf493211 100644 --- a/mesh/src/crdt_kv/replica.rs +++ b/mesh/src/crdt_kv/replica.rs @@ -48,7 +48,7 @@ pub struct LamportClock { impl Clone for LamportClock { fn clone(&self) -> Self { Self { - counter: AtomicU64::new(self.counter.load(Ordering::SeqCst)), + counter: AtomicU64::new(self.counter.load(Ordering::Acquire)), } } } @@ -63,14 +63,14 @@ impl LamportClock { /// Increment and return new timestamp pub fn tick(&self) -> u64 { - let mut current = self.counter.load(Ordering::SeqCst); + let mut current = self.counter.load(Ordering::Acquire); loop { let new_value = current.saturating_add(1); match self.counter.compare_exchange( current, new_value, - Ordering::SeqCst, - Ordering::SeqCst, + Ordering::AcqRel, + Ordering::Acquire, ) { Ok(_) => return new_value, Err(actual) => current = actual, @@ -80,14 +80,14 @@ impl LamportClock { /// Update clock to max(local, remote) + 1 pub fn update(&self, remote_timestamp: u64) -> u64 { - let mut current = self.counter.load(Ordering::SeqCst); + let mut current = self.counter.load(Ordering::Acquire); loop { let new_value = current.max(remote_timestamp).saturating_add(1); match self.counter.compare_exchange( current, new_value, - Ordering::SeqCst, - Ordering::SeqCst, + Ordering::AcqRel, + Ordering::Acquire, ) { Ok(_) => return new_value, Err(actual) => current = actual, diff --git a/mesh/src/crdt_kv/tests.rs b/mesh/src/crdt_kv/tests.rs index 891ce790c6..7892c3660a 100644 --- a/mesh/src/crdt_kv/tests.rs +++ b/mesh/src/crdt_kv/tests.rs @@ -234,11 +234,11 @@ fn test_operation_log_json_serialization() { let log = map.get_operation_log(); - // Serialize to JSON - let json = log.to_json().unwrap(); + // Serialize to bytes + let bytes = log.to_bytes().unwrap(); // Deserialize - let deserialized_log = OperationLog::from_json(&json).unwrap(); + let deserialized_log = OperationLog::from_bytes(&bytes).unwrap(); assert_eq!(log.len(), deserialized_log.len()); } diff --git a/mesh/src/incremental.rs b/mesh/src/incremental.rs index fe27a79deb..d24cd80534 100644 --- a/mesh/src/incremental.rs +++ b/mesh/src/incremental.rs @@ -91,7 +91,7 @@ impl IncrementalUpdateCollector { fn collect_serializable_updates( &self, all_items: std::collections::BTreeMap, - last_sent_map: &mut HashMap, + last_sent_map: &HashMap, store_name: &str, get_id: impl Fn(&S) -> String, ) -> Vec @@ -129,14 +129,14 @@ impl IncrementalUpdateCollector { /// Collect incremental updates for a specific store type pub fn collect_updates_for_store(&self, store_type: StoreType) -> Vec { let mut updates = Vec::new(); - let mut last_sent = self.last_sent.write(); + let last_sent = self.last_sent.read(); match store_type { StoreType::Worker => { let all_workers = self.stores.worker.all(); updates = self.collect_serializable_updates( all_workers, - &mut last_sent.worker, + &last_sent.worker, "worker", |state: &WorkerState| state.worker_id.clone(), ); @@ -145,7 +145,7 @@ impl IncrementalUpdateCollector { let all_policies = self.stores.policy.all(); updates = self.collect_serializable_updates( all_policies, - &mut last_sent.policy, + &last_sent.policy, "policy", |state: &PolicyState| state.model_id.clone(), ); @@ -154,7 +154,7 @@ impl IncrementalUpdateCollector { let all_apps = self.stores.app.all(); updates = self.collect_serializable_updates( all_apps, - &mut last_sent.app, + &last_sent.app, "app", |state: &AppState| state.key.clone(), ); @@ -163,7 +163,7 @@ impl IncrementalUpdateCollector { let all_members = self.stores.membership.all(); updates = self.collect_serializable_updates( all_members, - &mut last_sent.membership, + &last_sent.membership, "membership", |state: &MembershipState| state.name.clone(), ); @@ -288,7 +288,7 @@ mod tests { load: 0.5, version: 1, }; - let _ = stores.worker.insert(key, worker_state, "node1".to_string()); + let _ = stores.worker.insert(key, worker_state); // Collect updates let updates = collector.collect_updates_for_store(StoreType::Worker); @@ -316,9 +316,7 @@ mod tests { load: 0.8, version: 2, }; - let _ = stores - .worker - .insert(key2, worker_state2, "node1".to_string()); + let _ = stores.worker.insert(key2, worker_state2); // Should collect new version let updates3 = collector.collect_updates_for_store(StoreType::Worker); @@ -338,7 +336,7 @@ mod tests { config: b"config_data".to_vec(), version: 1, }; - let _ = stores.policy.insert(key, policy_state, "node1".to_string()); + let _ = stores.policy.insert(key, policy_state); let updates = collector.collect_updates_for_store(StoreType::Policy); assert_eq!(updates.len(), 1); @@ -356,7 +354,7 @@ mod tests { value: b"app_value".to_vec(), version: 1, }; - let _ = stores.app.insert(key, app_state, "node1".to_string()); + let _ = stores.app.insert(key, app_state); let updates = collector.collect_updates_for_store(StoreType::App); assert_eq!(updates.len(), 1); @@ -376,9 +374,7 @@ mod tests { version: 1, metadata: std::collections::BTreeMap::new(), }; - let _ = stores - .membership - .insert(key, membership_state, "node1".to_string()); + let _ = stores.membership.insert(key, membership_state); let updates = collector.collect_updates_for_store(StoreType::Membership); assert_eq!(updates.len(), 1); @@ -402,7 +398,6 @@ mod tests { load: 0.5, version: 1, }, - "node1".to_string(), ); let policy_key = "policy:model1".to_string(); @@ -414,7 +409,6 @@ mod tests { config: vec![], version: 1, }, - "node1".to_string(), ); let all_updates = collector.collect_all_updates(); @@ -438,7 +432,6 @@ mod tests { load: 0.5, version: 1, }, - "node1".to_string(), ); let updates = collector.collect_updates_for_store(StoreType::Worker); @@ -503,7 +496,6 @@ mod tests { load: 0.5, version: 1, }, - "node1".to_string(), ); let updates1 = collector.collect_updates_for_store(StoreType::Worker); @@ -522,7 +514,6 @@ mod tests { load: 0.8, version: 2, }, - "node1".to_string(), ); let updates2 = collector.collect_updates_for_store(StoreType::Worker); @@ -541,7 +532,6 @@ mod tests { load: 0.3, version: 3, }, - "node1".to_string(), ); let updates3 = collector.collect_updates_for_store(StoreType::Worker); diff --git a/mesh/src/node_state_machine.rs b/mesh/src/node_state_machine.rs index 55fd9e3868..f99a36fc6c 100644 --- a/mesh/src/node_state_machine.rs +++ b/mesh/src/node_state_machine.rs @@ -384,7 +384,6 @@ mod tests { version: 1, metadata: Default::default(), }, - "test".to_string(), ); let _ = stores.worker.insert( @@ -397,7 +396,6 @@ mod tests { load: 0.5, version: 1, }, - "test".to_string(), ); let _ = stores.policy.insert( @@ -408,7 +406,6 @@ mod tests { config: vec![], version: 1, }, - "test".to_string(), ); // Now should not need snapshot @@ -482,7 +479,6 @@ mod tests { value: vec![1, 2, 3], version: 1, }, - "test".to_string(), ); // Record update with changed state @@ -533,7 +529,6 @@ mod tests { value: vec![], version: 1, }, - "test".to_string(), ); // Hash should change diff --git a/mesh/src/ping_server.rs b/mesh/src/ping_server.rs index ac88799122..43986734b1 100644 --- a/mesh/src/ping_server.rs +++ b/mesh/src/ping_server.rs @@ -9,7 +9,10 @@ use anyhow::Result; use futures::Stream; use tokio::sync::mpsc; use tokio_stream::StreamExt; -use tonic::{transport::Server, Response, Status}; +use tonic::{ + transport::{server::TcpIncoming, Server}, + Response, Status, +}; use tracing as log; use tracing::instrument; @@ -69,13 +72,7 @@ impl GossipService { } }; - let proto_store_type = match store_type { - LocalStoreType::Membership => gossip::StoreType::Membership as i32, - LocalStoreType::App => gossip::StoreType::App as i32, - LocalStoreType::Worker => gossip::StoreType::Worker as i32, - LocalStoreType::Policy => gossip::StoreType::Policy as i32, - LocalStoreType::RateLimit => gossip::StoreType::RateLimit as i32, - }; + let proto_store_type = store_type.to_proto(); // Get all entries from the store let entries: Vec<(String, Vec)> = match store_type { @@ -293,6 +290,20 @@ impl GossipService { Ok(()) } + pub async fn serve_ping_with_listener>( + self, + listener: tokio::net::TcpListener, + signal: F, + ) -> Result<()> { + let incoming = TcpIncoming::from(listener); + let service = GossipServer::new(self); + Server::builder() + .add_service(service) + .serve_with_incoming_shutdown(incoming, signal) + .await?; + Ok(()) + } + fn merge_state(&self, incoming_nodes: Vec) -> bool { let mut state = self.state.write(); let mut updated = false; @@ -404,13 +415,7 @@ impl Gossip for GossipService { if !all_updates.is_empty() { for (store_type, updates) in all_updates { - let proto_store_type = match store_type { - LocalStoreType::Membership => gossip::StoreType::Membership as i32, - LocalStoreType::App => gossip::StoreType::App as i32, - LocalStoreType::Worker => gossip::StoreType::Worker as i32, - LocalStoreType::Policy => gossip::StoreType::Policy as i32, - LocalStoreType::RateLimit => gossip::StoreType::RateLimit as i32, - }; + let proto_store_type = store_type.to_proto(); sequence_counter += 1; let batch_size: usize = updates.iter().map(|u| u.value.len()).sum(); @@ -516,13 +521,7 @@ impl Gossip for GossipService { store_type, peer_id ); - let proto_store_type = match store_type { - LocalStoreType::Membership => gossip::StoreType::Membership as i32, - LocalStoreType::App => gossip::StoreType::App as i32, - LocalStoreType::Worker => gossip::StoreType::Worker as i32, - LocalStoreType::Policy => gossip::StoreType::Policy as i32, - LocalStoreType::RateLimit => gossip::StoreType::RateLimit as i32, - }; + let proto_store_type = store_type.to_proto(); let snapshot_request = StreamMessage { message_type: StreamMessageType::SnapshotRequest as i32, @@ -661,7 +660,6 @@ impl Gossip for GossipService { if let Err(err) = stores.app.insert( app_state.key.clone(), app_state, - state_update.actor.clone(), ) { log::warn!(error = %err, "Failed to apply app state update"); } @@ -683,7 +681,6 @@ impl Gossip for GossipService { stores.membership.insert( membership_state.name.clone(), membership_state, - state_update.actor.clone(), ) { log::warn!(error = %err, "Failed to apply membership state update"); @@ -914,17 +911,17 @@ impl Gossip for GossipService { match store_type { LocalStoreType::Membership => { if let Ok(membership_state) = serde_json::from_slice::(&entry.value) { - let _ = stores.membership.insert(key, membership_state, entry.actor.clone()); + let _ = stores.membership.insert(key, membership_state); } } LocalStoreType::App => { if let Ok(app_state) = serde_json::from_slice::(&entry.value) { - let _ = stores.app.insert(key, app_state, entry.actor.clone()); + let _ = stores.app.insert(key, app_state); } } LocalStoreType::Worker => { if let Ok(worker_state) = serde_json::from_slice::(&entry.value) { - let _ = stores.worker.insert(key, worker_state.clone(), entry.actor.clone()); + let _ = stores.worker.insert(key, worker_state.clone()); // Also update sync manager if available if let Some(ref sync_manager) = sync_manager { sync_manager.apply_remote_worker_state(worker_state, Some(entry.actor.clone())); @@ -933,7 +930,7 @@ impl Gossip for GossipService { } LocalStoreType::Policy => { if let Ok(policy_state) = serde_json::from_slice::(&entry.value) { - let _ = stores.policy.insert(key, policy_state.clone(), entry.actor.clone()); + let _ = stores.policy.insert(key, policy_state.clone()); // Also update sync manager if available if let Some(ref sync_manager) = sync_manager { // Check if this is a tree state update diff --git a/mesh/src/rate_limit_window.rs b/mesh/src/rate_limit_window.rs index 0b4341d148..9a4ed9782a 100644 --- a/mesh/src/rate_limit_window.rs +++ b/mesh/src/rate_limit_window.rs @@ -144,7 +144,6 @@ mod tests { value: serialized, version: 1, }, - "node1".to_string(), ); // Increment counter @@ -209,9 +208,7 @@ mod tests { version: 1, metadata: Default::default(), }; - let _ = stores - .membership - .insert(membership_key, membership_state, "test_node".to_string()); + let _ = stores.membership.insert(membership_key, membership_state); // Update rate limit membership so this node becomes an owner sync_manager.update_rate_limit_membership(); diff --git a/mesh/src/service.rs b/mesh/src/service.rs index 185bc55377..8f043d99ad 100644 --- a/mesh/src/service.rs +++ b/mesh/src/service.rs @@ -231,7 +231,7 @@ impl MeshServerHandler { }; self.stores .app - .insert(key.clone(), app_state, self.self_name.clone()) + .insert(key.clone(), app_state) .map_err(|err| anyhow::anyhow!("Failed to persist app state for key {key}: {err}"))?; node.metadata.insert(key, value); @@ -379,6 +379,24 @@ impl MeshServer { } pub async fn start(self) -> Result<()> { + self.start_inner(None).await + } + + pub async fn start_with_listener(self, listener: tokio::net::TcpListener) -> Result<()> { + let bound_addr = listener + .local_addr() + .map_err(|e| anyhow::anyhow!("Failed to read listener local addr: {e}"))?; + if bound_addr != self.self_addr { + return Err(anyhow::anyhow!( + "Listener/self_addr mismatch: listener={}, self_addr={}", + bound_addr, + self.self_addr + )); + } + self.start_inner(Some(listener)).await + } + + async fn start_inner(self, listener: Option) -> Result<()> { log::info!("Mesh server listening on {}", self.self_addr); let self_name = self.self_name.clone(); let self_address = self.self_addr; @@ -412,9 +430,15 @@ impl MeshServer { clippy::disallowed_methods, reason = "handle is awaited immediately below via tokio::select!, bounded by shutdown signal" )] - let listener = tokio::spawn(service.serve_ping_with_shutdown(async move { - _ = service_shutdown.changed().await; - })); + let server_handle = if let Some(tcp_listener) = listener { + tokio::spawn(service.serve_ping_with_listener(tcp_listener, async move { + _ = service_shutdown.changed().await; + })) + } else { + tokio::spawn(service.serve_ping_with_shutdown(async move { + _ = service_shutdown.changed().await; + })) + }; tokio::time::sleep(Duration::from_secs(1)).await; #[expect( clippy::disallowed_methods, @@ -423,7 +447,7 @@ impl MeshServer { let app_handle = tokio::spawn(controller.event_loop(self.signal_rx.clone())); tokio::select! { - res = listener => res??, + res = server_handle => res??, res = app_handle => res??, } @@ -601,19 +625,34 @@ macro_rules! mesh_run { }); handler }}; + + ($name:expr, $listener:expr, $addr:expr, $init_peer:expr) => {{ + tracing::info!("Starting mesh server : {}", $addr); + use $crate::MeshServerBuilder; + let (server, handler) = + MeshServerBuilder::new($name.to_string(), $addr, $init_peer).build(); + #[expect(clippy::disallowed_methods, reason = "test macro: spawned server runs for the test lifetime and handler is returned for assertions")] + tokio::spawn(async move { + if let Err(e) = server.start_with_listener($listener).await { + tracing::error!("Mesh server failed: {}", e); + } + }); + handler + }}; } #[cfg(test)] mod tests { use std::sync::Once; - use tokio::net::TcpListener; use tracing as log; use tracing_subscriber::{ filter::LevelFilter, layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, }; use super::*; + use crate::tests::test_utils::{bind_node, wait_for}; + static INIT: Once = Once::new(); fn init() { INIT.call_once(|| { @@ -627,100 +666,82 @@ mod tests { .try_init(); }); } - async fn find_free_port() -> (TcpListener, u16) { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let port = listener.local_addr().unwrap().port(); - log::info!("Found free port: {}", port); - (listener, port) - } - - async fn get_node() -> SocketAddr { - let (_listener, port) = find_free_port().await; - format!("127.0.0.1:{port}").parse().unwrap() - } - - fn print_state(handler: &MeshServerHandler) -> String { - let state = handler.state.read(); - let mut res = vec![]; - for (k, v) in state.iter() { - res.push(format!( - "{}: {:?} - {:?}", - k, - NodeStatus::try_from(v.status).unwrap(), - v.metadata - )); - } - res.join(", ") - } #[tokio::test] - #[ignore = "reason: only run manually due to flakiness"] + #[ignore = "SWIM failure detection for hard-shutdown nodes needs many gossip rounds; flaky under parallel CI load"] async fn test_state_synchronization() { init(); log::info!("Starting test_state_synchronization"); - // 1. setup node A and B for initial cluster - let addr_a = get_node().await; - let handler_a = mesh_run!("A", addr_a, None); - let addr_b = get_node().await; - let handler_b = mesh_run!("B", addr_b, Some(addr_a)); + // 1. Setup A-B cluster + let (listener_a, addr_a) = bind_node().await; + let handler_a = mesh_run!("A", listener_a, addr_a, None); + let (listener_b, addr_b) = bind_node().await; + let handler_b = mesh_run!("B", listener_b, addr_b, Some(addr_a)); + + wait_for( + || handler_a.state.read().len() == 2, + Duration::from_secs(15), + "A-B cluster formed", + ) + .await; - // 2. wait for node A and B to sync and write some data - tokio::time::sleep(Duration::from_secs(2)).await; handler_a .write_data("hello".into(), "world".into()) .unwrap(); - log::info!("================================================"); - // 3. add node C and D and wait for them to sync - let addr_c = get_node().await; - let handler_c = mesh_run!("C", addr_c, Some(addr_a)); - let addr_d = get_node().await; - let handler_d = mesh_run!("D", addr_d, Some(addr_c)); - tokio::time::sleep(Duration::from_secs(2)).await; - log::info!("================================================"); + // 2. Add C and D + let (listener_c, addr_c) = bind_node().await; + let handler_c = mesh_run!("C", listener_c, addr_c, Some(addr_a)); + let (listener_d, addr_d) = bind_node().await; + let handler_d = mesh_run!("D", listener_d, addr_d, Some(addr_c)); + + wait_for( + || handler_a.state.read().len() == 4, + Duration::from_secs(30), + "4-node cluster formed", + ) + .await; - // 4. add node E and wait for it to sync and kill it + // 3. Add E, let it join, then kill it { - let addr_e = get_node().await; - let handler_e = mesh_run!("E", addr_e, Some(addr_d)); - tokio::time::sleep(Duration::from_secs(3)).await; - log::info!("State E: {:?}", print_state(&handler_e)); - // killing_button.send(()).unwrap(); + let (listener_e, addr_e) = bind_node().await; + let handler_e = mesh_run!("E", listener_e, addr_e, Some(addr_d)); + + wait_for( + || handler_a.state.read().len() == 5, + Duration::from_secs(30), + "E joined cluster", + ) + .await; + handler_e.shutdown(); } + // 4. Gracefully shutdown D handler_d.graceful_shutdown().await.unwrap(); - tokio::time::sleep(Duration::from_secs(2)).await; - log::info!("================================================"); - - // 5. Poll until expected state is reached (with timeout) - // Note: D stays in Leaving status because gracefully shutdown nodes are not pinged anymore (by design) - // E transitions Alive → Suspected → Down because it was abruptly shutdown - let final_state = String::from("A: Alive - {\"hello\": [119, 111, 114, 108, 100]}, B: Alive - {}, C: Alive - {}, D: Leaving - {}, E: Down - {}"); - - let max_wait = Duration::from_secs(30); - let poll_interval = Duration::from_millis(500); - let start = std::time::Instant::now(); - - loop { - let state_a = print_state(&handler_a); - let state_b = print_state(&handler_b); - let state_c = print_state(&handler_c); - - if state_a == final_state && state_b == final_state && state_c == final_state { - log::info!("All nodes converged to expected state"); - break; - } - if start.elapsed() > max_wait { - log::info!("================================================"); - panic!( - "Timeout waiting for state convergence.\nExpected: {final_state:?}\nState A: {state_a:?}\nState B: {state_b:?}\nState C: {state_c:?}" - ); - } + // 5. Wait for D=Leaving and E=Down (not Alive) on all remaining nodes + let check_statuses = |handler: &MeshServerHandler| { + let state = handler.state.read(); + let d_leaving = state + .get("D") + .is_some_and(|n| n.status == NodeStatus::Leaving as i32); + let e_not_alive = state + .get("E") + .is_some_and(|n| n.status != NodeStatus::Alive as i32); + d_leaving && e_not_alive + }; - tokio::time::sleep(poll_interval).await; + for (handler, name) in [(&handler_a, "A"), (&handler_b, "B"), (&handler_c, "C")] { + wait_for( + || check_statuses(handler), + Duration::from_secs(60), + &format!("D=Leaving, E not Alive on node {name}"), + ) + .await; } + + log::info!("All nodes converged to expected state"); } } diff --git a/mesh/src/stores.rs b/mesh/src/stores.rs index 33edb2572b..61d58c7daf 100644 --- a/mesh/src/stores.rs +++ b/mesh/src/stores.rs @@ -219,6 +219,18 @@ impl StoreType { } } + /// Convert to proto StoreType (i32) + pub fn to_proto(self) -> i32 { + use super::service::gossip::StoreType as ProtoStoreType; + match self { + StoreType::Membership => ProtoStoreType::Membership as i32, + StoreType::App => ProtoStoreType::App as i32, + StoreType::Worker => ProtoStoreType::Worker as i32, + StoreType::Policy => ProtoStoreType::Policy as i32, + StoreType::RateLimit => ProtoStoreType::RateLimit as i32, + } + } + /// Convert from proto StoreType (i32) to local StoreType pub fn from_proto(proto_value: i32) -> Self { match proto_value { @@ -303,6 +315,11 @@ pub struct PolicyState { pub version: u64, } +/// Helper function to get policy state key for a model +pub fn policy_key(model_id: &str) -> String { + format!("policy:{model_id}") +} + /// Helper function to get tree state key for a model pub fn tree_state_key(model_id: &str) -> String { format!("tree:{model_id}") @@ -326,12 +343,10 @@ macro_rules! define_state_store { self.inner.get(key) } - // Backward-compatible actor parameter; the CRDT OR-Map no longer needs it. pub fn insert( &self, key: String, value: $value_type, - _actor: String, ) -> Result, serde_json::Error> { self.inner.insert(key, value) } @@ -714,7 +729,7 @@ mod tests { metadata: BTreeMap::new(), }; - let _ = store.insert(key.clone(), state.clone(), "node1".to_string()); + let _ = store.insert(key.clone(), state.clone()); assert_eq!(store.get(&key).unwrap().name, "node1"); store.remove(&key); @@ -731,7 +746,7 @@ mod tests { version: 1, }; - let _ = store.insert(key.clone(), state.clone(), "node1".to_string()); + let _ = store.insert(key.clone(), state.clone()); assert_eq!(store.get(&key).unwrap().key, "app_key1"); } @@ -748,7 +763,7 @@ mod tests { version: 1, }; - let _ = store.insert(key.clone(), state.clone(), "node1".to_string()); + let _ = store.insert(key.clone(), state.clone()); assert_eq!(store.get(&key).unwrap().worker_id, "worker1"); } @@ -763,7 +778,7 @@ mod tests { version: 1, }; - let _ = store.insert(key.clone(), state.clone(), "node1".to_string()); + let _ = store.insert(key.clone(), state.clone()); assert_eq!(store.get(&key).unwrap().model_id, "model1"); } diff --git a/mesh/src/sync.rs b/mesh/src/sync.rs index 3c033decc8..61d3ee853a 100644 --- a/mesh/src/sync.rs +++ b/mesh/src/sync.rs @@ -9,7 +9,7 @@ use tracing::debug; use super::{ service::gossip::NodeStatus, stores::{ - tree_state_key, PolicyState, RateLimitConfig, StateStores, WorkerState, + policy_key, tree_state_key, PolicyState, RateLimitConfig, StateStores, WorkerState, GLOBAL_RATE_LIMIT_COUNTER_KEY, GLOBAL_RATE_LIMIT_KEY, }, tree_ops::{TreeOperation, TreeState}, @@ -81,7 +81,7 @@ impl MeshSyncManager { /// Sync policy state to mesh stores pub fn sync_policy_state(&self, model_id: String, policy_type: String, config: Vec) { - let key = format!("policy:{model_id}"); + let key = policy_key(&model_id); let model_id_for_update = model_id.clone(); let updated_state = self.stores.policy.update(key, move |current| { @@ -114,7 +114,7 @@ impl MeshSyncManager { /// Remove policy state from mesh stores pub fn remove_policy_state(&self, model_id: &str) { - let key = format!("policy:{model_id}"); + let key = policy_key(model_id); self.stores.policy.remove(&key); debug!("Removed policy state from mesh model={}", model_id); } @@ -131,7 +131,7 @@ impl MeshSyncManager { /// Get policy state from mesh stores pub fn get_policy_state(&self, model_id: &str) -> Option { - let key = format!("policy:{model_id}"); + let key = policy_key(model_id); self.stores.policy.get(&key) } @@ -181,7 +181,7 @@ impl MeshSyncManager { /// Apply policy state update from remote node /// The actor should be extracted from the state update context (e.g., from StateUpdate message) pub fn apply_remote_policy_state(&self, state: PolicyState, actor: Option) { - let key = format!("policy:{}", state.model_id); + let key = policy_key(&state.model_id); let actor = actor.unwrap_or_else(|| "remote".to_string()); let mut current_version = 0; @@ -420,8 +420,7 @@ impl MeshSyncManager { version: new_version, }; - let actor = self.self_name.clone(); - if let Err(err) = self.stores.policy.insert(key, state, actor) { + if let Err(err) = self.stores.policy.insert(key, state) { return Err(format!("Failed to persist tree state: {err}")); } debug!( @@ -938,7 +937,6 @@ mod tests { version: 1, metadata: BTreeMap::new(), }, - "node1".to_string(), ); let _ = manager.stores.membership.insert( @@ -950,7 +948,6 @@ mod tests { version: 1, metadata: BTreeMap::new(), }, - "node1".to_string(), ); manager.update_rate_limit_membership(); @@ -974,7 +971,6 @@ mod tests { version: 1, metadata: BTreeMap::new(), }, - "node1".to_string(), ); let _ = manager.stores.membership.insert( @@ -986,7 +982,6 @@ mod tests { version: 1, metadata: BTreeMap::new(), }, - "node1".to_string(), ); manager.update_rate_limit_membership(); @@ -1056,7 +1051,6 @@ mod tests { value: serialized, version: 1, }, - "node1".to_string(), ); let retrieved = manager.get_global_rate_limit_config().unwrap(); @@ -1079,7 +1073,6 @@ mod tests { value: serialized, version: 1, }, - "node1".to_string(), ); // Setup membership diff --git a/mesh/src/tests/comprehensive.rs b/mesh/src/tests/comprehensive.rs index 343b65ad67..86e6add622 100644 --- a/mesh/src/tests/comprehensive.rs +++ b/mesh/src/tests/comprehensive.rs @@ -12,26 +12,21 @@ use std::{ collections::BTreeMap, - net::SocketAddr, sync::{Arc, Once}, time::Duration, }; -use tokio::net::TcpListener; use tracing as log; use tracing_subscriber::{ filter::LevelFilter, layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, }; -use super::test_utils; +use super::test_utils::{self, bind_node, wait_for}; // Internal crate imports - now can access private modules use crate::{ node_state_machine::{ConvergenceConfig, NodeReadiness, NodeStateMachine}, partition::{PartitionConfig, PartitionDetector, PartitionState}, - service::{ - gossip::{NodeState as GossipNodeState, NodeStatus}, - MeshServerHandler, - }, + service::gossip::{NodeState as GossipNodeState, NodeStatus}, stores::{AppState, StateStores}, sync::MeshSyncManager, }; @@ -58,33 +53,6 @@ fn init_test_logging() { }); } -/// Test utility: Find a free port for node binding -async fn find_free_port() -> (TcpListener, u16) { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let port = listener.local_addr().unwrap().port(); - log::debug!("Found free port: {}", port); - (listener, port) -} - -/// Test utility: Get a free socket address -async fn get_node_addr() -> SocketAddr { - let (_listener, port) = find_free_port().await; - format!("127.0.0.1:{port}").parse().unwrap() -} - -/// Test utility: Print cluster state for debugging -fn print_cluster_state(handler: &MeshServerHandler) -> String { - let state = handler.state.read(); - let mut res = vec![]; - for (k, v) in state.iter() { - let status = NodeStatus::try_from(v.status) - .map(|s| format!("{s:?}")) - .unwrap_or_else(|_| format!("Unknown({})", v.status)); - res.push(format!("{}: {} v={}", k, status, v.version)); - } - res.join(", ") -} - #[test] fn test_infrastructure_utilities() { init_test_logging(); @@ -212,11 +180,7 @@ fn test_state_stores_basic_operations() { value: vec![1, 2, 3], version: 1, }; - let _ = stores.app.insert( - "key1".to_string(), - app_state.clone(), - "test_node".to_string(), - ); + let _ = stores.app.insert("key1".to_string(), app_state.clone()); let value = stores.app.get("key1"); assert!(value.is_some()); assert_eq!(value.unwrap().value, vec![1, 2, 3]); @@ -258,20 +222,16 @@ async fn test_single_node_creation_and_shutdown() { init_test_logging(); log::info!("Starting test_single_node_creation_and_shutdown"); - let addr = get_node_addr().await; - let handler = crate::mesh_run!("single_node", addr, None); - - // Wait for node to initialize - tokio::time::sleep(Duration::from_millis(500)).await; + let (listener, addr) = bind_node().await; + let handler = crate::mesh_run!("single_node", listener, addr, None); - // Verify node is in cluster state - { - let state = handler.state.read(); - assert!(state.contains_key("single_node")); - } // Drop the read lock before shutdown + wait_for( + || handler.state.read().contains_key("single_node"), + Duration::from_secs(5), + "single_node appears in cluster state", + ) + .await; - // Test graceful shutdown - must work for single node scenario - // as clusters may scale down to 1 node in production handler.graceful_shutdown().await.unwrap(); log::info!("Single node shutdown completed"); } @@ -281,19 +241,21 @@ async fn test_single_node_data_operations() { init_test_logging(); log::info!("Starting test_single_node_data_operations"); - let addr = get_node_addr().await; - let handler = crate::mesh_run!("data_node", addr, None); + let (listener, addr) = bind_node().await; + let handler = crate::mesh_run!("data_node", listener, addr, None); - tokio::time::sleep(Duration::from_millis(500)).await; + wait_for( + || handler.state.read().contains_key("data_node"), + Duration::from_secs(5), + "data_node appears in cluster state", + ) + .await; - // Write data handler .write_data("test_key".into(), "test_value".into()) .unwrap(); - // Verify data was written - let value = handler.stores.app.get("test_key"); - assert!(value.is_some()); + assert!(handler.stores.app.get("test_key").is_some()); handler.shutdown(); log::info!("Data operations test completed"); @@ -304,12 +266,16 @@ async fn test_single_node_subsystems_initialized() { init_test_logging(); log::info!("Starting test_single_node_subsystems_initialized"); - let addr = get_node_addr().await; - let handler = crate::mesh_run!("subsystem_node", addr, None); + let (listener, addr) = bind_node().await; + let handler = crate::mesh_run!("subsystem_node", listener, addr, None); - tokio::time::sleep(Duration::from_millis(500)).await; + wait_for( + || handler.state.read().contains_key("subsystem_node"), + Duration::from_secs(5), + "subsystem_node appears in cluster state", + ) + .await; - // Now we can access private methods assert!(handler.partition_detector().is_some()); assert!(handler.state_machine().is_some()); @@ -328,32 +294,23 @@ async fn test_two_node_cluster_formation() { init_test_logging(); log::info!("Starting test_two_node_cluster_formation"); - // Start node A - let addr_a = get_node_addr().await; - let handler_a = crate::mesh_run!("node_a", addr_a, None); + let (listener_a, addr_a) = bind_node().await; + let handler_a = crate::mesh_run!("node_a", listener_a, addr_a, None); - tokio::time::sleep(Duration::from_secs(1)).await; + let (listener_b, addr_b) = bind_node().await; + let handler_b = crate::mesh_run!("node_b", listener_b, addr_b, Some(addr_a)); - // Start node B, joining through A - let addr_b = get_node_addr().await; - let handler_b = crate::mesh_run!("node_b", addr_b, Some(addr_a)); + wait_for( + || handler_a.state.read().len() == 2 && handler_b.state.read().len() == 2, + Duration::from_secs(15), + "both nodes see each other", + ) + .await; - // Wait for cluster formation - tokio::time::sleep(Duration::from_secs(3)).await; - - // Verify both nodes see each other - { - let state_a = handler_a.state.read(); - let state_b = handler_b.state.read(); - - assert!(state_a.contains_key("node_a")); - assert!(state_a.contains_key("node_b")); - assert!(state_b.contains_key("node_a")); - assert!(state_b.contains_key("node_b")); - - log::info!("State A: {:?}", print_cluster_state(&handler_a)); - log::info!("State B: {:?}", print_cluster_state(&handler_b)); - } // Drop locks before shutdown + let state_a = handler_a.state.read(); + assert!(state_a.contains_key("node_a")); + assert!(state_a.contains_key("node_b")); + drop(state_a); handler_a.shutdown(); handler_b.shutdown(); @@ -365,49 +322,66 @@ async fn test_two_node_data_synchronization() { init_test_logging(); log::info!("Starting test_two_node_data_synchronization"); - let addr_a = get_node_addr().await; - let handler_a = crate::mesh_run!("sync_node_a", addr_a, None); + let (listener_a, addr_a) = bind_node().await; + let handler_a = crate::mesh_run!("sync_node_a", listener_a, addr_a, None); - let addr_b = get_node_addr().await; - let handler_b = crate::mesh_run!("sync_node_b", addr_b, Some(addr_a)); + let (listener_b, addr_b) = bind_node().await; + let handler_b = crate::mesh_run!("sync_node_b", listener_b, addr_b, Some(addr_a)); - tokio::time::sleep(Duration::from_secs(2)).await; + // Wait for cluster formation + wait_for( + || handler_a.state.read().len() == 2 && handler_b.state.read().len() == 2, + Duration::from_secs(15), + "both nodes see each other", + ) + .await; // Write data on node A handler_a .write_data("shared_key".into(), "shared_value".into()) .unwrap(); - // Wait for automatic sync via sync_stream - tokio::time::sleep(Duration::from_secs(5)).await; - - // Verify data exists on both nodes - let value_a = handler_a.stores.app.get("shared_key"); - let value_b = handler_b.stores.app.get("shared_key"); - - log::info!("Value on A: {:?}", value_a); - log::info!("Value on B: {:?}", value_b); + // Poll until data syncs to B via incremental sync stream + wait_for( + || { + handler_b + .stores + .app + .get("shared_key") + .is_some_and(|v| v.value == b"shared_value") + }, + Duration::from_secs(15), + "shared_key synced to node B", + ) + .await; + + // Allow the sync cycle to settle before writing a second update. + // The incremental collector runs on a 1s interval and the mark_sent + // bookkeeping must complete before the next version can be detected. + tokio::time::sleep(Duration::from_secs(2)).await; - // Update data on node A to see if it syncs again + // Update data on node A handler_a .write_data("shared_key".into(), "shared_value2".into()) .unwrap(); - // Wait for the second update to be synced (increased wait time) - tokio::time::sleep(Duration::from_secs(5)).await; - - // Verify data exists on both nodes - let value_a = handler_a.stores.app.get("shared_key"); - let value_b = handler_b.stores.app.get("shared_key"); - - log::info!("Value on A: {:?}", value_a); - log::info!("Value on B: {:?}", value_b); - - assert!(value_a.is_some()); - assert!(value_b.is_some()); + // Poll until updated value syncs to B + wait_for( + || { + handler_b + .stores + .app + .get("shared_key") + .is_some_and(|v| v.value == b"shared_value2") + }, + Duration::from_secs(30), + "updated shared_key synced to node B", + ) + .await; - // Verify values are the same - assert_eq!(value_a.unwrap().value, value_b.unwrap().value); + let value_a = handler_a.stores.app.get("shared_key").unwrap(); + let value_b = handler_b.stores.app.get("shared_key").unwrap(); + assert_eq!(value_a.value, value_b.value); handler_a.shutdown(); handler_b.shutdown(); @@ -419,35 +393,46 @@ async fn test_two_node_heartbeat_monitoring() { init_test_logging(); log::info!("Starting test_two_node_heartbeat_monitoring"); - let addr_a = get_node_addr().await; - let handler_a = crate::mesh_run!("heartbeat_a", addr_a, None); + let (listener_a, addr_a) = bind_node().await; + let handler_a = crate::mesh_run!("heartbeat_a", listener_a, addr_a, None); - let addr_b = get_node_addr().await; - let handler_b = crate::mesh_run!("heartbeat_b", addr_b, Some(addr_a)); + let (listener_b, addr_b) = bind_node().await; + let handler_b = crate::mesh_run!("heartbeat_b", listener_b, addr_b, Some(addr_a)); - // Wait for cluster formation sync - tokio::time::sleep(Duration::from_secs(3)).await; - - // Both nodes should be alive - { - let state_a = handler_a.state.read(); - let node_b_status = state_a.get("heartbeat_b").map(|n| n.status); - assert_eq!(node_b_status, Some(NodeStatus::Alive as i32)); - } + // Wait for cluster formation + wait_for( + || { + handler_a + .state + .read() + .get("heartbeat_b") + .is_some_and(|n| n.status == NodeStatus::Alive as i32) + }, + Duration::from_secs(15), + "node A sees heartbeat_b as Alive", + ) + .await; // Shutdown node B abruptly handler_b.shutdown(); - // Wait for detection - tokio::time::sleep(Duration::from_secs(5)).await; + // Poll until A detects B as no longer Alive + wait_for( + || { + handler_a + .state + .read() + .get("heartbeat_b") + .is_some_and(|n| n.status != NodeStatus::Alive as i32) + }, + Duration::from_secs(30), + "node A detects heartbeat_b as not Alive", + ) + .await; - // Node A should detect B as suspected or down - let state_a = handler_a.state.read(); - let node_b_status = state_a.get("heartbeat_b").map(|n| n.status); - log::info!("Node B status after shutdown: {:?}", node_b_status); + let status = handler_a.state.read().get("heartbeat_b").map(|n| n.status); + log::info!("Node B status after shutdown: {:?}", status); - // Status should have changed from Alive - assert_ne!(node_b_status, Some(NodeStatus::Alive as i32)); handler_a.shutdown(); log::info!("Two-node heartbeat monitoring test completed"); } @@ -459,49 +444,35 @@ async fn test_two_node_heartbeat_monitoring() { // #[tokio::test] -#[ignore = "Long-running test with complex state convergence"] async fn test_three_node_cluster_formation() { init_test_logging(); log::info!("Starting test_three_node_cluster_formation"); - // Start node A - let addr_a = get_node_addr().await; - let handler_a = crate::mesh_run!("cluster_a", addr_a, None); - - tokio::time::sleep(Duration::from_secs(1)).await; - - // Start node B joining through A - let addr_b = get_node_addr().await; - let handler_b = crate::mesh_run!("cluster_b", addr_b, Some(addr_a)); + let (listener_a, addr_a) = bind_node().await; + let handler_a = crate::mesh_run!("cluster_a", listener_a, addr_a, None); - tokio::time::sleep(Duration::from_secs(1)).await; + let (listener_b, addr_b) = bind_node().await; + let handler_b = crate::mesh_run!("cluster_b", listener_b, addr_b, Some(addr_a)); - // Start node C joining through A - let addr_c = get_node_addr().await; - let handler_c = crate::mesh_run!("cluster_c", addr_c, Some(addr_a)); + let (listener_c, addr_c) = bind_node().await; + let handler_c = crate::mesh_run!("cluster_c", listener_c, addr_c, Some(addr_a)); - // Wait for full cluster formation - tokio::time::sleep(Duration::from_secs(6)).await; - - // Verify all nodes see each other - { - let state_a = handler_a.state.read(); - let state_b = handler_b.state.read(); - let state_c = handler_c.state.read(); - - log::info!("State A: {}", print_cluster_state(&handler_a)); - log::info!("State B: {}", print_cluster_state(&handler_b)); - log::info!("State C: {}", print_cluster_state(&handler_c)); - - // All nodes should see all 3 nodes - assert_eq!(state_a.len(), 3); - assert_eq!(state_b.len(), 3); - assert_eq!(state_c.len(), 3); + wait_for( + || { + handler_a.state.read().len() == 3 + && handler_b.state.read().len() == 3 + && handler_c.state.read().len() == 3 + }, + Duration::from_secs(30), + "all 3 nodes see each other", + ) + .await; - assert!(state_a.contains_key("cluster_a")); - assert!(state_a.contains_key("cluster_b")); - assert!(state_a.contains_key("cluster_c")); - } // Drop locks before shutdown + let state_a = handler_a.state.read(); + assert!(state_a.contains_key("cluster_a")); + assert!(state_a.contains_key("cluster_b")); + assert!(state_a.contains_key("cluster_c")); + drop(state_a); handler_a.shutdown(); handler_b.shutdown(); @@ -510,73 +481,80 @@ async fn test_three_node_cluster_formation() { } #[tokio::test] -#[ignore = "Long-running test with complex state convergence"] async fn test_multi_node_data_propagation() { init_test_logging(); log::info!("Starting test_multi_node_data_propagation"); - let addr_a = get_node_addr().await; - let handler_a = crate::mesh_run!("prop_a", addr_a, None); + let (listener_a, addr_a) = bind_node().await; + let handler_a = crate::mesh_run!("prop_a", listener_a, addr_a, None); - let addr_b = get_node_addr().await; - let handler_b = crate::mesh_run!("prop_b", addr_b, Some(addr_a)); + let (listener_b, addr_b) = bind_node().await; + let handler_b = crate::mesh_run!("prop_b", listener_b, addr_b, Some(addr_a)); - let addr_c = get_node_addr().await; - let handler_c = crate::mesh_run!("prop_c", addr_c, Some(addr_a)); + let (listener_c, addr_c) = bind_node().await; + let handler_c = crate::mesh_run!("prop_c", listener_c, addr_c, Some(addr_a)); - tokio::time::sleep(Duration::from_secs(3)).await; + // Wait for 3-node cluster + wait_for( + || { + handler_a.state.read().len() == 3 + && handler_b.state.read().len() == 3 + && handler_c.state.read().len() == 3 + }, + Duration::from_secs(30), + "all 3 nodes see each other", + ) + .await; // Write data on node A handler_a .write_data("propagated_key".into(), "propagated_value".into()) .unwrap(); - // Wait for automatic sync via sync_stream - tokio::time::sleep(Duration::from_secs(5)).await; - - // Verify data reached all nodes - let value_a = handler_a.stores.app.get("propagated_key"); - let value_b = handler_b.stores.app.get("propagated_key"); - let value_c = handler_c.stores.app.get("propagated_key"); - - log::info!("Value on A: {:?}", value_a); - log::info!("Value on B: {:?}", value_b); - log::info!("Value on C: {:?}", value_c); - - assert!(value_a.is_some()); - assert!(value_b.is_some()); - assert!(value_c.is_some()); + // Poll until data reaches B and C + wait_for( + || { + handler_b.stores.app.get("propagated_key").is_some() + && handler_c.stores.app.get("propagated_key").is_some() + }, + Duration::from_secs(30), + "propagated_key synced to B and C", + ) + .await; - // Verify all values are the same - let val_a = value_a.unwrap().value; - assert_eq!(val_a, value_b.unwrap().value); - assert_eq!(val_a, value_c.unwrap().value); + let val_a = handler_a.stores.app.get("propagated_key").unwrap().value; + assert_eq!( + val_a, + handler_b.stores.app.get("propagated_key").unwrap().value + ); + assert_eq!( + val_a, + handler_c.stores.app.get("propagated_key").unwrap().value + ); - // Write data on node B to verify continued propagation + // Write updated data on node B handler_b - .write_data("propagated_key".into(), "propagated_value".into()) + .write_data("propagated_key".into(), "propagated_value2".into()) .unwrap(); - // Wait for automatic sync via sync_stream - tokio::time::sleep(Duration::from_secs(5)).await; - - // Verify data reached all nodes - let value_a = handler_a.stores.app.get("propagated_key"); - let value_b = handler_b.stores.app.get("propagated_key"); - let value_c = handler_c.stores.app.get("propagated_key"); - - log::info!("Value on A: {:?}", value_a); - log::info!("Value on B: {:?}", value_b); - log::info!("Value on C: {:?}", value_c); - - assert!(value_a.is_some()); - assert!(value_b.is_some()); - assert!(value_c.is_some()); - - // Verify all values are the same - let val_a = value_a.unwrap().value; - assert_eq!(val_a, value_b.unwrap().value); - assert_eq!(val_a, value_c.unwrap().value); + // Poll until updated value reaches A and C + wait_for( + || { + handler_a + .stores + .app + .get("propagated_key") + .is_some_and(|v| v.value == b"propagated_value2") + && handler_c + .stores + .app + .get("propagated_key") + .is_some_and(|v| v.value == b"propagated_value2") + }, + Duration::from_secs(30), + "updated propagated_key synced to A and C", + ) + .await; handler_a.shutdown(); handler_b.shutdown(); @@ -585,79 +563,104 @@ async fn test_multi_node_data_propagation() { } #[tokio::test] -#[ignore = "Long-running test with complex state convergence"] +#[ignore = "SWIM failure detection for hard-shutdown nodes needs many gossip rounds; flaky under parallel CI load"] async fn test_five_node_cluster_with_failure() { init_test_logging(); log::info!("Starting test_five_node_cluster_with_failure"); - // Setup initial cluster with nodes A and B - let addr_a = get_node_addr().await; - let handler_a = crate::mesh_run!("multi_a", addr_a, None); + let (listener_a, addr_a) = bind_node().await; + let handler_a = crate::mesh_run!("multi_a", listener_a, addr_a, None); - let addr_b = get_node_addr().await; - let handler_b = crate::mesh_run!("multi_b", addr_b, Some(addr_a)); + let (listener_b, addr_b) = bind_node().await; + let handler_b = crate::mesh_run!("multi_b", listener_b, addr_b, Some(addr_a)); - tokio::time::sleep(Duration::from_secs(2)).await; + // Wait for A-B cluster + wait_for( + || handler_a.state.read().len() == 2, + Duration::from_secs(15), + "A-B cluster formed", + ) + .await; - // Write some data handler_a .write_data("test_data".into(), "initial_value".into()) .unwrap(); - log::info!("Initial data written"); - // Add nodes C and D - let addr_c = get_node_addr().await; - let handler_c = crate::mesh_run!("multi_c", addr_c, Some(addr_a)); + // Add C and D + let (listener_c, addr_c) = bind_node().await; + let handler_c = crate::mesh_run!("multi_c", listener_c, addr_c, Some(addr_a)); - let addr_d = get_node_addr().await; - let handler_d = crate::mesh_run!("multi_d", addr_d, Some(addr_c)); + let (listener_d, addr_d) = bind_node().await; + let handler_d = crate::mesh_run!("multi_d", listener_d, addr_d, Some(addr_c)); - tokio::time::sleep(Duration::from_secs(2)).await; - log::info!("Nodes C and D joined"); + wait_for( + || handler_a.state.read().len() == 4, + Duration::from_secs(30), + "4-node cluster formed", + ) + .await; - // Add node E and then shut it down + // Add E, wait for it to join, then kill it { - let addr_e = get_node_addr().await; - let handler_e = crate::mesh_run!("multi_e", addr_e, Some(addr_d)); + let (listener_e, addr_e) = bind_node().await; + let handler_e = crate::mesh_run!("multi_e", listener_e, addr_e, Some(addr_d)); - tokio::time::sleep(Duration::from_secs(3)).await; - log::info!("Node E joined, state: {}", print_cluster_state(&handler_e)); + wait_for( + || handler_a.state.read().len() == 5, + Duration::from_secs(30), + "5-node cluster formed", + ) + .await; handler_e.shutdown(); log::info!("Node E shutdown"); } - // Gracefully shutdown node D + // Gracefully shutdown D handler_d.graceful_shutdown().await.unwrap(); - tokio::time::sleep(Duration::from_secs(2)).await; log::info!("Node D gracefully shutdown"); - // Wait for state convergence - tokio::time::sleep(Duration::from_secs(10)).await; - - log::info!("Final state A: {}", print_cluster_state(&handler_a)); - log::info!("Final state B: {}", print_cluster_state(&handler_b)); - log::info!("Final state C: {}", print_cluster_state(&handler_c)); + // Wait for D to be Leaving + wait_for( + || { + handler_a + .state + .read() + .get("multi_d") + .is_some_and(|n| n.status == NodeStatus::Leaving as i32) + }, + Duration::from_secs(30), + "node D detected as Leaving", + ) + .await; + + // Wait for E to be detected as not Alive (Suspected or Down). + // SWIM failure detection requires multiple gossip rounds, so allow ample time + // especially when other tests are running in parallel. + wait_for( + || { + handler_a + .state + .read() + .get("multi_e") + .is_some_and(|n| n.status != NodeStatus::Alive as i32) + }, + Duration::from_secs(60), + "node E detected as not Alive", + ) + .await; - // Verify remaining nodes have consistent view let state_a = handler_a.state.read(); - let _state_b = handler_b.state.read(); - let _state_c = handler_c.state.read(); - - // All remaining nodes should see nodes A, B, C assert!(state_a.contains_key("multi_a")); assert!(state_a.contains_key("multi_b")); assert!(state_a.contains_key("multi_c")); - - // D should be in Leaving state assert_eq!( state_a.get("multi_d").map(|n| n.status), Some(NodeStatus::Leaving as i32) ); - - // E should eventually be Down let e_status = state_a.get("multi_e").map(|n| n.status); log::info!("Node E final status: {:?}", e_status); + drop(state_a); handler_a.shutdown(); handler_b.shutdown(); @@ -670,49 +673,31 @@ async fn test_cluster_formation_different_join_patterns() { init_test_logging(); log::info!("Starting test_cluster_formation_different_join_patterns"); - // Create initial node - let addr_a = get_node_addr().await; - let handler_a = crate::mesh_run!("pattern_a", addr_a, None); - - tokio::time::sleep(Duration::from_secs(1)).await; - - // Node B joins through A - let addr_b = get_node_addr().await; - let handler_b = crate::mesh_run!("pattern_b", addr_b, Some(addr_a)); + let (listener_a, addr_a) = bind_node().await; + let handler_a = crate::mesh_run!("pattern_a", listener_a, addr_a, None); - tokio::time::sleep(Duration::from_secs(1)).await; + let (listener_b, addr_b) = bind_node().await; + let handler_b = crate::mesh_run!("pattern_b", listener_b, addr_b, Some(addr_a)); - // Node C joins through B (chain) - let addr_c = get_node_addr().await; - let handler_c = crate::mesh_run!("pattern_c", addr_c, Some(addr_b)); + // Node C joins through B (chain topology) + let (listener_c, addr_c) = bind_node().await; + let handler_c = crate::mesh_run!("pattern_c", listener_c, addr_c, Some(addr_b)); - tokio::time::sleep(Duration::from_secs(1)).await; + // Node D joins through A (star topology) + let (listener_d, addr_d) = bind_node().await; + let handler_d = crate::mesh_run!("pattern_d", listener_d, addr_d, Some(addr_a)); - // Node D joins through A (star pattern) - let addr_d = get_node_addr().await; - let handler_d = crate::mesh_run!("pattern_d", addr_d, Some(addr_a)); - - // Wait for convergence - need more time for gossip to propagate through chain topology - // With 4 nodes in different join patterns (chain + star), state needs multiple rounds to converge - tokio::time::sleep(Duration::from_secs(8)).await; - - // Verify all nodes see all 4 nodes regardless of join pattern - { - let state_a = handler_a.state.read(); - let state_b = handler_b.state.read(); - let state_c = handler_c.state.read(); - let state_d = handler_d.state.read(); - - log::info!("State A: {}", print_cluster_state(&handler_a)); - log::info!("State B: {}", print_cluster_state(&handler_b)); - log::info!("State C: {}", print_cluster_state(&handler_c)); - log::info!("State D: {}", print_cluster_state(&handler_d)); - - assert_eq!(state_a.len(), 4); - assert_eq!(state_b.len(), 4); - assert_eq!(state_c.len(), 4); - assert_eq!(state_d.len(), 4); - } // Drop locks before shutdown + wait_for( + || { + handler_a.state.read().len() == 4 + && handler_b.state.read().len() == 4 + && handler_c.state.read().len() == 4 + && handler_d.state.read().len() == 4 + }, + Duration::from_secs(30), + "all 4 nodes see each other (chain + star topology)", + ) + .await; handler_a.shutdown(); handler_b.shutdown(); diff --git a/mesh/src/tests/mod.rs b/mesh/src/tests/mod.rs index 7a40d8a552..2fd6b90d8c 100644 --- a/mesh/src/tests/mod.rs +++ b/mesh/src/tests/mod.rs @@ -5,4 +5,4 @@ #[cfg(test)] mod comprehensive; -mod test_utils; +pub(crate) mod test_utils; diff --git a/mesh/src/tests/test_utils.rs b/mesh/src/tests/test_utils.rs index 40744ca46d..a0d82bcd5a 100644 --- a/mesh/src/tests/test_utils.rs +++ b/mesh/src/tests/test_utils.rs @@ -1,10 +1,13 @@ //! Test utilities for mesh module use std::{ collections::{BTreeMap, HashMap}, + net::SocketAddr, sync::Arc, + time::Duration, }; use parking_lot::RwLock; +use tokio::net::TcpListener; use crate::{ service::{gossip::NodeState, ClusterState}, @@ -12,6 +15,31 @@ use crate::{ sync::MeshSyncManager, }; +/// Bind to an ephemeral port and return the listener + address. +/// The caller must keep the listener alive and pass it to `mesh_run!` +/// to avoid a TOCTOU port race. +pub async fn bind_node() -> (TcpListener, SocketAddr) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tracing::debug!("Bound node to {}", addr); + (listener, addr) +} + +/// Poll `condition` every 100ms until it returns true or `timeout` expires. +pub async fn wait_for(condition: F, timeout: Duration, msg: &str) +where + F: Fn() -> bool, +{ + let start = std::time::Instant::now(); + while !condition() { + assert!( + start.elapsed() <= timeout, + "Timeout after {timeout:?} waiting for: {msg}" + ); + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + /// Create test StateStores with a given node name pub fn create_test_stores(self_name: String) -> Arc { Arc::new(StateStores::with_self_name(self_name)) diff --git a/model_gateway/tests/mesh_integration_test.rs b/model_gateway/tests/mesh_integration_test.rs index 071b305fab..9f22af512d 100644 --- a/model_gateway/tests/mesh_integration_test.rs +++ b/model_gateway/tests/mesh_integration_test.rs @@ -114,7 +114,6 @@ async fn test_rate_limit_cluster_consistency() { version: 1, metadata: std::collections::BTreeMap::new(), }, - name.to_string(), ); } } @@ -133,7 +132,6 @@ async fn test_rate_limit_cluster_consistency() { value: serialized.clone(), version: 1, }, - "node1".to_string(), ); } @@ -380,7 +378,6 @@ async fn test_rate_limit_window_reset() { value: serialized, version: 1, }, - "node1".to_string(), ); // Recreate manager with updated stores