diff --git a/testsuite/forge-cli/src/main.rs b/testsuite/forge-cli/src/main.rs index 794e564efaba1..42dc545c18846 100644 --- a/testsuite/forge-cli/src/main.rs +++ b/testsuite/forge-cli/src/main.rs @@ -811,9 +811,12 @@ fn get_multi_region_test(test_name: &str) -> Option { Some(test) } -fn wrap_with_realistic_env(test: T) -> CompositeNetworkTest { +fn wrap_with_realistic_env( + num_validators: usize, + test: T, +) -> CompositeNetworkTest { CompositeNetworkTest::new_with_two_wrappers( - MultiRegionNetworkEmulationTest::default(), + MultiRegionNetworkEmulationTest::default_for_validator_count(num_validators), CpuChaosTest::default(), test, ) @@ -858,8 +861,9 @@ fn wrap_with_two_region_env(test: T) -> CompositeNetwo } fn run_consensus_only_realistic_env_max_tps() -> ForgeConfig { + let num_validators = 20; ForgeConfig::default() - .with_initial_validator_count(NonZeroUsize::new(20).unwrap()) + .with_initial_validator_count(NonZeroUsize::new(num_validators).unwrap()) .with_emit_job( EmitJobRequest::default() .mode(EmitJobMode::MaxLoad { @@ -868,7 +872,7 @@ fn run_consensus_only_realistic_env_max_tps() -> ForgeConfig { .txn_expiration_time_secs(5 * 60), ) .add_network_test(CompositeNetworkTest::new( - MultiRegionNetworkEmulationTest::default(), + MultiRegionNetworkEmulationTest::default_for_validator_count(num_validators), CpuChaosTest::default(), )) .with_genesis_helm_config_fn(Arc::new(|helm_values| { @@ -1119,7 +1123,7 @@ fn realistic_env_sweep_wrap( .with_validator_override_node_config_fn(Arc::new(|config, _| { config.execution.processed_transactions_detailed_counters = true; })) - .add_network_test(wrap_with_realistic_env(test)) + .add_network_test(wrap_with_realistic_env(num_validators, test)) // Test inherits the main EmitJobRequest, so update here for more precise latency measurements .with_emit_job( EmitJobRequest::default().latency_polling_interval(Duration::from_millis(100)), @@ -1388,10 +1392,11 @@ fn workload_vs_perf_benchmark() -> ForgeConfig { } fn realistic_env_graceful_overload(duration: Duration) -> ForgeConfig { + let num_validators = 20; ForgeConfig::default() - .with_initial_validator_count(NonZeroUsize::new(20).unwrap()) + .with_initial_validator_count(NonZeroUsize::new(num_validators).unwrap()) .with_initial_fullnode_count(20) - .add_network_test(wrap_with_realistic_env(TwoTrafficsTest { + .add_network_test(wrap_with_realistic_env(num_validators, TwoTrafficsTest { inner_traffic: EmitJobRequest::default() .mode(EmitJobMode::ConstTps { tps: 15000 }) .init_gas_price_multiplier(20), @@ -1952,7 +1957,7 @@ fn realistic_env_max_load_test( ForgeConfig::default() .with_initial_validator_count(NonZeroUsize::new(num_validators).unwrap()) .with_initial_fullnode_count(num_fullnodes) - .add_network_test(wrap_with_realistic_env(TwoTrafficsTest { + .add_network_test(wrap_with_realistic_env(num_validators, TwoTrafficsTest { inner_traffic: EmitJobRequest::default() .mode(EmitJobMode::MaxLoad { mempool_backlog }) .init_gas_price_multiplier(20), @@ -2013,7 +2018,7 @@ fn realistic_network_tuned_for_throughput_test() -> ForgeConfig { let mut forge_config = ForgeConfig::default() .with_initial_validator_count(NonZeroUsize::new(VALIDATOR_COUNT).unwrap()) - .add_network_test(MultiRegionNetworkEmulationTest::default()) + .add_network_test(MultiRegionNetworkEmulationTest::default_for_validator_count(VALIDATOR_COUNT)) .with_emit_job(EmitJobRequest::default().mode(EmitJobMode::MaxLoad { mempool_backlog: (TARGET_TPS as f64 * VFN_LATENCY_S) as usize, })) @@ -2326,8 +2331,9 @@ fn quorum_store_reconfig_enable_test() -> ForgeConfig { } fn mainnet_like_simulation_test() -> ForgeConfig { + let num_validators = 20; ForgeConfig::default() - .with_initial_validator_count(NonZeroUsize::new(20).unwrap()) + .with_initial_validator_count(NonZeroUsize::new(num_validators).unwrap()) .with_emit_job( EmitJobRequest::default() .mode(EmitJobMode::MaxLoad { @@ -2336,7 +2342,7 @@ fn mainnet_like_simulation_test() -> ForgeConfig { .txn_expiration_time_secs(5 * 60), ) .add_network_test(CompositeNetworkTest::new( - MultiRegionNetworkEmulationTest::default(), + MultiRegionNetworkEmulationTest::default_for_validator_count(num_validators), CpuChaosTest::default(), )) .with_genesis_helm_config_fn(Arc::new(|helm_values| { diff --git a/testsuite/forge-cli/src/suites/dag.rs b/testsuite/forge-cli/src/suites/dag.rs index f309c6403cba1..1e0848db8d307 100644 --- a/testsuite/forge-cli/src/suites/dag.rs +++ b/testsuite/forge-cli/src/suites/dag.rs @@ -60,7 +60,7 @@ fn dag_realistic_env_max_load_test( ForgeConfig::default() .with_initial_validator_count(NonZeroUsize::new(num_validators).unwrap()) .with_initial_fullnode_count(num_fullnodes) - .add_network_test(wrap_with_realistic_env(TwoTrafficsTest { + .add_network_test(wrap_with_realistic_env(num_validators, TwoTrafficsTest { inner_traffic: EmitJobRequest::default() .mode(EmitJobMode::MaxLoad { mempool_backlog: 50000, diff --git a/testsuite/forge/src/backend/k8s/swarm.rs b/testsuite/forge/src/backend/k8s/swarm.rs index 9211bed8a1381..9a14b69963ee8 100644 --- a/testsuite/forge/src/backend/k8s/swarm.rs +++ b/testsuite/forge/src/backend/k8s/swarm.rs @@ -735,8 +735,8 @@ trait ChaosExperimentOps { async fn list_stress_chaos(&self) -> Result>; async fn ensure_chaos_experiments_active(&self) -> Result<()> { - let timeout_duration = Duration::from_secs(300); // 5 minutes - let polling_interval = Duration::from_secs(5); + let timeout_duration = Duration::from_secs(600); // 10 minutes + let polling_interval = Duration::from_secs(10); tokio::time::timeout(timeout_duration, async { loop { @@ -793,6 +793,8 @@ fn check_all_injected(status: &Option) -> bool { .map_or(false, |conditions| { conditions.iter().any(|c| { c.r#type == ChaosConditionType::AllInjected && c.status == ConditionStatus::True + }) && conditions.iter().any(|c| { + c.r#type == ChaosConditionType::Selected && c.status == ConditionStatus::True }) }) } @@ -870,19 +872,31 @@ mod tests { ) -> (Vec, Vec) { let network_chaos = NetworkChaos { status: Some(ChaosStatus { - conditions: Some(vec![ChaosCondition { - r#type: ChaosConditionType::AllInjected, - status: network_status, - }]), + conditions: Some(vec![ + ChaosCondition { + r#type: ChaosConditionType::AllInjected, + status: network_status.clone(), + }, + ChaosCondition { + r#type: ChaosConditionType::Selected, + status: network_status, + }, + ]), }), ..NetworkChaos::new("test", Default::default()) }; let stress_chaos = StressChaos { status: Some(ChaosStatus { - conditions: Some(vec![ChaosCondition { - r#type: ChaosConditionType::AllInjected, - status: stress_status, - }]), + conditions: Some(vec![ + ChaosCondition { + r#type: ChaosConditionType::AllInjected, + status: stress_status.clone(), + }, + ChaosCondition { + r#type: ChaosConditionType::Selected, + status: stress_status, + }, + ]), }), ..StressChaos::new("test", Default::default()) }; diff --git a/testsuite/testcases/src/data/six_region_link_stats.csv b/testsuite/testcases/src/data/six_region_link_stats.csv new file mode 100644 index 0000000000000..58fc7e486e6be --- /dev/null +++ b/testsuite/testcases/src/data/six_region_link_stats.csv @@ -0,0 +1,31 @@ +sending_region,receiving_region,bitrate_bps,avgrtt +gcp--us-central1,aws--eu-west-1,300000000,103.435 +gcp--us-central1,aws--ap-northeast-1,300000000,133.996 +gcp--us-central1,aws--sa-east-1,300000000,145.483 +gcp--us-central1,aws--eu-central1,300000000,107.671 +gcp--us-central1,gcp--ca-central1,300000000,29.748 +aws--sa-east-1,gcp--us-central1,300000000,145.703 +aws--sa-east-1,aws--eu-west-1,300000000,176.894 +aws--sa-east-1,aws--ap-northeast-1,300000000,255.289 +aws--sa-east-1,aws--eu-central1,300000000,203.508 +aws--sa-east-1,gcp--ca-central-1,300000000,124.307 +aws--eu-west-1,gcp--us-central1,300000000,104.169 +aws--eu-west-1,aws--sa-east-1,300000000,176.813 +aws--eu-west-1,aws--ap-northeast-1,300000000,198.555 +aws--eu-west-1,aws--eu-central1,300000000,23.493 +aws--eu-west-1,gcp--ca-central-1,300000000,68.622 +aws--ap-northeast-1,gcp--us-central1,300000000,128.999 +aws--ap-northeast-1,aws--eu-west-1,300000000,198.539 +aws--ap-northeast-1,aws--sa-east-1,300000000,255.323 +aws--ap-northeast-1,aws--eu-central1,300000000,223.400 +aws--ap-northeast-1,gcp--ca-central-1,300000000,142.549 +aws--eu-central1,gcp--us-central1,300000000,107.671 +aws--eu-central1,aws--sa-east-1,300000000,203.508 +aws--eu-central1,aws--eu-west-1,300000000,23.493 +aws--eu-central1,aws--ap-northeast-1,300000000,223.400 +aws--eu-central1,gcp--ca-central-1,300000000,89.889 +gcp--ca-central-1,gcp--us-central1,300000000,29.748 +gcp--ca-central-1,aws--sa-east-1,300000000,124.307 +gcp--ca-central-1,aws--eu-west-1,300000000,68.622 +gcp--ca-central-1,aws--ap-northeast-1,300000000,142.549 +gcp--ca-central-1,aws--eu-central1,300000000,89.889 \ No newline at end of file diff --git a/testsuite/testcases/src/multi_region_network_test.rs b/testsuite/testcases/src/multi_region_network_test.rs index 840d37699152e..cbe5c10865c27 100644 --- a/testsuite/testcases/src/multi_region_network_test.rs +++ b/testsuite/testcases/src/multi_region_network_test.rs @@ -19,6 +19,7 @@ use std::{collections::BTreeMap, sync::Arc}; /// is measuring TCP bandwidth only which is primarily affected by RTT, and not the actual bandwidth /// across the regions, which would vary according to competing traffic, etc. const FOUR_REGION_LINK_STATS: &[u8] = include_bytes!("data/four_region_link_stats.csv"); +const SIX_REGION_LINK_STATS: &[u8] = include_bytes!("data/six_region_link_stats.csv"); /// The two regions were chosen as the most distant regions among the four regions set. const TWO_REGION_LINK_STATS: &[u8] = include_bytes!("data/two_region_link_stats.csv"); @@ -79,8 +80,8 @@ fn create_link_stats_table_with_peer_groups( "At least 2 regions are required for inter-region network chaos." ); assert!( - number_of_regions <= 4, - "ChaosMesh only supports simulating up to 4 regions." + number_of_regions <= 6, + "ChaosMesh only supports simulating up to 6 regions." ); // Create the link stats table with peer groups @@ -237,10 +238,23 @@ impl MultiRegionNetworkEmulationConfig { ..Default::default() } } + + pub fn four_regions() -> Self { + Self { + link_stats_table: get_link_stats_table(FOUR_REGION_LINK_STATS), + ..Default::default() + } + } + + pub fn six_regions() -> Self { + Self { + link_stats_table: get_link_stats_table(SIX_REGION_LINK_STATS), + ..Default::default() + } + } } /// A test to emulate network conditions for a multi-region setup. -#[derive(Default)] pub struct MultiRegionNetworkEmulationTest { network_emulation_config: MultiRegionNetworkEmulationConfig, } @@ -252,6 +266,18 @@ impl MultiRegionNetworkEmulationTest { } } + pub fn default_for_validator_count(num_validators: usize) -> Self { + if num_validators > 100 { + Self { + network_emulation_config: MultiRegionNetworkEmulationConfig::six_regions(), + } + } else { + Self { + network_emulation_config: MultiRegionNetworkEmulationConfig::four_regions(), + } + } + } + /// Creates a new SwarmNetEm to be injected via chaos. Note: network /// emulation is only done for the validators in the swarm (and not /// the fullnodes).