diff --git a/rs/tests/consensus/catch_up_test_common/src/lib.rs b/rs/tests/consensus/catch_up_test_common/src/lib.rs index 8435a1b9a2e0..f2e238c81f47 100644 --- a/rs/tests/consensus/catch_up_test_common/src/lib.rs +++ b/rs/tests/consensus/catch_up_test_common/src/lib.rs @@ -78,7 +78,7 @@ fn test(env: TestEnv, expect_catch_up: bool) { Height::from(slow_node_shut_down_height), log.clone(), ); - malicious_node.vm().kill(); + block_on(async { malicious_node.vm().await.kill().await }); info!(log, "Killed the slow node"); info!(log, "Wait another DKG interval, then restart the slow node"); @@ -94,7 +94,7 @@ fn test(env: TestEnv, expect_catch_up: bool) { log.clone(), ) }); - malicious_node.vm().start(); + block_on(async { malicious_node.vm().await.start().await }); info!(log, "Restarted slow node"); // Wait until the node is available again diff --git a/rs/tests/consensus/node_graceful_leaving_test.rs b/rs/tests/consensus/node_graceful_leaving_test.rs index a09e00b2ac43..9a2cf6dd07b7 100644 --- a/rs/tests/consensus/node_graceful_leaving_test.rs +++ b/rs/tests/consensus/node_graceful_leaving_test.rs @@ -112,7 +112,9 @@ fn test(env: TestEnv) { ); info!(log, "Kill nodes after removal (last shot to the victims)"); - nns_nodes_to_remove.iter().for_each(|node| node.vm().kill()); + nns_nodes_to_remove + .iter() + .for_each(|node| block_on(async { node.vm().await.kill().await })); // Assert that `update` call can still be executed, this ensures that removed+killed nodes are not part of the consensus committee. let update_message = b"This beautiful prose should be persisted for future generations"; block_on(async { assert_subnet_can_make_progress(update_message, nns_node).await }); diff --git a/rs/tests/consensus/orchestrator/node_assign_test.rs b/rs/tests/consensus/orchestrator/node_assign_test.rs index aa66f7f993e3..172aa39521df 100644 --- a/rs/tests/consensus/orchestrator/node_assign_test.rs +++ b/rs/tests/consensus/orchestrator/node_assign_test.rs @@ -169,7 +169,7 @@ fn test(env: TestEnv) { let kill_nodes_count = UNASSIGNED_NODES_COUNT / 3; info!(logger, "Kill {} of the new nodes", kill_nodes_count); for n in newly_assigned_nodes.iter().take(kill_nodes_count) { - n.vm().kill(); + block_on(async { n.vm().await.kill().await }); } // Second loop to paralelize the effects of the previous one @@ -191,7 +191,13 @@ fn test(env: TestEnv) { // Kill one more node and break consensus. info!(logger, "Kill one more node and break consensus"); - newly_assigned_nodes[kill_nodes_count].vm().kill(); + block_on(async { + newly_assigned_nodes[kill_nodes_count] + .vm() + .await + .kill() + .await + }); info!(logger, "Wait for it to become unavailable"); newly_assigned_nodes[kill_nodes_count] .await_status_is_unavailable() @@ -211,7 +217,13 @@ fn test(env: TestEnv) { // Restart node to start consensus. info!(logger, "Restart node to start consensus"); - newly_assigned_nodes[kill_nodes_count].vm().start(); + block_on(async { + newly_assigned_nodes[kill_nodes_count] + .vm() + .await + .start() + .await + }); info!(logger, "Wait for subnet to restart"); // Wait for 1 DKG interval to ensure that subnet makes progress again. let target_height = diff --git a/rs/tests/consensus/tecdsa/tecdsa_remove_nodes_test.rs b/rs/tests/consensus/tecdsa/tecdsa_remove_nodes_test.rs index df4128c74377..273c5bb25b10 100644 --- a/rs/tests/consensus/tecdsa/tecdsa_remove_nodes_test.rs +++ b/rs/tests/consensus/tecdsa/tecdsa_remove_nodes_test.rs @@ -22,6 +22,7 @@ use anyhow::Result; use ic_system_test_driver::driver::group::SystemTestGroup; use ic_system_test_driver::systest; +use ic_system_test_driver::util::block_on; use canister_test::Canister; use ic_base_types::NodeId; @@ -125,7 +126,9 @@ fn test(env: TestEnv) { EndpointsStatus::AllUnhealthy, ); info!(log, "Kill nodes after removal (last shot to the victims)"); - nns_nodes_to_remove.iter().for_each(|node| node.vm().kill()); + nns_nodes_to_remove + .iter() + .for_each(|node| block_on(async { node.vm().await.kill().await })); info!(log, "Verify signature"); block_on(async { let msg_can = MessageCanister::from_canister_id(&nns_agent, canister_id); diff --git a/rs/tests/consensus/upgrade/common.rs b/rs/tests/consensus/upgrade/common.rs index 1693ffe0f63e..778426aede51 100644 --- a/rs/tests/consensus/upgrade/common.rs +++ b/rs/tests/consensus/upgrade/common.rs @@ -362,7 +362,7 @@ pub fn stop_node(logger: &Logger, app_node: &IcNodeSnapshot) { .await_status_is_healthy() .expect("Node not healthy"); info!(logger, "Kill node: {}", app_node.get_ip_addr()); - app_node.vm().kill(); + block_on(async { app_node.vm().await.kill().await }); app_node .await_status_is_unavailable() .expect("Node still healthy"); @@ -375,7 +375,7 @@ pub fn start_node(logger: &Logger, app_node: &IcNodeSnapshot) { .await_status_is_unavailable() .expect("Node still healthy"); info!(logger, "Starting node: {}", app_node.get_ip_addr()); - app_node.vm().start(); + block_on(async { app_node.vm().await.start().await }); app_node .await_status_is_healthy() .expect("Node not healthy"); diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index f311c43a2051..a5bca4b33898 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -20,6 +20,7 @@ use crate::driver::{ }, test_setup::InfraProvider, }; +use crate::util::block_on; use anyhow::{Context, Result, bail}; use config_tool::hostos::guestos_bootstrap_image::BootstrapOptions; use config_tool::setupos::{ @@ -271,13 +272,13 @@ pub fn setup_and_start_vms( &node, &t_farm, )?); - t_farm.attach_disk_images( + block_on(t_farm.attach_disk_images( &group_name, &vm_name, "usb-storage", vec![image_spec], - )?; - t_farm.start_vm(&group_name, &vm_name)?; + ))?; + block_on(t_farm.start_vm(&group_name, &vm_name))?; } } std::fs::remove_file(conf_img_path)?; @@ -346,19 +347,19 @@ pub fn setup_and_start_nested_vms( &t_ic_gateway_url, t_nns_public_key_override.as_deref(), )?; - let config_image_spec = AttachImageSpec::new(t_farm.upload_file( + let config_image_spec = AttachImageSpec::new(block_on(t_farm.upload_file( &t_group_name, config_image, NESTED_CONFIG_IMAGE_PATH, - )?); + ))?); - t_farm.attach_disk_images( + block_on(t_farm.attach_disk_images( &t_group_name, &vm_name, "usb-storage", vec![t_setupos_image_spec, config_image_spec], - )?; - t_farm.start_vm(&t_group_name, &vm_name)?; + ))?; + block_on(t_farm.start_vm(&t_group_name, &vm_name))?; Ok(()) })); @@ -413,7 +414,7 @@ pub fn upload_config_disk_image( ) -> FarmResult { let compressed_img_path = mk_compressed_img_path(); let target_file = PathBuf::from(&node.node_path).join(compressed_img_path.clone()); - let image_id = farm.upload_file(group_name, target_file, &compressed_img_path)?; + let image_id = block_on(farm.upload_file(group_name, target_file, &compressed_img_path))?; info!(farm.logger, "Uploaded image: {}", image_id); Ok(image_id) } diff --git a/rs/tests/driver/src/driver/farm.rs b/rs/tests/driver/src/driver/farm.rs index 3289eed512ab..65def10d252c 100644 --- a/rs/tests/driver/src/driver/farm.rs +++ b/rs/tests/driver/src/driver/farm.rs @@ -12,7 +12,7 @@ use crate::driver::test_env_api::HasFarmUrl; use anyhow::Result; use chrono::{DateTime, Utc}; use ic_crypto_sha2::Sha256; -use reqwest::blocking::{Client, RequestBuilder, multipart}; +use reqwest::{Client, RequestBuilder, multipart}; use serde::de::Error; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use slog::info; @@ -56,8 +56,8 @@ pub struct Farm { } impl Farm { - pub fn new(base_url: Url, logger: Logger) -> Self { - let client = reqwest::blocking::ClientBuilder::new() + pub async fn new(base_url: Url, logger: Logger) -> Self { + let client = reqwest::ClientBuilder::new() .timeout(TIMEOUT_SETTINGS.max_http_timeout) .build() .expect("This should not fail."); @@ -69,8 +69,8 @@ impl Farm { } } - pub fn from_test_env(env: &TestEnv, context: &str) -> Self { - let client = reqwest::blocking::ClientBuilder::new() + pub async fn from_test_env(env: &TestEnv, context: &str) -> Self { + let client = reqwest::ClientBuilder::new() .timeout(TIMEOUT_SETTINGS.max_http_timeout) .build() .expect("This should not fail."); @@ -82,16 +82,19 @@ impl Farm { } } - pub fn acquire_playnet_certificate(&self, group_name: &str) -> FarmResult { + pub async fn acquire_playnet_certificate( + &self, + group_name: &str, + ) -> FarmResult { let path = format!("group/{group_name}/playnet/certificate"); let rb = self.post(&path); - let rbb = || rb.try_clone().expect("could not clone a request builder"); - let resp = self.retry_until_success_long(rbb)?; - let playnet_cert = resp.json::()?; + let rbb = async move || rb.try_clone().expect("could not clone a request builder"); + let resp = self.retry_until_success_long(rbb).await?; + let playnet_cert = resp.json::().await?; Ok(playnet_cert) } - pub fn create_group( + pub async fn create_group( &self, group_base_name: &str, group_name: &str, @@ -107,14 +110,14 @@ impl Farm { let spec = spec.add_meta(group_base_name); let body = CreateGroupRequest { ttl, spec }; let rb = Self::json(self.post(&path), &body); - let rbb = || rb.try_clone().expect("could not clone a request builder"); - let _resp = self.retry_until_success(rbb)?; + let rbb = async move || rb.try_clone().expect("could not clone a request builder"); + let _resp = self.retry_until_success(rbb).await?; Ok(()) } /// creates a vm under the group `group_name` and returns the associated /// IpAddr - pub fn create_vm( + pub async fn create_vm( &self, group_name: &str, mut vm: CreateVmRequest, @@ -125,9 +128,9 @@ impl Farm { .unwrap_or_else(|| vm.required_host_features.clone()); let path = format!("group/{}/vm/{}", group_name, &vm.name); let rb = Self::json(self.post(&path), &vm); - let rbb = || rb.try_clone().expect("could not clone a request builder"); - let resp = self.retry_until_success_long(rbb)?; - let created_vm = resp.json::()?; + let rbb = async move || rb.try_clone().expect("could not clone a request builder"); + let resp = self.retry_until_success_long(rbb).await?; + let created_vm = resp.json::().await?; // Emit a json log event, to be consumed by log post-processing tools. let ipv6 = created_vm.ipv6; emit_vm_created_event( @@ -141,13 +144,13 @@ impl Farm { Ok(created_vm) } - pub fn claim_file(&self, group_name: &str, file_id: &FileId) -> FarmResult { + pub async fn claim_file(&self, group_name: &str, file_id: &FileId) -> FarmResult { let path = format!("group/{group_name}/file/{file_id}"); let rb = self.put(&path); - let rbb = || rb.try_clone().expect("could not clone a request builder"); - match self.retry_until_success(rbb) { + let rbb = async move || rb.try_clone().expect("could not clone a request builder"); + match self.retry_until_success(rbb).await { Ok(resp) => { - let expiration = resp.json::()?; + let expiration = resp.json::().await?; Ok(ClaimResult::FileClaimed(expiration)) } Err(FarmError::NotFound { message: _ }) => Ok(ClaimResult::FileNotFound), @@ -156,7 +159,7 @@ impl Farm { } /// uploads an image an returns the image id - pub fn upload_file>( + pub async fn upload_file>( &self, group_name: &str, path: P, @@ -171,16 +174,18 @@ impl Farm { .post(&format!("group/{group_name}/file")) .timeout(TIMEOUT_SETTINGS_LONG.max_http_timeout); let path = (&path).to_owned(); - let rbb = || { + let rbb = async move || { let form = multipart::Form::new() .file(filename.to_string(), path) + .await .expect("could not create multipart for image"); rb.try_clone() .expect("could not clone a request builder") .multipart(form) }; - let resp = self.retry_until_success_long(rbb)?; - let mut file_ids = resp.json::()?.image_ids; + + let resp = self.retry_until_success_long(rbb).await?; + let mut file_ids = resp.json::().await?.image_ids; if file_ids.len() != 1 || !file_ids.contains_key(filename) { return Err(FarmError::InvalidResponse { message: format!( @@ -191,7 +196,7 @@ impl Farm { Ok(FileId(file_ids.remove(filename).unwrap())) } - pub fn attach_disk_images( + pub async fn attach_disk_images( &self, group_name: &str, vm_name: &str, @@ -204,43 +209,44 @@ impl Farm { drives: image_specs, }; let rb = Self::json(req, &attach_drives_req); - let rbb = || rb.try_clone().expect("could not clone a request builder"); - let _resp = self.retry_until_success_long(rbb)?; + let rbb = async move || rb.try_clone().expect("could not clone a request builder"); + let _resp = self.retry_until_success_long(rbb).await?; Ok(()) } - pub fn start_vm(&self, group_name: &str, vm_name: &str) -> FarmResult<()> { + pub async fn start_vm(&self, group_name: &str, vm_name: &str) -> FarmResult<()> { let path = format!("group/{group_name}/vm/{vm_name}/start"); let rb = self.put(&path); - let rbb = || rb.try_clone().expect("could not clone a request builder"); - let _resp = self.retry_until_success(rbb)?; + let rbb = async move || rb.try_clone().expect("could not clone a request builder"); + let _resp = self.retry_until_success(rbb).await?; let url = self.url_from_path(&format!("group/{group_name}/vm/{vm_name}/console/")[..]); emit_vm_console_link_event(&self.logger, url, vm_name); Ok(()) } - pub fn destroy_vm(&self, group_name: &str, vm_name: &str) -> FarmResult<()> { + pub async fn destroy_vm(&self, group_name: &str, vm_name: &str) -> FarmResult<()> { let path = format!("group/{group_name}/vm/{vm_name}/destroy"); let rb = self.put(&path); - let rbb = || rb.try_clone().expect("could not clone a request builder"); - let _resp = self.retry_until_success(rbb)?; + let rbb = async move || rb.try_clone().expect("could not clone a request builder"); + let _resp = self.retry_until_success(rbb).await?; Ok(()) } - pub fn reboot_vm(&self, group_name: &str, vm_name: &str) -> FarmResult<()> { + pub async fn reboot_vm(&self, group_name: &str, vm_name: &str) -> FarmResult<()> { let path = format!("group/{group_name}/vm/{vm_name}/reboot"); let rb = self.put(&path); - let rbb = || rb.try_clone().expect("could not clone a request builder"); - let _resp = self.retry_until_success(rbb)?; + let rbb = async move || rb.try_clone().expect("could not clone a request builder"); + let _resp = self.retry_until_success(rbb).await?; Ok(()) } // delete with large timeout but only one attempt, because it takes a long time and farm's // garbage collector would interfere with retries. - pub fn delete_group(&self, group_name: &str) { + pub async fn delete_group(&self, group_name: &str) { // bump TTL, so that farm garbage collector does not remove while we remove if self .set_group_ttl(group_name, Duration::from_secs(120)) + .await .is_err() { warn!(self.logger, "Failed to bump TTL before deleting group."); @@ -248,12 +254,12 @@ impl Farm { let path = format!("group/{group_name}"); let mut req = self.delete(&path); req = req.timeout(Duration::from_secs(130)); // longer than VM soft shutdown timeout (120s) - match req.send() { + match req.send().await { Err(e) => error!(self.logger, "Sending a request to Farm failed: {:?}", e), Ok(r) if !r.status().is_success() => warn!( self.logger, "unexpected response from Farm: {:?}", - r.text().unwrap() + r.text().await.unwrap() ), _ => {} }; @@ -262,16 +268,16 @@ impl Farm { /// Creates DNS records under the suffix: `..farm.dfinity.systems`. /// The records will be garbage collected some time after the group has expired. /// The suffix will be returned from this function such that the FQDNs can be constructed. - pub fn create_dns_records( + pub async fn create_dns_records( &self, group_name: &str, dns_records: Vec, ) -> FarmResult { let path = format!("group/{group_name}/dns"); let rb = Self::json(self.post(&path), &dns_records); - let rbb = || rb.try_clone().expect("could not clone a request builder"); - let resp = self.retry_until_success_long(rbb)?; - let create_dns_records_result = resp.json::()?; + let rbb = async move || rb.try_clone().expect("could not clone a request builder"); + let resp = self.retry_until_success_long(rbb).await?; + let create_dns_records_result = resp.json::().await?; Ok(create_dns_records_result.suffix) } @@ -279,24 +285,24 @@ impl Farm { /// where ix is the index of the acquired playnet of the given group. /// The records will be garbage collected some time after the group has expired. /// The suffix will be returned from this function such that the FQDNs can be constructed. - pub fn create_playnet_dns_records( + pub async fn create_playnet_dns_records( &self, group_name: &str, dns_records: Vec, ) -> FarmResult { let path = format!("group/{group_name}/playnet/dns"); let rb = Self::json(self.post(&path), &dns_records); - let rbb = || rb.try_clone().expect("could not clone a request builder"); - let resp = self.retry_until_success_long(rbb)?; - let create_dns_records_result = resp.json::()?; + let rbb = async move || rb.try_clone().expect("could not clone a request builder"); + let resp = self.retry_until_success_long(rbb).await?; + let create_dns_records_result = resp.json::().await?; Ok(create_dns_records_result.suffix) } - pub fn set_group_ttl(&self, group_name: &str, duration: Duration) -> FarmResult<()> { + pub async fn set_group_ttl(&self, group_name: &str, duration: Duration) -> FarmResult<()> { let path = format!("group/{}/ttl/{}", group_name, duration.as_secs()); let rb = self.put(&path); - let rbb = || rb.try_clone().expect("could not clone a request builder"); - let _resp = self.retry_until_success(rbb)?; + let rbb = async move || rb.try_clone().expect("could not clone a request builder"); + let _resp = self.retry_until_success(rbb).await?; Ok(()) } @@ -315,9 +321,13 @@ impl Farm { self.client.delete(url) } - pub fn download_file(&self, url: Url, mut sink: Box) -> FarmResult<()> { - let resp = self.client.get(url).send()?; - sink.write_all(resp.bytes().expect("failed to get bytes").as_ref())?; + pub async fn download_file( + &self, + url: Url, + mut sink: Box, + ) -> FarmResult<()> { + let resp = self.client.get(url).send().await?; + sink.write_all(resp.bytes().await.expect("failed to get bytes").as_ref())?; Ok(()) } @@ -331,36 +341,36 @@ impl Farm { Url::parse(&format!("{}{}", self.base_url, path)).expect("should not fail!") } - fn retry_until_success_long RequestBuilder>( + async fn retry_until_success_long RequestBuilder>( &self, rbb: F, - ) -> FarmResult { - self.retry_until_success_(rbb, TIMEOUT_SETTINGS_LONG) + ) -> FarmResult { + self.retry_until_success_(rbb, TIMEOUT_SETTINGS_LONG).await } - fn retry_until_success RequestBuilder>( + async fn retry_until_success RequestBuilder>( &self, rbb: F, - ) -> FarmResult { - self.retry_until_success_(rbb, TIMEOUT_SETTINGS) + ) -> FarmResult { + self.retry_until_success_(rbb, TIMEOUT_SETTINGS).await } - fn retry_until_success_ RequestBuilder>( + async fn retry_until_success_ RequestBuilder>( &self, rbb: F, t_settings: TimeoutSettings, - ) -> FarmResult { + ) -> FarmResult { let started_at = Instant::now(); let mut req_sent_successfully = false; loop { - let mut req = rbb(); + let mut req = rbb().await; let http_timeout = match t_settings.retry_timeout.checked_sub(started_at.elapsed()) { Some(t) if t > t_settings.min_http_timeout => t.min(t_settings.max_http_timeout), _ => break, }; // cond: MIN_HTTP_REQ_TIMEOUT < http_timeout <= MAX_HTTP_REQ_TIMEOUT req = req.timeout(http_timeout); - match req.send() { + match req.send().await { Err(e) => { req_sent_successfully = false; error!(self.logger, "sending a request to Farm failed: {:?}", e); @@ -371,17 +381,25 @@ impl Farm { return Ok(r); }; if r.status().as_u16() == 404 { - let body = r.text().unwrap_or_default(); + let body = r.text().await.unwrap_or_default(); return Err(FarmError::NotFound { message: body }); } if r.status().is_server_error() { - error!(self.logger, "unexpected response from Farm: {:?}", r.text()); + error!( + self.logger, + "unexpected response from Farm: {:?}", + r.text().await + ); } else { - warn!(self.logger, "unexpected response from Farm: {:?}", r.text()); + warn!( + self.logger, + "unexpected response from Farm: {:?}", + r.text().await + ); } } } - std::thread::sleep(t_settings.linear_backoff); + tokio::time::sleep(t_settings.linear_backoff).await; } Err(FarmError::TooManyRetries { message: String::from(if req_sent_successfully { diff --git a/rs/tests/driver/src/driver/group.rs b/rs/tests/driver/src/driver/group.rs index f8b61c1c8959..efaa745d5de1 100644 --- a/rs/tests/driver/src/driver/group.rs +++ b/rs/tests/driver/src/driver/group.rs @@ -29,6 +29,7 @@ use crate::driver::{ test_env::{TestEnv, TestEnvAttribute}, test_setup::{GroupSetup, InfraProvider}, }; +use crate::util::block_on; use anyhow::{Result, bail}; use chrono::Utc; use clap::Parser; @@ -950,8 +951,8 @@ impl SystemTestGroup { let env = ensure_setup_env(ctx); let group_setup = GroupSetup::read_attribute(&env); let farm_url = env.get_farm_url().unwrap(); - let farm = Farm::new(farm_url, env.logger()); + let farm = block_on(Farm::new(farm_url, env.logger())); let group_name = group_setup.infra_group_name; - farm.delete_group(&group_name); + block_on(farm.delete_group(&group_name)); } } diff --git a/rs/tests/driver/src/driver/ic.rs b/rs/tests/driver/src/driver/ic.rs index aba4b39e2ce4..9adbbf616c07 100644 --- a/rs/tests/driver/src/driver/ic.rs +++ b/rs/tests/driver/src/driver/ic.rs @@ -8,6 +8,7 @@ use crate::driver::{ test_env_api::{HasRegistryLocalStore, HasTopologySnapshot}, test_setup::GroupSetup, }; +use crate::util::block_on; use anyhow::Result; use ic_prep_lib::prep_state_directory::IcPrepStateDir; use ic_prep_lib::{node::NodeSecretKeyStore, subnet_configuration::SubnetRunningState}; @@ -233,7 +234,7 @@ impl InternetComputer { env: &TestEnv, ) -> Result> { // propagate required host features and resource settings to all vms - let farm = Farm::from_test_env(env, "Internet Computer"); + let farm = block_on(Farm::from_test_env(env, "Internet Computer")); for node in self .subnets .iter_mut() diff --git a/rs/tests/driver/src/driver/ic_gateway_vm.rs b/rs/tests/driver/src/driver/ic_gateway_vm.rs index 29802f3042bc..3f73b2734302 100644 --- a/rs/tests/driver/src/driver/ic_gateway_vm.rs +++ b/rs/tests/driver/src/driver/ic_gateway_vm.rs @@ -125,7 +125,7 @@ impl IcGatewayVm { let playnet = self.load_or_create_playnet(env, vm_ipv6, vm_ipv4)?; let ic_gateway_fqdn = playnet.playnet_cert.playnet.clone(); - self.configure_dns_records(env, &playnet, &ic_gateway_fqdn)?; + block_on(self.configure_dns_records(env, &playnet, &ic_gateway_fqdn))?; // Emit log events for A and AAAA records emit_ic_gateway_records_event(&logger, &ic_gateway_fqdn, &playnet); @@ -197,7 +197,7 @@ impl IcGatewayVm { } /// Configures DNS records based on infrastructure provider. - fn configure_dns_records( + async fn configure_dns_records( &self, env: &TestEnv, playnet: &Playnet, @@ -233,10 +233,10 @@ impl IcGatewayVm { }) } - let base_domain = env.create_playnet_dns_records(records); + let base_domain = env.create_playnet_dns_records(records).await; // Wait for DNS propagation by checking a random subdomain - block_on(await_dns_propagation(&env.logger(), &base_domain))?; + await_dns_propagation(&env.logger(), &base_domain).await?; Ok(()) } diff --git a/rs/tests/driver/src/driver/keepalive_task.rs b/rs/tests/driver/src/driver/keepalive_task.rs index 6befba36f985..34c8a1f3d1df 100644 --- a/rs/tests/driver/src/driver/keepalive_task.rs +++ b/rs/tests/driver/src/driver/keepalive_task.rs @@ -6,6 +6,7 @@ use crate::driver::{ test_env_api::HasFarmUrl, test_setup::GroupSetup, }; +use crate::util::block_on; use slog::{debug, info}; pub(crate) const KEEPALIVE_TASK_NAME: &str = "keepalive"; @@ -21,9 +22,9 @@ pub(crate) fn keepalive_task(group_ctx: GroupContext) -> () { match GroupSetup::try_read_attribute(&env) { Ok(group_setup) => { let farm_url = env.get_farm_url().unwrap(); - let farm = Farm::new(farm_url.clone(), env.logger()); + let farm = block_on(Farm::new(farm_url.clone(), env.logger())); let group_name = group_setup.infra_group_name; - if let Err(e) = farm.set_group_ttl(&group_name, GROUP_TTL) { + if let Err(e) = block_on(farm.set_group_ttl(&group_name, GROUP_TTL)) { panic!( "{}", format!( diff --git a/rs/tests/driver/src/driver/nested.rs b/rs/tests/driver/src/driver/nested.rs index 7a52fc2d0061..6840c242a8a9 100644 --- a/rs/tests/driver/src/driver/nested.rs +++ b/rs/tests/driver/src/driver/nested.rs @@ -11,7 +11,7 @@ use crate::driver::{ test_env::TestEnvAttribute, test_setup::GroupSetup, }; -use crate::util::create_agent; +use crate::util::{block_on, create_agent}; use ic_agent::{Agent, AgentError}; use std::fs; @@ -55,7 +55,7 @@ impl NestedNodes { } pub fn setup_and_start(&mut self, env: &TestEnv) -> Result<()> { - let farm = Farm::from_test_env(env, "Internet Computer"); + let farm = block_on(Farm::from_test_env(env, "Internet Computer")); let group_setup = GroupSetup::read_attribute(env); let group_name: String = group_setup.infra_group_name; diff --git a/rs/tests/driver/src/driver/resource.rs b/rs/tests/driver/src/driver/resource.rs index 394aeb1e813d..3b8bf44ba146 100644 --- a/rs/tests/driver/src/driver/resource.rs +++ b/rs/tests/driver/src/driver/resource.rs @@ -14,6 +14,7 @@ use crate::driver::test_env_api::{ }; use crate::driver::test_setup::{GroupSetup, InfraProvider}; use crate::driver::universal_vm::UniversalVm; +use crate::util::block_on; use anyhow; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; @@ -283,7 +284,7 @@ pub fn allocate_resources( threads.push(std::thread::spawn(move || { ( vm_name, - farm_cloned.create_vm(&group_name, create_vm_request), + block_on(farm_cloned.create_vm(&group_name, create_vm_request)), ) })); } diff --git a/rs/tests/driver/src/driver/test_env_api.rs b/rs/tests/driver/src/driver/test_env_api.rs index e75475e02113..3f4dd22412ba 100644 --- a/rs/tests/driver/src/driver/test_env_api.rs +++ b/rs/tests/driver/src/driver/test_env_api.rs @@ -1352,19 +1352,19 @@ impl HasGroupSetup for TestEnv { match InfraProvider::read_attribute(self) { InfraProvider::Farm => { let farm_base_url = FarmBaseUrl::read_attribute(self); - let farm = Farm::new(farm_base_url.into(), self.logger()); + let farm = block_on(Farm::new(farm_base_url.into(), self.logger())); let group_spec = GroupSpec { vm_allocation: None, required_host_features: vec![], preferred_network: None, metadata: None, }; - farm.create_group( + block_on(farm.create_group( &group_setup.group_base_name, &group_setup.infra_group_name, group_setup.group_timeout, group_spec, - ) + )) .unwrap(); } }; @@ -2050,10 +2050,11 @@ impl IcNodeContainer for SubnetSnapshot { /* ### VM Control ### */ +#[async_trait] pub trait VmControl { - fn kill(&self); - fn reboot(&self); - fn start(&self); + async fn kill(&self); + async fn reboot(&self); + async fn start(&self); } pub struct HostedVm { @@ -2065,41 +2066,47 @@ pub struct HostedVm { /// VmControl enables a user to interact with VMs, i.e. change their state. /// All functions belonging to this trait crash if a respective operation is for any reason /// unsuccessful. +#[async_trait] impl VmControl for HostedVm { - fn kill(&self) { + async fn kill(&self) { self.farm .destroy_vm(&self.group_name, &self.vm_name) + .await .expect("could not kill VM"); } - fn reboot(&self) { + async fn reboot(&self) { self.farm .reboot_vm(&self.group_name, &self.vm_name) + .await .expect("could not reboot VM"); } - fn start(&self) { + async fn start(&self) { self.farm .start_vm(&self.group_name, &self.vm_name) + .await .expect("could not start VM"); } } +#[async_trait] pub trait HasVm { /// Returns a handle used for controlling a VM, i.e. starting, stopping and rebooting. - fn vm(&self) -> Box; + async fn vm(&self) -> Box; } +#[async_trait] impl HasVm for T where - T: HasTestEnv + HasVmName, + T: HasTestEnv + HasVmName + Sync, { /// Returns a handle used for controlling a VM, i.e. starting, stopping and rebooting. - fn vm(&self) -> Box { + async fn vm(&self) -> Box { let env = self.test_env(); let pot_setup = GroupSetup::read_attribute(&env); let farm_base_url = self.get_farm_url().unwrap(); - let farm = Farm::new(farm_base_url, env.logger()); + let farm = Farm::new(farm_base_url, env.logger()).await; let vm_name = self.vm_name(); Box::new(HostedVm { @@ -2493,35 +2500,38 @@ where let env = self.test_env(); let log = env.logger(); let farm_base_url = self.get_farm_url().unwrap(); - let farm = Farm::new(farm_base_url, log); + let farm = block_on(Farm::new(farm_base_url, log)); let group_setup = GroupSetup::read_attribute(&env); let group_name = group_setup.infra_group_name; - farm.create_dns_records(&group_name, dns_records) + block_on(farm.create_dns_records(&group_name, dns_records)) .expect("Failed to create DNS records") } } +#[async_trait] pub trait CreatePlaynetDnsRecords { /// Creates DNS records under the suffix: `.ic{ix}.farm.dfinity.systems` /// where `ix` is the index of the acquired playnet. /// /// The records will be garbage collected some time after the group has expired. /// The suffix will be returned from this function such that the FQDNs can be constructed. - fn create_playnet_dns_records(&self, dns_records: Vec) -> String; + async fn create_playnet_dns_records(&self, dns_records: Vec) -> String; } +#[async_trait] impl CreatePlaynetDnsRecords for T where - T: HasTestEnv, + T: HasTestEnv + std::marker::Sync, { - fn create_playnet_dns_records(&self, dns_records: Vec) -> String { + async fn create_playnet_dns_records(&self, dns_records: Vec) -> String { let env = self.test_env(); let log = env.logger(); let farm_base_url = self.get_farm_url().unwrap(); - let farm = Farm::new(farm_base_url, log); + let farm = Farm::new(farm_base_url, log).await; let group_setup = GroupSetup::read_attribute(&env); let group_name = group_setup.infra_group_name; farm.create_playnet_dns_records(&group_name, dns_records) + .await .expect("Failed to create playnet DNS records") } } @@ -2540,10 +2550,10 @@ where let env = self.test_env(); let log = env.logger(); let farm_base_url = self.get_farm_url().unwrap(); - let farm = Farm::new(farm_base_url, log); + let farm = block_on(Farm::new(farm_base_url, log)); let group_setup = GroupSetup::read_attribute(&env); let group_name = group_setup.infra_group_name; - farm.acquire_playnet_certificate(&group_name) + block_on(farm.acquire_playnet_certificate(&group_name)) .expect("Failed to acquire a certificate for a playnet") } } diff --git a/rs/tests/driver/src/driver/universal_vm.rs b/rs/tests/driver/src/driver/universal_vm.rs index fd864304abe1..cdec9936fcfd 100644 --- a/rs/tests/driver/src/driver/universal_vm.rs +++ b/rs/tests/driver/src/driver/universal_vm.rs @@ -14,6 +14,7 @@ use crate::driver::test_env::SshKeyGen; use crate::driver::test_env::{TestEnv, TestEnvAttribute}; use crate::driver::test_env_api::{HasTestEnv, HasVmName, RetrieveIpv4Addr, SshSession}; use crate::driver::test_setup::{GroupSetup, InfraProvider}; +use crate::util::block_on; use anyhow::{Result, bail}; use chrono::Duration; use chrono::Utc; @@ -108,7 +109,7 @@ impl UniversalVm { } pub fn start(&self, env: &TestEnv) -> Result<()> { - let farm = Farm::from_test_env(env, "universal VM"); + let farm = block_on(Farm::from_test_env(env, "universal VM")); let pot_setup = GroupSetup::read_attribute(env); env.ssh_keygen()?; @@ -132,11 +133,11 @@ impl UniversalVm { let config_ssh_img = universal_vm_dir.join(CONF_SSH_IMG_FNAME); create_universal_vm_config_image(&config_ssh_dir, &config_ssh_img, "SSH")?; - let ssh_config_img_file_spec = AttachImageSpec::new(farm.upload_file( + let ssh_config_img_file_spec = AttachImageSpec::new(block_on(farm.upload_file( &pot_setup.infra_group_name, config_ssh_img, CONF_SSH_IMG_FNAME, - )?); + ))?); image_specs.push(ssh_config_img_file_spec); } @@ -156,7 +157,8 @@ impl UniversalVm { let file_id = id_of_file(config_img.clone())?; let mut file_spec = AttachImageSpec::new(file_id.clone()); - let upload = match farm.claim_file(&pot_setup.infra_group_name, &file_id)? { + let upload = match block_on(farm.claim_file(&pot_setup.infra_group_name, &file_id))? + { ClaimResult::FileClaimed(file_expiration) => { if let Some(expiration) = file_expiration.expiration { let now = Utc::now(); @@ -174,11 +176,11 @@ impl UniversalVm { }; if upload { - file_spec = AttachImageSpec::new(farm.upload_file( + file_spec = AttachImageSpec::new(block_on(farm.upload_file( &pot_setup.infra_group_name, config_img, CONF_IMG_FNAME, - )?); + ))?); info!(env.logger(), "Uploaded image: {}", file_id); } else { info!( @@ -191,13 +193,13 @@ impl UniversalVm { } if InfraProvider::read_attribute(env) == InfraProvider::Farm { - farm.attach_disk_images( + block_on(farm.attach_disk_images( &pot_setup.infra_group_name, &self.name, "usb-storage", image_specs, - )?; - farm.start_vm(&pot_setup.infra_group_name, &self.name)?; + ))?; + block_on(farm.start_vm(&pot_setup.infra_group_name, &self.name))?; } Ok(()) diff --git a/rs/tests/message_routing/global_reboot_test.rs b/rs/tests/message_routing/global_reboot_test.rs index e0bfb7bd32c4..27a072b55710 100644 --- a/rs/tests/message_routing/global_reboot_test.rs +++ b/rs/tests/message_routing/global_reboot_test.rs @@ -123,7 +123,7 @@ pub fn test_on_subnets(env: TestEnv, subnets: Vec) { // Step 5: Reboot all nodes and wait till they become reachable again. info!(log, "Rebooting all nodes ..."); for n in all_nodes.iter().cloned() { - n.vm().reboot(); + block_on(async { n.vm().await.reboot().await }); assert_nodes_health_statuses(log.clone(), &[n], EndpointsStatus::AllUnhealthy); } info!(log, "Waiting for endpoints to be reachable again ..."); diff --git a/rs/tests/message_routing/memory_safety_test.rs b/rs/tests/message_routing/memory_safety_test.rs index d3b372a75c9b..58d9f4a766f3 100644 --- a/rs/tests/message_routing/memory_safety_test.rs +++ b/rs/tests/message_routing/memory_safety_test.rs @@ -25,15 +25,13 @@ use ic_agent::export::Principal; use ic_base_types::PrincipalId; use ic_registry_subnet_type::SubnetType; use ic_system_test_driver::driver::group::SystemTestGroup; -use ic_system_test_driver::systest; -use ic_system_test_driver::{ - driver::{ - ic::{InternetComputer, Subnet}, - test_env::TestEnv, - test_env_api::{HasPublicApiUrl, HasTopologySnapshot, HasVm, IcNodeContainer}, - }, - util::*, +use ic_system_test_driver::driver::{ + ic::{InternetComputer, Subnet}, + test_env::TestEnv, + test_env_api::{HasPublicApiUrl, HasTopologySnapshot, HasVm, IcNodeContainer}, }; +use ic_system_test_driver::systest; +use ic_system_test_driver::util::block_on; use ic_types::Height; use ic_utils::interfaces::ManagementCanister; use rand::Rng; @@ -90,7 +88,7 @@ pub fn test(env: TestEnv) { modify_mem_and_verify(&mut rng, &canister_id, &agent, i as u8).await }); info!(log, "restarting the node with id={}", node.node_id); - node.vm().reboot(); + block_on(async { node.vm().await.reboot().await }); // Wait until the re-started node becomes ready. info!(log, "waiting for the node to be become ready..."); node.await_status_is_healthy().unwrap(); diff --git a/rs/tests/message_routing/rejoin_test_lib/rejoin_test_lib.rs b/rs/tests/message_routing/rejoin_test_lib/rejoin_test_lib.rs index e4f1c1e21ede..78b5d00750d7 100644 --- a/rs/tests/message_routing/rejoin_test_lib/rejoin_test_lib.rs +++ b/rs/tests/message_routing/rejoin_test_lib/rejoin_test_lib.rs @@ -73,7 +73,7 @@ pub async fn rejoin_test( "Killing a node: {} ...", rejoin_node.get_public_url() ); - rejoin_node.vm().kill(); + rejoin_node.vm().await.kill().await; rejoin_node .await_status_is_unavailable() .expect("Node still healthy"); @@ -88,14 +88,14 @@ pub async fn rejoin_test( info!(logger, "Killing {} nodes ...", allowed_failures); for node_to_kill in nodes_to_kill { info!(logger, "Killing node {} ...", node_to_kill.get_public_url()); - node_to_kill.vm().kill(); + node_to_kill.vm().await.kill().await; node_to_kill .await_status_is_unavailable() .expect("Node still healthy"); } info!(logger, "Start the first killed node again..."); - rejoin_node.vm().start(); + rejoin_node.vm().await.start().await; rejoin_node .await_status_is_healthy() .expect("Started node did not report healthy status"); @@ -167,7 +167,7 @@ pub async fn rejoin_test_large_state( "Killing a node: {} ...", rejoin_node.get_public_url() ); - rejoin_node.vm().kill(); + rejoin_node.vm().await.kill().await; rejoin_node .await_status_is_unavailable() .expect("Node still healthy"); @@ -204,14 +204,14 @@ pub async fn rejoin_test_large_state( info!(logger, "Killing {} nodes ...", allowed_failures); for node_to_kill in nodes_to_kill { info!(logger, "Killing node {} ...", node_to_kill.get_public_url()); - node_to_kill.vm().kill(); + node_to_kill.vm().await.kill().await; node_to_kill .await_status_is_unavailable() .expect("Node still healthy"); } info!(logger, "Start the first killed node again..."); - rejoin_node.vm().start(); + rejoin_node.vm().await.start().await; rejoin_node .await_status_is_healthy() .expect("Started node did not report healthy status"); @@ -374,7 +374,7 @@ pub async fn rejoin_test_long_rounds( "Killing a node: {} ...", rejoin_node.get_public_url() ); - rejoin_node.vm().kill(); + rejoin_node.vm().await.kill().await; rejoin_node .await_status_is_unavailable() .expect("Node still healthy"); @@ -394,7 +394,7 @@ pub async fn rejoin_test_long_rounds( wait_for_cup(&logger, latest_certified_height, reference_node.clone()).await; info!(logger, "Start the killed node again ..."); - rejoin_node.vm().start(); + rejoin_node.vm().await.start().await; info!(logger, "Waiting for the next CUP ..."); let last_cup_height = wait_for_cup( diff --git a/rs/tests/networking/canister_http_fault_tolerance_test.rs b/rs/tests/networking/canister_http_fault_tolerance_test.rs index 6722afe0e25a..cb704569502f 100644 --- a/rs/tests/networking/canister_http_fault_tolerance_test.rs +++ b/rs/tests/networking/canister_http_fault_tolerance_test.rs @@ -34,6 +34,7 @@ use ic_system_test_driver::driver::test_env_api::{ }; use ic_system_test_driver::systest; use ic_system_test_driver::util; +use ic_system_test_driver::util::block_on; use ic_types::{CanisterId, PrincipalId}; use ic_utils::interfaces::ManagementCanister; use proxy_canister::UnvalidatedCanisterHttpRequestArgs; @@ -179,7 +180,7 @@ pub fn test(env: TestEnv) { }); info!(&logger, "Killing one of the node now."); - killed_app_endpoint.vm().kill(); + block_on(async { killed_app_endpoint.vm().await.kill().await }); // Wait the node is actually killed let http_client = reqwest::blocking::ClientBuilder::new() @@ -205,7 +206,7 @@ pub fn test(env: TestEnv) { // Recover the killed node and observe it caught up on state info!(&logger, "Restarting the killed node now."); - killed_app_endpoint.vm().start(); + block_on(async { killed_app_endpoint.vm().await.start().await }); let healthy_runtime = &util::runtime_from_url( healthy_app_endpoint.get_public_url(), healthy_app_endpoint.effective_canister_id(), diff --git a/rs/tests/networking/network_large_test.rs b/rs/tests/networking/network_large_test.rs index af763e914a63..adb2e618fa86 100644 --- a/rs/tests/networking/network_large_test.rs +++ b/rs/tests/networking/network_large_test.rs @@ -133,7 +133,7 @@ pub fn test(env: TestEnv) { info!(log, "Step 5: Kill {} nodes", FAULTY); let nodes: Vec<_> = subnet.nodes().collect(); for node in nodes.iter().take(FAULTY) { - node.vm().kill(); + block_on(async { node.vm().await.kill().await }); } for node in nodes.iter().take(FAULTY) { node.await_status_is_unavailable() @@ -155,7 +155,7 @@ pub fn test(env: TestEnv) { "Step 7: Kill an additonal node causing consensus to stop due to {} (f+1) faulty nodes", FAULTY + 1 ); - nodes[FAULTY].vm().kill(); + block_on(async { nodes[FAULTY].vm().await.kill().await }); nodes[FAULTY] .await_status_is_unavailable() .expect("Node still healthy"); @@ -172,7 +172,7 @@ pub fn test(env: TestEnv) { }; info!(log, "Step 8: Restart one node again",); - nodes[FAULTY].vm().start(); + block_on(async { nodes[FAULTY].vm().await.start().await }); for n in nodes.iter().skip(FAULTY) { n.await_status_is_healthy().unwrap(); } diff --git a/rs/tests/node/kill_start_test.rs b/rs/tests/node/kill_start_test.rs index 77c01b4cd887..d07c128a7228 100644 --- a/rs/tests/node/kill_start_test.rs +++ b/rs/tests/node/kill_start_test.rs @@ -14,6 +14,7 @@ use ic_system_test_driver::driver::ic::InternetComputer; use ic_system_test_driver::driver::test_env::TestEnv; use ic_system_test_driver::driver::test_env_api::*; use ic_system_test_driver::systest; +use ic_system_test_driver::util::block_on; use nested::util::block_on_bash_script_and_log; use slog::{error, info}; use std::time::Duration; @@ -31,7 +32,7 @@ fn test(env: TestEnv) { let log = &env.logger(); let node = &env.get_first_healthy_system_node_snapshot(); let node_id = node.node_id; - let vm = node.vm(); + let vm = block_on(node.vm()); let num_kill_start_iterations = std::env::var("NUM_KILL_START_ITERATIONS") .expect("NUM_KILL_START_ITERATIONS not set") .parse::() @@ -44,7 +45,7 @@ fn test(env: TestEnv) { node.await_status_is_healthy().expect("Node not healthy"); info!(log, "Killing node: {node_id} ..."); - vm.kill(); + block_on(vm.kill()); node.await_status_is_unavailable() .expect("Node still healthy"); @@ -52,7 +53,7 @@ fn test(env: TestEnv) { std::thread::sleep(POST_KILL_SLEEP_DURATION); info!(log, "Starting node: {node_id} ..."); - vm.start(); + block_on(vm.start()); if let Err(err) = node.await_status_is_healthy() { error!( log, diff --git a/rs/tests/node/reboot_toy.rs b/rs/tests/node/reboot_toy.rs index dc0091529d52..b4109ea14bf8 100644 --- a/rs/tests/node/reboot_toy.rs +++ b/rs/tests/node/reboot_toy.rs @@ -3,6 +3,7 @@ use ic_registry_subnet_type::SubnetType; use ic_system_test_driver::{ driver::{group::SystemTestGroup, ic::InternetComputer, test_env::TestEnv, test_env_api::*}, systest, + util::block_on, }; use anyhow::Result; @@ -57,14 +58,14 @@ EOT } // Reboot once to get things fully initialized - node.vm().reboot(); + block_on(async { node.vm().await.reboot().await }); node.await_status_is_unavailable().unwrap(); node.await_status_is_healthy().unwrap(); // Reboot again to measure timing let reboot_start = Instant::now(); - node.vm().reboot(); + block_on(async { node.vm().await.reboot().await }); node.await_status_is_unavailable().unwrap(); node.await_status_is_healthy().unwrap();