Minimizing node restarts in local swarm smoke tests (#2098)

- Allowing setting (per-node) validator configuration,
  so that we don't need to restart the node to do so
- dissallowing starting already running nodes (which restarts them silently)

Test plan:
- all smoke tests. added test for not starting a node twice

crates/aptos-genesis/src/builder.rs

@@ -31,6 +31,7 @@ use std::{
     io::{Read, Write},
     num::NonZeroUsize,
     path::{Path, PathBuf},
+    sync::Arc,
     time::{SystemTime, UNIX_EPOCH},
 };
 
@@ -359,6 +360,8 @@ fn write_yaml<T: Serialize>(path: &Path, object: &T) -> anyhow::Result<()> {
 const ONE_DAY: u64 = 86400;
 const ONE_YEAR: u64 = 31536000;
 
+pub type InitConfigFn = Arc<dyn Fn(usize, &mut NodeConfig) + Send + Sync>;
+
 /// Builder that builds a network of validator nodes that can run locally
 #[derive(Clone)]
 pub struct Builder {
@@ -367,6 +370,7 @@ pub struct Builder {
     num_validators: NonZeroUsize,
     randomize_first_validator_ports: bool,
     template: NodeConfig,
+    init_config: Option<InitConfigFn>,
     min_price_per_gas_unit: u64,
     allow_new_validators: bool,
     min_stake: u64,
@@ -395,6 +399,7 @@ impl Builder {
             num_validators: NonZeroUsize::new(1).unwrap(),
             randomize_first_validator_ports: true,
             template: NodeConfig::default_for_validator(),
+            init_config: None,
             min_price_per_gas_unit: 1,
             allow_new_validators: false,
             min_stake: 0,
@@ -421,6 +426,11 @@ impl Builder {
         self
     }
 
+    pub fn with_init_config(mut self, init_config: Option<InitConfigFn>) -> Self {
+        self.init_config = init_config;
+        self
+    }
+
     pub fn with_min_price_per_gas_unit(mut self, min_price_per_gas_unit: u64) -> Self {
         self.min_price_per_gas_unit = min_price_per_gas_unit;
         self
@@ -506,8 +516,12 @@ impl Builder {
     {
         let name = index.to_string();
 
-        let mut validator =
-            ValidatorNodeConfig::new(name, self.config_dir.as_path(), self.template.clone())?;
+        let mut config = self.template.clone();
+        if let Some(init_config) = &self.init_config {
+            (init_config)(index, &mut config);
+        }
+
+        let mut validator = ValidatorNodeConfig::new(name, self.config_dir.as_path(), config)?;
 
         validator.init_keys(Some(rng.gen()))?;
 

testsuite/forge/src/backend/local/mod.rs

@@ -3,6 +3,7 @@
 
 use crate::{Factory, GenesisConfig, Result, Swarm, Version};
 use anyhow::{bail, Context};
+use aptos_genesis::builder::InitConfigFn;
 use rand::rngs::StdRng;
 use std::{
     collections::HashMap,
@@ -112,7 +113,7 @@ impl LocalFactory {
         R: ::rand::RngCore + ::rand::CryptoRng,
     {
         let version = self.versions.keys().max().unwrap();
-        self.new_swarm_with_version(rng, number_of_validators, version, None, 1)
+        self.new_swarm_with_version(rng, number_of_validators, version, None, 1, None)
             .await
     }
 
@@ -123,14 +124,16 @@ impl LocalFactory {
         version: &Version,
         genesis_modules: Option<Vec<Vec<u8>>>,
         min_price_per_gas_unit: u64,
+        init_config: Option<InitConfigFn>,
     ) -> Result<LocalSwarm>
     where
         R: ::rand::RngCore + ::rand::CryptoRng,
     {
         let mut builder = LocalSwarm::builder(self.versions.clone())
             .number_of_validators(number_of_validators)
             .initial_version(version.clone())
-            .min_price_per_gas_unit(min_price_per_gas_unit);
+            .min_price_per_gas_unit(min_price_per_gas_unit)
+            .with_init_config(init_config);
         if let Some(genesis_modules) = genesis_modules {
             builder = builder.genesis_modules(genesis_modules);
         }
@@ -169,7 +172,7 @@ impl Factory for LocalFactory {
             None => None,
         };
         let swarm = self
-            .new_swarm_with_version(rng, node_num, version, genesis_modules, 1)
+            .new_swarm_with_version(rng, node_num, version, genesis_modules, 1, None)
             .await?;
 
         Ok(Box::new(swarm))

testsuite/forge/src/backend/local/node.rs

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use crate::{FullNode, HealthCheckError, LocalVersion, Node, NodeExt, Validator, Version};
-use anyhow::{anyhow, Context, Result};
+use anyhow::{anyhow, ensure, Context, Result};
 use aptos_config::config::NodeConfig;
 use aptos_logger::debug;
 use aptos_sdk::types::{account_address::AccountAddress, PeerId};
@@ -81,6 +81,8 @@ impl LocalNode {
     }
 
     pub fn start(&mut self) -> Result<()> {
+        ensure!(self.process.is_none(), "node {} already running", self.name);
+
         // Ensure log file exists
         let log_file = OpenOptions::new()
             .create(true)

testsuite/forge/src/backend/local/swarm.rs

@@ -7,7 +7,7 @@ use crate::{
 };
 use anyhow::{anyhow, bail, Result};
 use aptos_config::{config::NodeConfig, keys::ConfigKey};
-use aptos_genesis::builder::FullnodeNodeConfig;
+use aptos_genesis::builder::{FullnodeNodeConfig, InitConfigFn};
 use aptos_sdk::{
     crypto::ed25519::Ed25519PrivateKey,
     types::{
@@ -76,6 +76,7 @@ pub struct LocalSwarmBuilder {
     versions: Arc<HashMap<Version, LocalVersion>>,
     initial_version: Option<Version>,
     template: NodeConfig,
+    init_config: Option<InitConfigFn>,
     number_of_validators: NonZeroUsize,
     dir: Option<PathBuf>,
     genesis_modules: Option<Vec<Vec<u8>>>,
@@ -88,6 +89,7 @@ impl LocalSwarmBuilder {
             versions,
             initial_version: None,
             template: NodeConfig::default_for_validator(),
+            init_config: None,
             number_of_validators: NonZeroUsize::new(1).unwrap(),
             dir: None,
             genesis_modules: None,
@@ -105,6 +107,11 @@ impl LocalSwarmBuilder {
         self
     }
 
+    pub fn with_init_config(mut self, init_config: Option<InitConfigFn>) -> Self {
+        self.init_config = init_config;
+        self
+    }
+
     pub fn number_of_validators(mut self, number_of_validators: NonZeroUsize) -> Self {
         self.number_of_validators = number_of_validators;
         self
@@ -129,6 +136,7 @@ impl LocalSwarmBuilder {
     where
         R: ::rand::RngCore + ::rand::CryptoRng,
     {
+        println!("Building a new swarm");
         let dir = if let Some(dir) = self.dir {
             if dir.exists() {
                 fs::remove_dir_all(&dir)?;
@@ -154,6 +162,7 @@ impl LocalSwarmBuilder {
             )?
             .with_num_validators(self.number_of_validators)
             .with_template(self.template)
+            .with_init_config(self.init_config)
             .with_min_price_per_gas_unit(self.min_price_per_gas_unit)
             .with_min_lockup_duration_secs(0)
             .with_max_lockup_duration_secs(86400)
@@ -198,6 +207,7 @@ impl LocalSwarmBuilder {
             root_account,
             chain_id: ChainId::test(),
             root_key,
+            launched: false,
         })
     }
 }
@@ -214,6 +224,8 @@ pub struct LocalSwarm {
     root_account: LocalAccount,
     chain_id: ChainId,
     root_key: ConfigKey<Ed25519PrivateKey>,
+
+    launched: bool,
 }
 
 impl LocalSwarm {
@@ -222,18 +234,28 @@ impl LocalSwarm {
     }
 
     pub async fn launch(&mut self) -> Result<()> {
+        if self.launched {
+            return Err(anyhow!("Swarm already launched"));
+        }
+        self.launched = true;
+
         // Start all the validators
         for validator in self.validators.values_mut() {
             validator.start()?;
         }
 
+        self.wait_all_alive(Duration::from_secs(60)).await?;
+        println!("Swarm launched successfully.");
+        Ok(())
+    }
+
+    pub async fn wait_all_alive(&mut self, timeout: Duration) -> Result<()> {
         // Wait for all of them to startup
-        let deadline = Instant::now() + Duration::from_secs(60);
+        let deadline = Instant::now() + timeout;
         self.wait_for_startup().await?;
         self.wait_for_connectivity(deadline).await?;
         self.liveness_check(deadline).await?;
-
-        println!("Swarm launched successfully.");
+        println!("Swarm alive.");
         Ok(())
     }