diff --git a/Cargo.lock b/Cargo.lock index b1b0ca3..9e73d20 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7135,6 +7135,7 @@ dependencies = [ "reqwest", "rustc_version", "rustls", + "solana-accounts-db", "solana-core", "solana-ledger", "solana-logger", diff --git a/Cargo.toml b/Cargo.toml index 1443be6..3b630c4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ log = "0.4.21" rand = "0.8.5" reqwest = { version = "0.11.23", features = ["blocking", "brotli", "deflate", "gzip", "rustls-tls", "json"] } rustls = { version = "0.21.11", default-features = false, features = ["quic"] } +solana-accounts-db = "1.18.8" solana-core = "1.18.8" solana-ledger = "1.18.8" solana-logger = "1.18.8" diff --git a/PROGRESS.md b/PROGRESS.md index bad7044..cf52123 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -15,22 +15,24 @@ - [x] Create Genesis - [x] Generate faucet and bootstrap accounts - [x] Build genesis -- [x] Docker Build - - [x] Build Bootstrap Image - - [x] Push Image to registry +- [x] Docker Build & Push + - [x] Bootstrap + - [x] Validator (regular) + - [ ] RPC nodes + - [ ] Client - [ ] Create & Deploy Secrets - [x] Bootstrap - - [ ] Validator (regular) + - [x] Validator (regular) - [ ] RPC nodes - [ ] Client - [ ] Create & Deploy Selector - [x] Bootstrap - - [ ] Validator (regular) + - [x] Validator (regular) - [ ] RPC nodes - [ ] Client - [ ] Create & Deploy Replica Set - [x] Bootstrap - - [ ] Validator (regular) + - [x] Validator (regular) - [ ] RPC nodes - [ ] Client - [ ] Create & Deploy Services @@ -42,11 +44,11 @@ - [x] Build and deploy Load Balancer (sits in front of bootstrap and RPC nodes) - [ ] Add metrics - [x] Bootstrap - - [ ] Validator (regular) + - [x] Validator (regular) - [ ] RPC nodes - [ ] Client - [ ] Create accounts - - [ ] Validator (regular) + - [x] Validator (regular) - [ ] RPC - [ ] Client - [ ] Add feature flags to configure: diff --git a/README.md b/README.md index 5c922a4..907a854 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,6 @@ kubectl create ns cargo run --bin cluster -- -n --local-path - --validator-lab-dir ``` #### Build specific Agave release @@ -35,16 +34,15 @@ cargo run --bin cluster -- cargo run --bin cluster -- -n --release-channel # note: MUST include the "v" - --validator-lab-dir ``` -#### Build from Local Repo and Configure Genesis and Bootstrap Validator Image +#### Build from Local Repo and Configure Genesis and Bootstrap and Validator Image Example: ``` cargo run --bin cluster -- -n --local-path /home/sol/solana - --validator-lab-dir /home/sol/validator-lab + --num_validators # genesis config. Optional: Many of these have defaults --hashes-per-tick --enable-warmup-epochs @@ -59,14 +57,20 @@ cargo run --bin cluster -- --tag # e.g. v1 --base-image # e.g. ubuntu:20.04 --image-name # e.g. cluster-image + # validator config + --full-rpc + --internal-node-sol + --internal-node-stake-sol + # kubernetes config + --cpu-requests + --memory-requests ``` ## Metrics 1) Setup metrics database: ``` -cd scripts/ -./init-metrics -c -# enter password when promted +./init-metrics -c -u +# enter password when prompted ``` 2) add the following to your `cluster` command from above ``` diff --git a/init-metrics b/init-metrics new file mode 100755 index 0000000..fa3df04 Binary files /dev/null and b/init-metrics differ diff --git a/scripts/init-metrics.sh b/scripts/init-metrics.sh deleted file mode 100755 index ddd5aaa..0000000 --- a/scripts/init-metrics.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env bash -set -e - -here=$(dirname "$0") - -# https://gist.github.com/cdown/1163649 -urlencode() { - declare s="$1" - declare l=$((${#s} - 1)) - for i in $(seq 0 $l); do - declare c="${s:$i:1}" - case $c in - [a-zA-Z0-9.~_-]) - echo -n "$c" - ;; - *) - printf '%%%02X' "'$c" - ;; - esac - done -} - -usage() { - exitcode=0 - if [[ -n "$1" ]]; then - exitcode=1 - echo "Error: $*" - fi - cat <, - _validator: Option, + validator: Option, _rpc: Option, _clients: Vec, } @@ -21,6 +21,7 @@ impl ClusterImages { pub fn set_item(&mut self, item: Validator, validator_type: ValidatorType) { match validator_type { ValidatorType::Bootstrap => self.bootstrap = Some(item), + ValidatorType::Standard => self.validator = Some(item), _ => panic!("{validator_type} not implemented yet!"), } } @@ -31,10 +32,16 @@ impl ClusterImages { .ok_or_else(|| "Bootstrap validator is not available".into()) } + pub fn validator(&mut self) -> Result<&mut Validator, Box> { + self.validator + .as_mut() + .ok_or_else(|| "Validator is not available".into()) + } + pub fn get_validators(&self) -> impl Iterator { self.bootstrap .iter() - .chain(self._validator.iter()) + .chain(self.validator.iter()) .chain(self._rpc.iter()) .filter_map(Some) } diff --git a/src/docker.rs b/src/docker.rs index 1bbc502..b912f87 100644 --- a/src/docker.rs +++ b/src/docker.rs @@ -73,8 +73,8 @@ impl DockerConfig { ) -> Result<(), Box> { let validator_type = docker_image.validator_type(); match validator_type { - ValidatorType::Bootstrap => (), - ValidatorType::Standard | ValidatorType::RPC | ValidatorType::Client => { + ValidatorType::Bootstrap | ValidatorType::Standard => (), + ValidatorType::RPC | ValidatorType::Client => { return Err(format!( "Build docker image for validator type: {validator_type} not supported yet" ) @@ -138,9 +138,11 @@ impl DockerConfig { fn write_startup_script_to_docker_directory( file_name: &str, docker_dir: &Path, - ) -> std::io::Result<()> { + validator_type: &ValidatorType, + ) -> Result<(), Box> { let script_path = docker_dir.join(file_name); - StartupScripts::write_script_to_file(StartupScripts::bootstrap(), &script_path) + let script_content = validator_type.script()?; + StartupScripts::write_script_to_file(script_content, &script_path).map_err(|e| e.into()) } fn create_dockerfile( @@ -154,11 +156,21 @@ impl DockerConfig { } fs::create_dir_all(docker_path)?; - if validator_type == &ValidatorType::Bootstrap { - let files_to_copy = ["bootstrap-startup-script.sh", "common.sh"]; - for file_name in files_to_copy.iter() { - Self::write_startup_script_to_docker_directory(file_name, docker_path)?; + match validator_type { + ValidatorType::Bootstrap | ValidatorType::Standard => { + let files_to_copy = [ + format!("{validator_type}-startup-script.sh"), + "common.sh".to_string(), + ]; + for file_name in files_to_copy.iter() { + Self::write_startup_script_to_docker_directory( + file_name, + docker_path, + validator_type, + )?; + } } + ValidatorType::RPC | ValidatorType::Client => todo!(), } let startup_script_directory = format!("./docker-build/{validator_type}"); diff --git a/src/k8s_helpers.rs b/src/k8s_helpers.rs index e4cd00a..d2f5ec3 100644 --- a/src/k8s_helpers.rs +++ b/src/k8s_helpers.rs @@ -1,12 +1,12 @@ use { - crate::{docker::DockerImage, ValidatorType}, + crate::docker::DockerImage, k8s_openapi::{ api::{ apps::v1::{ReplicaSet, ReplicaSetSpec}, core::v1::{ - Container, EnvVar, PodSecurityContext, PodSpec, PodTemplateSpec, Probe, - ResourceRequirements, Secret, Service, ServicePort, ServiceSpec, Volume, - VolumeMount, + Container, EnvVar, EnvVarSource, ObjectFieldSelector, PodSecurityContext, PodSpec, + PodTemplateSpec, Probe, ResourceRequirements, Secret, Service, ServicePort, + ServiceSpec, Volume, VolumeMount, }, }, apimachinery::pkg::{api::resource::Quantity, apis::meta::v1::LabelSelector}, @@ -59,7 +59,7 @@ pub fn create_selector(key: &str, value: &str) -> BTreeMap { #[allow(clippy::too_many_arguments)] pub fn create_replica_set( - name: ValidatorType, + name: String, namespace: String, label_selector: BTreeMap, image_name: DockerImage, @@ -167,3 +167,28 @@ pub fn create_service( ..Default::default() } } + +pub fn create_environment_variable( + name: String, + value: Option, + field_path: Option, +) -> EnvVar { + match field_path { + Some(path) => EnvVar { + name, + value_from: Some(EnvVarSource { + field_ref: Some(ObjectFieldSelector { + field_path: path, + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }, + None => EnvVar { + name, + value, + ..Default::default() + }, + } +} diff --git a/src/kubernetes.rs b/src/kubernetes.rs index cc0f050..50ee819 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -20,7 +20,7 @@ use { Client, }, log::*, - solana_sdk::{pubkey::Pubkey, signature::keypair::read_keypair_file, signer::Signer}, + solana_sdk::pubkey::Pubkey, std::{collections::BTreeMap, error::Error, path::Path}, }; @@ -64,6 +64,10 @@ impl<'a> Kubernetes<'a> { } } + pub fn set_shred_version(&mut self, shred_version: u16) { + self.validator_config.shred_version = Some(shred_version); + } + pub async fn namespace_exists(&self) -> Result { let namespaces: Api = Api::all(self.k8s_client.clone()); let namespace_list = namespaces.list(&ListParams::default()).await?; @@ -86,10 +90,6 @@ impl<'a> Kubernetes<'a> { let vote_key_path = config_dir.join("bootstrap-validator/vote-account.json"); let stake_key_path = config_dir.join("bootstrap-validator/stake-account.json"); - let bootstrap_keypair = read_keypair_file(&identity_key_path) - .expect("Failed to read bootstrap validator keypair file"); - self.add_known_validator(bootstrap_keypair.pubkey()); - let mut secrets = BTreeMap::new(); secrets.insert( "faucet".to_string(), @@ -119,7 +119,37 @@ impl<'a> Kubernetes<'a> { k8s_helpers::create_secret(secret_name.to_string(), secrets) } - fn add_known_validator(&mut self, pubkey: Pubkey) { + pub fn create_validator_secret( + &self, + validator_index: usize, + config_dir: &Path, + ) -> Result> { + let secret_name = format!("validator-accounts-secret-{validator_index}"); + + let mut secrets = BTreeMap::new(); + secrets.insert( + "identity".to_string(), + SecretType::File { + path: config_dir.join(format!("validator-identity-{validator_index}.json")), + }, + ); + + let secret_types = ["vote", "stake"]; + for &type_name in secret_types.iter() { + secrets.insert( + type_name.to_string(), + SecretType::File { + path: config_dir.join(format!( + "validator-{type_name}-account-{validator_index}.json" + )), + }, + ); + } + + k8s_helpers::create_secret(secret_name.to_string(), secrets) + } + + pub fn add_known_validator(&mut self, pubkey: Pubkey) { self.validator_config.known_validators.push(pubkey); info!("pubkey added to known validators: {:?}", pubkey); } @@ -172,7 +202,7 @@ impl<'a> Kubernetes<'a> { command.extend(self.generate_bootstrap_command_flags()); k8s_helpers::create_replica_set( - ValidatorType::Bootstrap, + ValidatorType::Bootstrap.to_string(), self.namespace.clone(), label_selector.clone(), image_name.clone(), @@ -228,7 +258,7 @@ impl<'a> Kubernetes<'a> { api.create(&post_params, replica_set).await } - pub fn create_bootstrap_service( + pub fn create_service( &self, service_name: &str, label_selector: &BTreeMap, @@ -264,10 +294,7 @@ impl<'a> Kubernetes<'a> { ) } - pub async fn check_replica_set_ready( - &self, - replica_set_name: &str, - ) -> Result { + pub async fn is_replica_set_ready(&self, replica_set_name: &str) -> Result { let replica_sets: Api = Api::namespaced(self.k8s_client.clone(), self.namespace.as_str()); let replica_set = replica_sets.get(replica_set_name).await?; @@ -283,7 +310,7 @@ impl<'a> Kubernetes<'a> { Ok(available_validators >= desired_validators) } - pub fn create_metrics_secret(&self) -> Result> { + pub fn create_metrics_secret(&self) -> Result> { let mut data = BTreeMap::new(); if let Some(metrics) = &self.metrics { data.insert( @@ -317,4 +344,131 @@ impl<'a> Kubernetes<'a> { ..Default::default() } } + + fn set_non_bootstrap_environment_variables(&self) -> Vec { + vec![ + k8s_helpers::create_environment_variable( + "NAMESPACE".to_string(), + None, + Some("metadata.namespace".to_string()), + ), + k8s_helpers::create_environment_variable( + "BOOTSTRAP_RPC_ADDRESS".to_string(), + Some("bootstrap-validator-service.$(NAMESPACE).svc.cluster.local:8899".to_string()), + None, + ), + k8s_helpers::create_environment_variable( + "BOOTSTRAP_GOSSIP_ADDRESS".to_string(), + Some("bootstrap-validator-service.$(NAMESPACE).svc.cluster.local:8001".to_string()), + None, + ), + k8s_helpers::create_environment_variable( + "BOOTSTRAP_FAUCET_ADDRESS".to_string(), + Some("bootstrap-validator-service.$(NAMESPACE).svc.cluster.local:9900".to_string()), + None, + ), + ] + } + + fn set_load_balancer_environment_variables(&self) -> Vec { + vec![ + k8s_helpers::create_environment_variable( + "LOAD_BALANCER_RPC_ADDRESS".to_string(), + Some( + "bootstrap-and-non-voting-lb-service.$(NAMESPACE).svc.cluster.local:8899" + .to_string(), + ), + None, + ), + k8s_helpers::create_environment_variable( + "LOAD_BALANCER_GOSSIP_ADDRESS".to_string(), + Some( + "bootstrap-and-non-voting-lb-service.$(NAMESPACE).svc.cluster.local:8001" + .to_string(), + ), + None, + ), + k8s_helpers::create_environment_variable( + "LOAD_BALANCER_FAUCET_ADDRESS".to_string(), + Some( + "bootstrap-and-non-voting-lb-service.$(NAMESPACE).svc.cluster.local:9900" + .to_string(), + ), + None, + ), + ] + } + + fn add_known_validators_if_exists(&self, flags: &mut Vec) { + for key in self.validator_config.known_validators.iter() { + flags.push("--known-validator".to_string()); + flags.push(key.to_string()); + } + } + + fn generate_validator_command_flags(&self) -> Vec { + let mut flags: Vec = Vec::new(); + self.generate_command_flags(&mut flags); + + flags.push("--internal-node-stake-sol".to_string()); + flags.push(self.validator_config.internal_node_stake_sol.to_string()); + + flags.push("--internal-node-sol".to_string()); + flags.push(self.validator_config.internal_node_sol.to_string()); + + if let Some(shred_version) = self.validator_config.shred_version { + flags.push("--expected-shred-version".to_string()); + flags.push(shred_version.to_string()); + } + + self.add_known_validators_if_exists(&mut flags); + + flags + } + + pub fn create_validator_replica_set( + &mut self, + image: &DockerImage, + secret_name: Option, + label_selector: &BTreeMap, + validator_index: usize, + ) -> Result> { + let mut env_vars = self.set_non_bootstrap_environment_variables(); + if self.metrics.is_some() { + env_vars.push(self.get_metrics_env_var_secret()) + } + env_vars.append(&mut self.set_load_balancer_environment_variables()); + + let accounts_volume = Some(vec![Volume { + name: format!("validator-accounts-volume-{validator_index}"), + secret: Some(SecretVolumeSource { + secret_name, + ..Default::default() + }), + ..Default::default() + }]); + + let accounts_volume_mount = Some(vec![VolumeMount { + name: format!("validator-accounts-volume-{validator_index}"), + mount_path: "/home/solana/validator-accounts".to_string(), + ..Default::default() + }]); + + let mut command = + vec!["/home/solana/k8s-cluster-scripts/validator-startup-script.sh".to_string()]; + command.extend(self.generate_validator_command_flags()); + + k8s_helpers::create_replica_set( + format!("{}-{validator_index}", ValidatorType::Standard), + self.namespace.clone(), + label_selector.clone(), + image.clone(), + env_vars, + command.clone(), + accounts_volume, + accounts_volume_mount, + self.pod_requests.requests.clone(), + None, + ) + } } diff --git a/src/ledger_helper.rs b/src/ledger_helper.rs new file mode 100644 index 0000000..9f369b3 --- /dev/null +++ b/src/ledger_helper.rs @@ -0,0 +1,31 @@ +use { + crate::genesis::DEFAULT_MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, + log::*, + solana_accounts_db::hardened_unpack::open_genesis_config, + solana_sdk::shred_version::compute_shred_version, + std::{error::Error, path::Path}, +}; + +fn ledger_directory_exists(ledger_dir: &Path) -> Result<(), Box> { + if !ledger_dir.exists() { + return Err( + "Ledger Directory does not exist, have you created genesis yet??" + .to_string() + .into(), + ); + } + Ok(()) +} + +pub struct LedgerHelper {} + +impl LedgerHelper { + pub fn get_shred_version(ledger_dir: &Path) -> Result> { + ledger_directory_exists(ledger_dir)?; + let genesis_config = + open_genesis_config(ledger_dir, DEFAULT_MAX_GENESIS_ARCHIVE_UNPACKED_SIZE); + let shred_version = compute_shred_version(&genesis_config?.hash(), None); + info!("Shred Version: {}", shred_version); + Ok(shred_version) + } +} diff --git a/src/lib.rs b/src/lib.rs index ebe073a..089a81b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -56,6 +56,16 @@ pub enum ValidatorType { Client, } +impl ValidatorType { + fn script(&self) -> Result<&'static str, Box> { + match self { + ValidatorType::Bootstrap => Ok(startup_scripts::StartupScripts::bootstrap()), + ValidatorType::Standard => Ok(startup_scripts::StartupScripts::validator()), + _ => Err(format!("ValidatorType {:?} not supported!", self).into()), + } + } +} + #[derive(Clone, Debug, Default)] pub struct Metrics { pub host: String, @@ -94,6 +104,7 @@ pub mod docker; pub mod genesis; pub mod k8s_helpers; pub mod kubernetes; +pub mod ledger_helper; pub mod release; pub mod startup_scripts; pub mod validator; diff --git a/src/main.rs b/src/main.rs index bccc954..32c5e44 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,16 +5,18 @@ use { DEFAULT_MAX_LEDGER_SHREDS, DEFAULT_MIN_MAX_LEDGER_SHREDS, }, solana_sdk::{signature::keypair::read_keypair_file, signer::Signer}, - std::{fs, path::PathBuf}, + std::{fs, path::PathBuf, result::Result}, strum::VariantNames, validator_lab::{ cluster_images::ClusterImages, docker::{DockerConfig, DockerImage}, genesis::{ Genesis, GenesisFlags, DEFAULT_BOOTSTRAP_NODE_SOL, DEFAULT_BOOTSTRAP_NODE_STAKE_SOL, - DEFAULT_FAUCET_LAMPORTS, DEFAULT_MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, + DEFAULT_FAUCET_LAMPORTS, DEFAULT_INTERNAL_NODE_SOL, DEFAULT_INTERNAL_NODE_STAKE_SOL, + DEFAULT_MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, }, kubernetes::{Kubernetes, PodRequests}, + ledger_helper::LedgerHelper, release::{BuildConfig, BuildType, DeployMethod}, validator::{LabelType, Validator}, validator_config::ValidatorConfig, @@ -66,6 +68,18 @@ fn parse_matches() -> clap::ArgMatches { .help("Absolute path to build directory for release-channel e.g. /home/sol/validator-lab-build"), ) + // non-bootstrap validators + .arg( + Arg::with_name("number_of_validators") + .long("num-validators") + .takes_value(true) + .default_value("1") + .help("Number of validators to deploy") + .validator(|s| match s.parse::() { + Ok(n) if n > 0 => Ok(()), + _ => Err(String::from("number_of_validators should be >= 0")), + }), + ) // Genesis Config .arg( Arg::with_name("hashes_per_tick") @@ -202,6 +216,20 @@ fn parse_matches() -> clap::ArgMatches { .long("full-rpc") .help("Validator config. Support full RPC services on all nodes"), ) + .arg( + Arg::with_name("internal_node_sol") + .long("internal-node-sol") + .takes_value(true) + .default_value(&DEFAULT_INTERNAL_NODE_SOL.to_string()) + .help("Amount to fund internal nodes in genesis config."), + ) + .arg( + Arg::with_name("internal_node_stake_sol") + .long("internal-node-stake-sol") + .takes_value(true) + .default_value(&DEFAULT_INTERNAL_NODE_STAKE_SOL.to_string()) + .help("Amount to stake internal nodes (Sol)."), + ) // kubernetes config .arg( Arg::with_name("cpu_requests") @@ -257,7 +285,7 @@ fn parse_matches() -> clap::ArgMatches { } #[tokio::main] -async fn main() { +async fn main() -> Result<(), Box> { if std::env::var("RUST_LOG").is_err() { std::env::set_var("RUST_LOG", "INFO"); } @@ -268,6 +296,8 @@ async fn main() { build_directory: matches.value_of("build_directory").map(PathBuf::from), }; + let num_validators = value_t_or_exit!(matches, "number_of_validators", usize); + let deploy_method = if let Some(local_path) = matches.value_of("local_path") { DeployMethod::Local(local_path.to_owned()) } else if let Some(release_channel) = matches.value_of("release_channel") { @@ -295,16 +325,18 @@ async fn main() { if let Ok(metadata) = fs::metadata(solana_root.get_root_path()) { if !metadata.is_dir() { - return error!( + return Err(format!( "Build path is not a directory: {}", solana_root.get_root_path().display() - ); + ) + .into()); } } else { - return error!( + return Err(format!( "Build directory not found: {}", solana_root.get_root_path().display() - ); + ) + .into()); } let build_config = BuildConfig::new( @@ -366,6 +398,9 @@ async fn main() { let limit_ledger_size = value_t_or_exit!(matches, "limit_ledger_size", u64); let mut validator_config = ValidatorConfig { + internal_node_sol: value_t_or_exit!(matches, "internal_node_sol", f64), + internal_node_stake_sol: value_t_or_exit!(matches, "internal_node_stake_sol", f64), + shred_version: None, // set after genesis created max_ledger_size: if limit_ledger_size < DEFAULT_MIN_MAX_LEDGER_SHREDS { clap::Error::with_description( format!("The provided --limit-ledger-size value was too small, the minimum value is {DEFAULT_MIN_MAX_LEDGER_SHREDS}"), @@ -405,59 +440,40 @@ async fn main() { ) .await; - match kub_controller.namespace_exists().await { - Ok(true) => (), - Ok(false) => { - error!( - "Namespace: '{}' doesn't exist. Exiting...", - environment_config.namespace - ); - return; - } - Err(err) => { - error!("Error: {err}"); - return; - } + let exists = kub_controller.namespace_exists().await?; + if !exists { + return Err(format!( + "Namespace: '{}' doesn't exist. Exiting...", + environment_config.namespace + ) + .into()); } - match build_config.prepare().await { - Ok(_) => info!("Validator setup prepared successfully"), - Err(err) => { - error!("Error: {err}"); - return; - } - } + build_config.prepare().await?; + info!("Validator setup prepared successfully"); let config_directory = solana_root.get_root_path().join("config-k8s"); let mut genesis = Genesis::new(config_directory.clone(), genesis_flags); - match genesis.generate_faucet() { - Ok(_) => info!("Generated faucet account"), - Err(err) => { - error!("generate faucet error! {err}"); - return; - } - } + genesis.generate_faucet()?; + info!("Generated faucet account"); - match genesis.generate_accounts(ValidatorType::Bootstrap, 1) { - Ok(_) => info!("Generated bootstrap account"), - Err(err) => { - error!("generate accounts error! {err}"); - return; - } - } + genesis.generate_accounts(ValidatorType::Bootstrap, 1)?; + info!("Generated bootstrap account"); // creates genesis and writes to binary file - match genesis + genesis .generate(solana_root.get_root_path(), &build_path) - .await - { - Ok(_) => info!("Created genesis successfully"), - Err(err) => { - error!("generate genesis error! {err}"); - return; - } - } + .await?; + info!("Created genesis successfully"); + + // generate standard validator accounts + genesis.generate_accounts(ValidatorType::Standard, num_validators)?; + info!("Generated {num_validators} validator account(s)"); + + let ledger_dir = config_directory.join("bootstrap-validator"); + let shred_version = LedgerHelper::get_shred_version(&ledger_dir)?; + kub_controller.set_shred_version(shred_version); //unwraps are safe here. since their requirement is enforced by argmatches let docker = DockerConfig::new( @@ -482,63 +498,41 @@ async fn main() { )); cluster_images.set_item(bootstrap_validator, ValidatorType::Bootstrap); + if num_validators > 0 { + let validator = Validator::new(DockerImage::new( + registry_name.clone(), + ValidatorType::Standard, + image_name.clone(), + image_tag.clone(), + )); + cluster_images.set_item(validator, ValidatorType::Standard); + } + if build_config.docker_build() { for v in cluster_images.get_validators() { - match docker.build_image(solana_root.get_root_path(), v.image()) { - Ok(_) => info!("{} image built successfully", v.validator_type()), - Err(err) => { - error!("Failed to build docker image {err}"); - return; - } - } + docker.build_image(solana_root.get_root_path(), v.image())?; + info!("{} image built successfully", v.validator_type()); } - match docker.push_images(cluster_images.get_validators()) { - Ok(_) => info!("Validator images pushed successfully"), - Err(err) => { - error!("Failed to push Validator docker image {err}"); - return; - } - } + docker.push_images(cluster_images.get_validators())?; + info!("Validator images pushed successfully"); } // metrics secret create once and use by all pods if kub_controller.metrics.is_some() { - let metrics_secret = match kub_controller.create_metrics_secret() { - Ok(secret) => secret, - Err(err) => { - error!("Failed to create metrics secret! {err}"); - return; - } - }; - match kub_controller.deploy_secret(&metrics_secret).await { - Ok(_) => (), - Err(err) => { - error!("{err}"); - return; - } - } + let metrics_secret = kub_controller.create_metrics_secret()?; + kub_controller.deploy_secret(&metrics_secret).await?; }; - let bootstrap_validator = cluster_images.bootstrap().expect("should be bootstrap"); - match kub_controller.create_bootstrap_secret("bootstrap-accounts-secret", &config_directory) { - Ok(secret) => bootstrap_validator.set_secret(secret), - Err(err) => { - error!("Failed to create bootstrap secret! {err}"); - return; - } - }; + let bootstrap_validator = cluster_images.bootstrap()?; + let secret = + kub_controller.create_bootstrap_secret("bootstrap-accounts-secret", &config_directory)?; + bootstrap_validator.set_secret(secret); - match kub_controller + kub_controller .deploy_secret(bootstrap_validator.secret()) - .await - { - Ok(_) => info!("Deployed Bootstrap Secret"), - Err(err) => { - error!("{err}"); - return; - } - } + .await?; + info!("Deployed Bootstrap Secret"); // Create Bootstrap labels // Bootstrap needs two labels, one for each service. @@ -546,6 +540,8 @@ async fn main() { let identity_path = config_directory.join("bootstrap-validator/identity.json"); let bootstrap_keypair = read_keypair_file(identity_path).expect("Failed to read bootstrap keypair file"); + kub_controller.add_known_validator(bootstrap_keypair.pubkey()); + bootstrap_validator.add_label( "load-balancer/name", "load-balancer-selector", @@ -556,7 +552,11 @@ async fn main() { "bootstrap-validator-selector", LabelType::Service, ); - bootstrap_validator.add_label("validator/type", "bootstrap", LabelType::Info); + bootstrap_validator.add_label( + "validator/type", + bootstrap_validator.validator_type().to_string(), + LabelType::Info, + ); bootstrap_validator.add_label( "validator/identity", bootstrap_keypair.pubkey().to_string(), @@ -564,46 +564,31 @@ async fn main() { ); // create bootstrap replica set - match kub_controller.create_bootstrap_validator_replica_set( + let replica_set = kub_controller.create_bootstrap_validator_replica_set( bootstrap_validator.image(), bootstrap_validator.secret().metadata.name.clone(), - bootstrap_validator.replica_set_labels(), - ) { - Ok(replica_set) => bootstrap_validator.set_replica_set(replica_set), - Err(err) => { - error!("Error creating bootstrap validator replicas_set: {err}"); - return; - } - }; + bootstrap_validator.info_labels(), + )?; + bootstrap_validator.set_replica_set(replica_set); // deploy bootstrap replica set - match kub_controller + kub_controller .deploy_replicas_set(bootstrap_validator.replica_set()) - .await - { - Ok(_) => { - info!( - "{} deployed successfully", - bootstrap_validator.replica_set_name() - ); - } - Err(err) => { - error!("Error! Failed to deploy bootstrap validator replicas_set. err: {err}"); - return; - } - }; + .await?; + info!( + "{} deployed successfully", + bootstrap_validator.replica_set_name() + ); // create and deploy bootstrap-service - let bootstrap_service = kub_controller.create_bootstrap_service( + let bootstrap_service = kub_controller.create_service( "bootstrap-validator-service", bootstrap_validator.service_labels(), ); - match kub_controller.deploy_service(&bootstrap_service).await { - Ok(_) => info!("bootstrap validator service deployed successfully"), - Err(err) => error!("Error! Failed to deploy bootstrap validator service. err: {err}"), - } + kub_controller.deploy_service(&bootstrap_service).await?; + info!("bootstrap validator service deployed successfully"); - //load balancer service. only create one and use for all bootstrap/rpc nodes + // load balancer service. only create one and use for all bootstrap/rpc nodes // service selector matches bootstrap selector let load_balancer_label = kub_controller.create_selector("load-balancer/name", "load-balancer-selector"); @@ -612,25 +597,76 @@ async fn main() { .create_validator_load_balancer("bootstrap-and-rpc-node-lb-service", &load_balancer_label); //deploy load balancer - match kub_controller.deploy_service(&load_balancer).await { - Ok(_) => info!("load balancer service deployed successfully"), - Err(err) => error!("Error! Failed to deploy load balancer service. err: {err}"), - } + kub_controller.deploy_service(&load_balancer).await?; + info!("load balancer service deployed successfully"); // wait for bootstrap replicaset to deploy - while { - match kub_controller - .check_replica_set_ready(bootstrap_validator.replica_set_name().as_str()) - .await - { - Ok(ok) => !ok, // Continue the loop if replica set is not ready: Ok(false) - Err(_) => panic!("Error occurred while checking replica set readiness"), - } - } { + while !kub_controller + .is_replica_set_ready(bootstrap_validator.replica_set_name().as_str()) + .await? + { info!( "replica set: {} not ready...", bootstrap_validator.replica_set_name() ); - std::thread::sleep(std::time::Duration::from_secs(1)); + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + } + + if num_validators == 0 { + info!("No validators to deploy. Returning"); + return Ok(()); + } + let validator = cluster_images.validator()?; + + for validator_index in 0..num_validators { + // Create and deploy validators secrets + let validator_secret = + kub_controller.create_validator_secret(validator_index, &config_directory)?; + validator.set_secret(validator_secret); + kub_controller.deploy_secret(validator.secret()).await?; + info!("Deployed Validator {validator_index} secret"); + + let identity_path = + config_directory.join(format!("validator-identity-{validator_index}.json")); + let validator_keypair = + read_keypair_file(identity_path).expect("Failed to read validator keypair file"); + + validator.add_label( + "validator/name", + &format!("validator-{validator_index}"), + LabelType::Service, + ); + validator.add_label( + "validator/type", + validator.validator_type().to_string(), + LabelType::Info, + ); + validator.add_label( + "validator/identity", + validator_keypair.pubkey().to_string(), + LabelType::Info, + ); + + let replica_set = kub_controller.create_validator_replica_set( + validator.image(), + validator.secret().metadata.name.clone(), + validator.info_labels(), + validator_index, + )?; + validator.set_replica_set(replica_set); + + kub_controller + .deploy_replicas_set(validator.replica_set()) + .await?; + info!("validator replica set ({validator_index}) deployed successfully"); + + let validator_service = kub_controller.create_service( + &format!("validator-service-{validator_index}"), + validator.service_labels(), + ); + kub_controller.deploy_service(&validator_service).await?; + info!("validator service ({validator_index}) deployed successfully"); } + + Ok(()) } diff --git a/src/release.rs b/src/release.rs index 76ab83a..7e19fe2 100644 --- a/src/release.rs +++ b/src/release.rs @@ -52,16 +52,18 @@ impl BuildConfig { } pub async fn prepare(&self) -> Result<(), Box> { + if self.build_type == BuildType::Skip { + info!("skipping build"); + return Ok(()); + } match &self.deploy_method { - DeployMethod::ReleaseChannel(channel) => match self.setup_tar_deploy(channel).await { - Ok(tar_directory) => { - info!("Successfully setup tar file"); - cat_file(&tar_directory.join("version.yml")).unwrap(); - } - Err(err) => return Err(err), - }, + DeployMethod::ReleaseChannel(channel) => { + let tar_directory = self.setup_tar_deploy(channel).await?; + info!("Successfully setup tar file"); + cat_file(&tar_directory.join("version.yml"))?; + } DeployMethod::Local(_) => { - self.setup_local_deploy()?; + self.build()?; } } info!("Completed Prepare Deploy"); @@ -88,15 +90,6 @@ impl BuildConfig { Ok(release_dir) } - fn setup_local_deploy(&self) -> Result<(), Box> { - if self.build_type != BuildType::Skip { - self.build()?; - } else { - info!("Build skipped due to --build-type skip"); - } - Ok(()) - } - fn build(&self) -> Result<(), Box> { let start_time = Instant::now(); let build_variant = if self.build_type == BuildType::Debug { diff --git a/src/startup_scripts.rs b/src/startup_scripts.rs index b6e599f..de9f62c 100644 --- a/src/startup_scripts.rs +++ b/src/startup_scripts.rs @@ -201,6 +201,422 @@ done "# } + pub fn validator() -> &'static str { + r#" +#!/bin/bash + +# Start Validator +# shellcheck disable=SC1091 +source /home/solana/k8s-cluster-scripts/common.sh + +args=( + --no-poh-speed-test + --no-os-network-limits-test +) +airdrops_enabled=1 +node_sol= +stake_sol= +identity=validator-accounts/identity.json +vote_account=validator-accounts/vote.json +no_restart=0 +gossip_entrypoint=$BOOTSTRAP_GOSSIP_ADDRESS +ledger_dir=/home/solana/ledger +faucet_address=$LOAD_BALANCER_FAUCET_ADDRESS + +# Define the paths to the validator cli. pre 1.18 is `solana-validator`. post 1.18 is `agave-validator` +agave_validator="/home/solana/.cargo/bin/agave-validator" +solana_validator="/home/solana/.cargo/bin/solana-validator" + +# Initialize program variable +program="" + +# Check if agave-validator exists and is executable +if [[ -x "$agave_validator" ]]; then + program="agave-validator" +elif [[ -x "$solana_validator" ]]; then + program="solana-validator" +else + echo "Neither agave-validator nor solana-validator could be found or is not executable." + exit 1 +fi + +echo "PROGRAM: $program" + +usage() { + if [[ -n $1 ]]; then + echo "$*" + echo + fi + cat <&1) + status=$? + + if [ $status -eq 0 ]; then + echo "Command succeeded: $description" + return 0 + else + echo "Command failed for: $description (Exit status $status)" + echo "$output" # Print the output which includes the error + + # Check for specific error message + if [[ "$output" == *"Vote account"*"already exists"* ]]; then + echo "Vote account already exists. Continuing without exiting." + vote_account_already_exists=true + return 0 + fi + if [[ "$output" == *"Stake account"*"already exists"* ]]; then + echo "Stake account already exists. Continuing without exiting." + stake_account_already_exists=true + return 0 + fi + + if [ "$retry_count" -lt $MAX_RETRIES ]; then + echo "Retrying in $RETRY_DELAY seconds..." + sleep $RETRY_DELAY + fi + fi + done + + echo "Max retry limit reached. Command still failed for: $description" + return 1 +} + +setup_validator() { + if ! run_solana_command "solana -u $LOAD_BALANCER_RPC_URL airdrop $node_sol $IDENTITY_FILE" "Airdrop"; then + echo "Aidrop command failed." + exit 1 + fi + + if ! run_solana_command "solana -u $LOAD_BALANCER_RPC_URL create-vote-account --allow-unsafe-authorized-withdrawer validator-accounts/vote.json $IDENTITY_FILE $IDENTITY_FILE -k $IDENTITY_FILE" "Create Vote Account"; then + if $vote_account_already_exists; then + echo "Vote account already exists. Skipping remaining commands." + else + echo "Create vote account failed." + exit 1 + fi + fi + + echo "created vote account" +} + +run_delegate_stake() { + echo "stake sol for account: $stake_sol" + if ! run_solana_command "solana -u $LOAD_BALANCER_RPC_URL create-stake-account validator-accounts/stake.json $stake_sol -k $IDENTITY_FILE" "Create Stake Account"; then + if $stake_account_already_exists; then + echo "Stake account already exists. Skipping remaining commands." + else + echo "Create stake account failed." + exit 1 + fi + fi + echo "created stake account" + + if [ "$stake_account_already_exists" != true ]; then + echo "stake account does not exist. so lets deligate" + if ! run_solana_command "solana -u $LOAD_BALANCER_RPC_URL delegate-stake validator-accounts/stake.json validator-accounts/vote.json --force -k $IDENTITY_FILE" "Delegate Stake"; then + echo "Delegate stake command failed." + exit 1 + fi + echo "delegated stake" + fi + + solana --url $LOAD_BALANCER_RPC_URL --keypair $IDENTITY_FILE stakes validator-accounts/stake.json +} + +echo "get airdrop and create vote account" +setup_validator +echo "create stake account and delegate stake" +run_delegate_stake + +echo running validator: + +echo "Validator Args" +for arg in "${args[@]}"; do + echo "$arg" +done + +while true; do + echo "$PS4$program ${args[*]}" + + $program "${args[@]}" & + pid=$! + echo "pid: $pid" + + if ((no_restart)); then + wait "$pid" + exit $? + fi + + while true; do + if [[ -z $pid ]] || ! kill -0 "$pid"; then + echo "\############## validator exited, restarting ##############" + break + fi + sleep 1 + done + + kill_node +done + "# + } + pub fn common() -> &'static str { r#" # |source| this file diff --git a/src/validator.rs b/src/validator.rs index 82f5640..20e0ac3 100644 --- a/src/validator.rs +++ b/src/validator.rs @@ -58,7 +58,7 @@ impl Validator { } } - pub fn replica_set_labels(&self) -> &BTreeMap { + pub fn info_labels(&self) -> &BTreeMap { &self.info_labels } diff --git a/src/validator_config.rs b/src/validator_config.rs index 900a32b..4a92178 100644 --- a/src/validator_config.rs +++ b/src/validator_config.rs @@ -2,6 +2,9 @@ use solana_sdk::pubkey::Pubkey; #[derive(Debug)] pub struct ValidatorConfig { + pub internal_node_sol: f64, + pub internal_node_stake_sol: f64, + pub shred_version: Option, pub max_ledger_size: Option, pub skip_poh_verify: bool, pub no_snapshot_fetch: bool,