diff --git a/PROGRESS.md b/PROGRESS.md index e97b4ff..45820d4 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -12,9 +12,9 @@ - [x] Setup build config Local - [x] Build from local commit - [x] Build from tar (release version) -- [ ] Create Genesis +- [x] Create Genesis - [x] Generate faucet and bootstrap accounts - - [ ] Build genesis + - [x] Build genesis - [ ] Docker Build - [ ] Build Bootstrap Image - [ ] Push Image to registry diff --git a/README.md b/README.md index 0580e91..2c0d47d 100644 --- a/README.md +++ b/README.md @@ -35,3 +35,20 @@ cargo run --bin cluster -- -n --release-channel # note: MUST include the "v" ``` + +#### Build from Local Repo and Configure Genesis +Example: +``` +cargo run --bin cluster -- + -n + --local-path /home/sol/solana + # genesis config. Optional: Many of these have defaults + --hashes-per-tick + --enable-warmup-epochs + --faucet-lamports + --bootstrap-validator-sol + --bootstrap-validator-stake-sol + --max-genesis-archive-unpacked-size + --target-lamports-per-signature + --slots-per-epoch +``` \ No newline at end of file diff --git a/fetch-spl.sh b/fetch-spl.sh new file mode 100755 index 0000000..bb8e84e --- /dev/null +++ b/fetch-spl.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# +# Fetches the latest SPL programs and produces the solana-genesis command-line +# arguments needed to install them +# + +set -e + +upgradeableLoader=BPFLoaderUpgradeab1e11111111111111111111111 + +fetch_program() { + declare name=$1 + declare version=$2 + declare address=$3 + declare loader=$4 + + declare so=spl_$name-$version.so + + if [[ $loader == "$upgradeableLoader" ]]; then + genesis_args+=(--upgradeable-program "$address" "$loader" "$so" none) + else + genesis_args+=(--bpf-program "$address" "$loader" "$so") + fi + + if [[ -r $so ]]; then + return + fi + + if [[ -r ~/.cache/solana-spl/$so ]]; then + cp ~/.cache/solana-spl/"$so" "$so" + else + echo "Downloading $name $version" + so_name="spl_${name//-/_}.so" + ( + set -x + curl -L --retry 5 --retry-delay 2 --retry-connrefused \ + -o "$so" \ + "https://github.com/solana-labs/solana-program-library/releases/download/$name-v$version/$so_name" + ) + + mkdir -p ~/.cache/solana-spl + cp "$so" ~/.cache/solana-spl/"$so" + fi + +} + +fetch_program token 3.5.0 TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA BPFLoader2111111111111111111111111111111111 +fetch_program token-2022 0.9.0 TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb BPFLoaderUpgradeab1e11111111111111111111111 +fetch_program memo 1.0.0 Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo BPFLoader1111111111111111111111111111111111 +fetch_program memo 3.0.0 MemoSq4gqABAXKb96qnH8TysNcWxMyWCqXgDLGmfcHr BPFLoader2111111111111111111111111111111111 +fetch_program associated-token-account 1.1.2 ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL BPFLoader2111111111111111111111111111111111 +fetch_program feature-proposal 1.0.0 Feat1YXHhH6t1juaWF74WLcfv4XoNocjXA6sPWHNgAse BPFLoader2111111111111111111111111111111111 + +echo "${genesis_args[@]}" > spl-genesis-args.sh + +echo +echo "Available SPL programs:" +ls -l spl_*.so + +echo +echo "solana-genesis command-line arguments (spl-genesis-args.sh):" +cat spl-genesis-args.sh diff --git a/src/genesis.rs b/src/genesis.rs index e2d7b6f..129db00 100644 --- a/src/genesis.rs +++ b/src/genesis.rs @@ -1,23 +1,123 @@ use { - crate::ValidatorType, + crate::{new_spinner_progress_bar, ValidatorType, SUN}, log::*, rand::Rng, solana_core::gen_keys::GenKeys, - solana_sdk::signature::{write_keypair_file, Keypair}, + solana_sdk::{ + native_token::sol_to_lamports, + signature::{write_keypair_file, Keypair}, + }, std::{ error::Error, + fs::File, + io::Read, path::{Path, PathBuf}, + process::Command, result::Result, }, }; +pub const DEFAULT_FAUCET_LAMPORTS: u64 = 500000000000000000; // from agave/ +pub const DEFAULT_MAX_GENESIS_ARCHIVE_UNPACKED_SIZE: u64 = 1073741824; // from agave/ +pub const DEFAULT_INTERNAL_NODE_STAKE_SOL: f64 = 10.0; +pub const DEFAULT_INTERNAL_NODE_SOL: f64 = 100.0; +pub const DEFAULT_BOOTSTRAP_NODE_STAKE_SOL: f64 = 10.0; +pub const DEFAULT_BOOTSTRAP_NODE_SOL: f64 = 100.0; + +fn fetch_spl(fetch_spl_file: &PathBuf) -> Result<(), Box> { + let output = Command::new("bash") + .arg(fetch_spl_file) + .output() // Capture the output of the script + .expect("Failed to run fetch-spl.sh script"); + + // Check if the script execution was successful + if output.status.success() { + Ok(()) + } else { + Err(format!( + "Failed to fun fetch-spl.sh script: {}", + String::from_utf8_lossy(&output.stderr) + ) + .into()) + } +} + +fn parse_spl_genesis_file(spl_file: &PathBuf) -> Result, Box> { + // Read entire file into a String + let mut file = File::open(spl_file)?; + let mut content = String::new(); + file.read_to_string(&mut content)?; + + // Split by whitespace + let mut args = Vec::new(); + let mut tokens_iter = content.split_whitespace(); + + while let Some(token) = tokens_iter.next() { + args.push(token.to_string()); + // Find flag delimiters + if token.starts_with("--") { + for next_token in tokens_iter.by_ref() { + if next_token.starts_with("--") { + args.push(next_token.to_string()); + } else { + args.push(next_token.to_string()); + break; + } + } + } + } + + Ok(args) +} + +pub struct GenesisFlags { + pub hashes_per_tick: String, + pub slots_per_epoch: Option, + pub target_lamports_per_signature: Option, + pub faucet_lamports: Option, + pub enable_warmup_epochs: bool, + pub max_genesis_archive_unpacked_size: Option, + pub cluster_type: String, + pub bootstrap_validator_sol: Option, + pub bootstrap_validator_stake_sol: Option, +} + +impl std::fmt::Display for GenesisFlags { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "GenesisFlags {{\n\ + hashes_per_tick: {:?},\n\ + slots_per_epoch: {:?},\n\ + target_lamports_per_signature: {:?},\n\ + faucet_lamports: {:?},\n\ + enable_warmup_epochs: {},\n\ + max_genesis_archive_unpacked_size: {:?},\n\ + cluster_type: {}\n\ + bootstrap_validator_sol: {:?},\n\ + bootstrap_validator_stake_sol: {:?},\n\ + }}", + self.hashes_per_tick, + self.slots_per_epoch, + self.target_lamports_per_signature, + self.faucet_lamports, + self.enable_warmup_epochs, + self.max_genesis_archive_unpacked_size, + self.cluster_type, + self.bootstrap_validator_sol, + self.bootstrap_validator_stake_sol, + ) + } +} + pub struct Genesis { config_dir: PathBuf, key_generator: GenKeys, + pub flags: GenesisFlags, } impl Genesis { - pub fn new(solana_root: &Path) -> Self { + pub fn new(solana_root: &Path, flags: GenesisFlags) -> Self { let config_dir = solana_root.join("config-k8s"); if config_dir.exists() { std::fs::remove_dir_all(&config_dir).unwrap(); @@ -29,6 +129,7 @@ impl Genesis { Self { config_dir, key_generator: GenKeys::new(seed), + flags, } } @@ -96,4 +197,119 @@ impl Genesis { } Ok(()) } + + fn setup_genesis_flags(&self) -> Vec { + let mut args = vec![ + "--bootstrap-validator-lamports".to_string(), + sol_to_lamports( + self.flags + .bootstrap_validator_sol + .unwrap_or(DEFAULT_BOOTSTRAP_NODE_SOL), + ) + .to_string(), + "--bootstrap-validator-stake-lamports".to_string(), + sol_to_lamports( + self.flags + .bootstrap_validator_stake_sol + .unwrap_or(DEFAULT_BOOTSTRAP_NODE_STAKE_SOL), + ) + .to_string(), + "--hashes-per-tick".to_string(), + self.flags.hashes_per_tick.clone(), + "--max-genesis-archive-unpacked-size".to_string(), + self.flags + .max_genesis_archive_unpacked_size + .unwrap_or(DEFAULT_MAX_GENESIS_ARCHIVE_UNPACKED_SIZE) + .to_string(), + "--faucet-lamports".to_string(), + self.flags + .faucet_lamports + .unwrap_or(DEFAULT_FAUCET_LAMPORTS) + .to_string(), + "--faucet-pubkey".to_string(), + self.config_dir + .join("faucet.json") + .to_string_lossy() + .to_string(), + "--cluster-type".to_string(), + self.flags.cluster_type.to_string(), + "--ledger".to_string(), + self.config_dir + .join("bootstrap-validator") + .to_string_lossy() + .to_string(), + ]; + + if self.flags.enable_warmup_epochs { + args.push("--enable-warmup-epochs".to_string()); + } + + args.push("--bootstrap-validator".to_string()); + ["identity", "vote-account", "stake-account"] + .iter() + .for_each(|account_type| { + args.push( + self.config_dir + .join(format!("bootstrap-validator/{}.json", account_type)) + .to_string_lossy() + .to_string(), + ); + }); + + if let Some(slots_per_epoch) = self.flags.slots_per_epoch { + args.push("--slots-per-epoch".to_string()); + args.push(slots_per_epoch.to_string()); + } + + if let Some(lamports_per_signature) = self.flags.target_lamports_per_signature { + args.push("--target-lamports-per-signature".to_string()); + args.push(lamports_per_signature.to_string()); + } + + args + } + + pub fn setup_spl_args( + &self, + solana_root_path: &PathBuf, + ) -> Result, Box> { + let fetch_spl_file = solana_root_path.join("fetch-spl.sh"); + fetch_spl(&fetch_spl_file)?; + + // add in spl stuff + let spl_file = solana_root_path.join("spl-genesis-args.sh"); + parse_spl_genesis_file(&spl_file) + } + + pub fn generate( + &mut self, + solana_root_path: &PathBuf, + build_path: &PathBuf, + ) -> Result<(), Box> { + let mut args = self.setup_genesis_flags(); + let mut spl_args = self.setup_spl_args(solana_root_path)?; + args.append(&mut spl_args); + + let progress_bar = new_spinner_progress_bar(); + progress_bar.set_message(format!("{SUN}Building Genesis...")); + + let executable_path = build_path.join("solana-genesis"); + let output = Command::new(executable_path) + .args(&args) + .output() + .expect("Failed to execute solana-genesis"); + + progress_bar.finish_and_clear(); + + if !output.status.success() { + return Err(format!( + "Failed to create genesis. err: {}", + String::from_utf8_lossy(&output.stderr) + ) + .into()); + } + info!("Genesis build complete"); + + Ok(()) + } } diff --git a/src/lib.rs b/src/lib.rs index 014a8ef..67cfe23 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -58,6 +58,7 @@ pub mod genesis; pub mod kubernetes; pub mod release; +static SUN: Emoji = Emoji("🌞 ", ""); static PACKAGE: Emoji = Emoji("📦 ", ""); static TRUCK: Emoji = Emoji("🚚 ", ""); diff --git a/src/main.rs b/src/main.rs index b85340e..3319913 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,7 +4,7 @@ use { std::fs, strum::VariantNames, validator_lab::{ - genesis::Genesis, + genesis::{Genesis, GenesisFlags}, kubernetes::Kubernetes, release::{BuildConfig, BuildType, DeployMethod}, SolanaRoot, ValidatorType, @@ -46,6 +46,68 @@ fn parse_matches() -> clap::ArgMatches { .args(&["local_path", "release_channel"]) .required(true), ) + // Genesis Config + .arg( + Arg::with_name("hashes_per_tick") + .long("hashes-per-tick") + .takes_value(true) + .default_value("auto") + .help("NUM_HASHES|sleep|auto - Override the default --hashes-per-tick for the cluster"), + ) + .arg( + Arg::with_name("slots_per_epoch") + .long("slots-per-epoch") + .takes_value(true) + .help("override the number of slots in an epoch"), + ) + .arg( + Arg::with_name("target_lamports_per_signature") + .long("target-lamports-per-signature") + .takes_value(true) + .help("Genesis config. target lamports per signature"), + ) + .arg( + Arg::with_name("faucet_lamports") + .long("faucet-lamports") + .takes_value(true) + .help("Override the default 500000000000000000 lamports minted in genesis"), + ) + .arg( + Arg::with_name("enable_warmup_epochs") + .long("enable-warmup-epochs") + .takes_value(true) + .possible_values(&["true", "false"]) + .default_value("true") + .help("Genesis config. enable warmup epoch. defaults to true"), + ) + .arg( + Arg::with_name("max_genesis_archive_unpacked_size") + .long("max-genesis-archive-unpacked-size") + .takes_value(true) + .help("Genesis config. max_genesis_archive_unpacked_size"), + ) + .arg( + Arg::with_name("cluster_type") + .long("cluster-type") + .possible_values(&["development", "devnet", "testnet", "mainnet-beta"]) + .takes_value(true) + .default_value("development") + .help( + "Selects the features that will be enabled for the cluster" + ), + ) + .arg( + Arg::with_name("bootstrap_validator_sol") + .long("bootstrap-validator-sol") + .takes_value(true) + .help("Genesis config. bootstrap validator sol"), + ) + .arg( + Arg::with_name("bootstrap_validator_stake_sol") + .long("bootstrap-validator-stake-sol") + .takes_value(true) + .help("Genesis config. bootstrap validator stake sol"), + ) .get_matches() } @@ -73,9 +135,17 @@ async fn main() { unreachable!("One of --local-path or --release-channel must be provided."); }; - let solana_root = match &deploy_method { - DeployMethod::Local(path) => SolanaRoot::new_from_path(path.into()), - DeployMethod::ReleaseChannel(_) => SolanaRoot::default(), + let (solana_root, build_path) = match &deploy_method { + DeployMethod::Local(path) => { + let root = SolanaRoot::new_from_path(path.into()); + let path = root.get_root_path().join("farf/bin"); + (root, path) + } + DeployMethod::ReleaseChannel(_) => { + let root = SolanaRoot::default(); + let path = root.get_root_path().join("solana-release/bin"); + (root, path) + } }; let build_type: BuildType = matches.value_of_t("build_type").unwrap(); @@ -110,10 +180,57 @@ async fn main() { } } - let build_config = BuildConfig::new(deploy_method, build_type, solana_root.get_root_path()) - .unwrap_or_else(|err| { - panic!("Error creating BuildConfig: {err}"); - }); + let build_config = BuildConfig::new(deploy_method, build_type, solana_root.get_root_path()); + + let genesis_flags = GenesisFlags { + hashes_per_tick: matches + .value_of("hashes_per_tick") + .unwrap_or_default() + .to_string(), + slots_per_epoch: matches.value_of("slots_per_epoch").map(|value_str| { + value_str + .parse() + .expect("Invalid value for slots_per_epoch") + }), + target_lamports_per_signature: matches.value_of("target_lamports_per_signature").map( + |value_str| { + value_str + .parse() + .expect("Invalid value for target_lamports_per_signature") + }, + ), + faucet_lamports: matches.value_of("faucet_lamports").map(|value_str| { + value_str + .parse() + .expect("Invalid value for faucet_lamports") + }), + enable_warmup_epochs: matches.value_of("enable_warmup_epochs").unwrap() == "true", + max_genesis_archive_unpacked_size: matches + .value_of("max_genesis_archive_unpacked_size") + .map(|value_str| { + value_str + .parse() + .expect("Invalid value for max_genesis_archive_unpacked_size") + }), + cluster_type: matches + .value_of("cluster_type") + .unwrap_or_default() + .to_string(), + bootstrap_validator_sol: matches + .value_of("bootstrap_validator_sol") + .map(|value_str| { + value_str + .parse() + .expect("Invalid value for bootstrap_validator_sol") + }), + bootstrap_validator_stake_sol: matches.value_of("bootstrap_validator_stake_sol").map( + |value_str| { + value_str + .parse() + .expect("Invalid value for bootstrap_validator_stake_sol") + }, + ), + }; match build_config.prepare().await { Ok(_) => info!("Validator setup prepared successfully"), @@ -123,8 +240,7 @@ async fn main() { } } - let mut genesis = Genesis::new(solana_root.get_root_path()); - + let mut genesis = Genesis::new(solana_root.get_root_path(), genesis_flags); match genesis.generate_faucet() { Ok(_) => (), Err(err) => { @@ -140,4 +256,13 @@ async fn main() { return; } } + + // creates genesis and writes to binary file + match genesis.generate(solana_root.get_root_path(), &build_path) { + Ok(_) => (), + Err(err) => { + error!("generate genesis error! {}", err); + return; + } + } } diff --git a/src/release.rs b/src/release.rs index a477f13..d498bff 100644 --- a/src/release.rs +++ b/src/release.rs @@ -28,7 +28,6 @@ pub enum BuildType { pub struct BuildConfig { deploy_method: DeployMethod, build_type: BuildType, - _build_path: PathBuf, solana_root_path: PathBuf, } @@ -37,18 +36,12 @@ impl BuildConfig { deploy_method: DeployMethod, build_type: BuildType, solana_root_path: &Path, - ) -> Result> { - let build_path = match deploy_method { - DeployMethod::Local(_) => solana_root_path.join("farf/bin"), - DeployMethod::ReleaseChannel(_) => solana_root_path.join("solana-release/bin"), - }; - - Ok(BuildConfig { + ) -> Self { + Self { deploy_method, build_type, - _build_path: build_path, solana_root_path: solana_root_path.to_path_buf(), - }) + } } pub async fn prepare(&self) -> Result<(), Box> {