diff --git a/core/src/snapshot_packager_service.rs b/core/src/snapshot_packager_service.rs index ee582e56480417..af522d88dd7f58 100644 --- a/core/src/snapshot_packager_service.rs +++ b/core/src/snapshot_packager_service.rs @@ -156,9 +156,10 @@ mod tests { } // Create a packageable snapshot - let output_tar_path = snapshot_utils::get_snapshot_archive_path( + let output_tar_path = snapshot_utils::build_snapshot_archive_path( snapshot_package_output_path, - &(42, Hash::default()), + 42, + &Hash::default(), ArchiveFormat::TarBzip2, ); let snapshot_package = AccountsPackage::new( diff --git a/core/tests/snapshots.rs b/core/tests/snapshots.rs index 17b4efd67f00c6..28e51d3a0f495e 100644 --- a/core/tests/snapshots.rs +++ b/core/tests/snapshots.rs @@ -148,7 +148,7 @@ mod tests { let old_last_bank = old_bank_forks.get(old_last_slot).unwrap(); let check_hash_calculation = false; - let (deserialized_bank, _timing) = snapshot_utils::bank_from_archive( + let (deserialized_bank, _timing) = snapshot_utils::bank_from_snapshot_archive( account_paths, &[], &old_bank_forks @@ -156,9 +156,10 @@ mod tests { .as_ref() .unwrap() .snapshot_path, - snapshot_utils::get_snapshot_archive_path( + snapshot_utils::build_snapshot_archive_path( snapshot_package_output_path.to_path_buf(), - &(old_last_bank.slot(), old_last_bank.get_accounts_hash()), + old_last_bank.slot(), + &old_last_bank.get_accounts_hash(), ArchiveFormat::TarBzip2, ), ArchiveFormat::TarBzip2, @@ -418,9 +419,10 @@ mod tests { let options = CopyOptions::new(); fs_extra::dir::copy(&last_snapshot_path, &saved_snapshots_dir, &options).unwrap(); - saved_archive_path = Some(snapshot_utils::get_snapshot_archive_path( + saved_archive_path = Some(snapshot_utils::build_snapshot_archive_path( snapshot_package_output_path.to_path_buf(), - &(slot, accounts_hash), + slot, + &accounts_hash, ArchiveFormat::TarBzip2, )); } diff --git a/download-utils/src/lib.rs b/download-utils/src/lib.rs index b911822f0245e6..b92cb9c2ca4614 100644 --- a/download-utils/src/lib.rs +++ b/download-utils/src/lib.rs @@ -259,9 +259,10 @@ pub fn download_snapshot<'a, 'b>( ArchiveFormat::TarGzip, ArchiveFormat::TarBzip2, ] { - let desired_snapshot_package = snapshot_utils::get_snapshot_archive_path( + let desired_snapshot_package = snapshot_utils::build_snapshot_archive_path( snapshot_output_dir.to_path_buf(), - &desired_snapshot_hash, + desired_snapshot_hash.0, + &desired_snapshot_hash.1, *compression, ); diff --git a/ledger/src/bank_forks_utils.rs b/ledger/src/bank_forks_utils.rs index cbe46b0c96930f..dfa85c3900a642 100644 --- a/ledger/src/bank_forks_utils.rs +++ b/ledger/src/bank_forks_utils.rs @@ -11,7 +11,7 @@ use log::*; use solana_runtime::{ bank_forks::BankForks, snapshot_config::SnapshotConfig, - snapshot_utils::{self, ArchiveFormat}, + snapshot_utils::{self, SnapshotArchiveInfo}, }; use solana_sdk::{clock::Slot, genesis_config::GenesisConfig, hash::Hash}; use std::{fs, path::PathBuf, process, result}; @@ -53,11 +53,9 @@ pub fn load( fs::create_dir_all(&snapshot_config.snapshot_path) .expect("Couldn't create snapshot directory"); - if let Some((archive_filename, (archive_slot, archive_hash, archive_format))) = - snapshot_utils::get_highest_snapshot_archive_path( - &snapshot_config.snapshot_package_output_path, - ) - { + if let Some(snapshot_archive_info) = snapshot_utils::get_highest_snapshot_archive_info( + &snapshot_config.snapshot_package_output_path, + ) { return load_from_snapshot( genesis_config, blockstore, @@ -67,10 +65,7 @@ pub fn load( process_options, transaction_status_sender, cache_block_meta_sender, - archive_filename, - archive_slot, - archive_hash, - archive_format, + &snapshot_archive_info, ); } else { info!("No snapshot package available; will load from genesis"); @@ -118,12 +113,12 @@ fn load_from_snapshot( process_options: ProcessOptions, transaction_status_sender: Option<&TransactionStatusSender>, cache_block_meta_sender: Option<&CacheBlockMetaSender>, - archive_filename: PathBuf, - archive_slot: Slot, - archive_hash: Hash, - archive_format: ArchiveFormat, + snapshot_archive_info: &SnapshotArchiveInfo, ) -> LoadResult { - info!("Loading snapshot package: {:?}", archive_filename); + info!( + "Loading snapshot package: {:?}", + &snapshot_archive_info.path + ); // Fail hard here if snapshot fails to load, don't silently continue if account_paths.is_empty() { @@ -131,12 +126,12 @@ fn load_from_snapshot( process::exit(1); } - let (deserialized_bank, timings) = snapshot_utils::bank_from_archive( + let (deserialized_bank, timings) = snapshot_utils::bank_from_snapshot_archive( &account_paths, &process_options.frozen_accounts, &snapshot_config.snapshot_path, - &archive_filename, - archive_format, + &snapshot_archive_info.path, + snapshot_archive_info.archive_format, genesis_config, process_options.debug_keys.clone(), Some(&crate::builtins::get(process_options.bpf_jit)), @@ -156,10 +151,11 @@ fn load_from_snapshot( deserialized_bank.get_accounts_hash(), ); - if deserialized_bank_slot_and_hash != (archive_slot, archive_hash) { + if deserialized_bank_slot_and_hash != (snapshot_archive_info.slot, snapshot_archive_info.hash) { error!( "Snapshot has mismatch:\narchive: {:?}\ndeserialized: {:?}", - archive_hash, deserialized_bank_slot_and_hash + (snapshot_archive_info.slot, snapshot_archive_info.hash), + deserialized_bank_slot_and_hash ); process::exit(1); } diff --git a/local-cluster/tests/local_cluster.rs b/local-cluster/tests/local_cluster.rs index 2db009940b80e4..bcf17cea41ba4e 100644 --- a/local-cluster/tests/local_cluster.rs +++ b/local-cluster/tests/local_cluster.rs @@ -1671,12 +1671,13 @@ fn test_snapshot_download() { wait_for_next_snapshot(&cluster, snapshot_package_output_path); trace!("found: {:?}", archive_filename); - let validator_archive_path = snapshot_utils::get_snapshot_archive_path( + let validator_archive_path = snapshot_utils::build_snapshot_archive_path( validator_snapshot_test_config .snapshot_output_path .path() .to_path_buf(), - &archive_snapshot_hash, + archive_snapshot_hash.0, + &archive_snapshot_hash.1, ArchiveFormat::TarBzip2, ); @@ -1746,12 +1747,13 @@ fn test_snapshot_restart_tower() { wait_for_next_snapshot(&cluster, snapshot_package_output_path); // Copy archive to validator's snapshot output directory - let validator_archive_path = snapshot_utils::get_snapshot_archive_path( + let validator_archive_path = snapshot_utils::build_snapshot_archive_path( validator_snapshot_test_config .snapshot_output_path .path() .to_path_buf(), - &archive_snapshot_hash, + archive_snapshot_hash.0, + &archive_snapshot_hash.1, ArchiveFormat::TarBzip2, ); fs::hard_link(archive_filename, &validator_archive_path).unwrap(); @@ -1806,9 +1808,9 @@ fn test_snapshots_blockstore_floor() { trace!("Waiting for snapshot tar to be generated with slot",); - let (archive_filename, (archive_slot, archive_hash, _)) = loop { + let archive_info = loop { let archive = - snapshot_utils::get_highest_snapshot_archive_path(&snapshot_package_output_path); + snapshot_utils::get_highest_snapshot_archive_info(&snapshot_package_output_path); if archive.is_some() { trace!("snapshot exists"); break archive.unwrap(); @@ -1817,16 +1819,17 @@ fn test_snapshots_blockstore_floor() { }; // Copy archive to validator's snapshot output directory - let validator_archive_path = snapshot_utils::get_snapshot_archive_path( + let validator_archive_path = snapshot_utils::build_snapshot_archive_path( validator_snapshot_test_config .snapshot_output_path .path() .to_path_buf(), - &(archive_slot, archive_hash), + archive_info.slot, + &archive_info.hash, ArchiveFormat::TarBzip2, ); - fs::hard_link(archive_filename, &validator_archive_path).unwrap(); - let slot_floor = archive_slot; + fs::hard_link(archive_info.path, &validator_archive_path).unwrap(); + let slot_floor = archive_info.slot; // Start up a new node from a snapshot let validator_stake = 5; @@ -3122,14 +3125,21 @@ fn wait_for_next_snapshot( last_slot ); loop { - if let Some((filename, (slot, hash, _))) = - snapshot_utils::get_highest_snapshot_archive_path(snapshot_package_output_path) + if let Some(snapshot_archive_info) = + snapshot_utils::get_highest_snapshot_archive_info(snapshot_package_output_path) { - trace!("snapshot for slot {} exists", slot); - if slot >= last_slot { - return (filename, (slot, hash)); + trace!("snapshot for slot {} exists", snapshot_archive_info.slot); + if snapshot_archive_info.slot >= last_slot { + return ( + snapshot_archive_info.path, + (snapshot_archive_info.slot, snapshot_archive_info.hash), + ); } - trace!("snapshot slot {} < last_slot {}", slot, last_slot); + trace!( + "snapshot slot {} < last_slot {}", + snapshot_archive_info.slot, + last_slot + ); } sleep(Duration::from_millis(5000)); } diff --git a/rpc/src/rpc.rs b/rpc/src/rpc.rs index 126f7aed84a878..131f9b9b675c55 100644 --- a/rpc/src/rpc.rs +++ b/rpc/src/rpc.rs @@ -49,7 +49,7 @@ use { inline_spl_token_v2_0::{SPL_TOKEN_ACCOUNT_MINT_OFFSET, SPL_TOKEN_ACCOUNT_OWNER_OFFSET}, non_circulating_supply::calculate_non_circulating_supply, snapshot_config::SnapshotConfig, - snapshot_utils::get_highest_snapshot_archive_path, + snapshot_utils, }, solana_sdk::{ account::{AccountSharedData, ReadableAccount}, @@ -2255,8 +2255,9 @@ pub mod rpc_minimal { meta.snapshot_config .and_then(|snapshot_config| { - get_highest_snapshot_archive_path(&snapshot_config.snapshot_package_output_path) - .map(|(_, (slot, _, _))| slot) + snapshot_utils::get_highest_snapshot_archive_slot( + &snapshot_config.snapshot_package_output_path, + ) }) .ok_or_else(|| RpcCustomError::NoSnapshot.into()) } diff --git a/rpc/src/rpc_service.rs b/rpc/src/rpc_service.rs index ac2ddb46fa1b11..8cec93a302103c 100644 --- a/rpc/src/rpc_service.rs +++ b/rpc/src/rpc_service.rs @@ -198,13 +198,14 @@ impl RequestMiddleware for RpcRequestMiddleware { if let Some(ref snapshot_config) = self.snapshot_config { if request.uri().path() == "/snapshot.tar.bz2" { // Convenience redirect to the latest snapshot - return if let Some((snapshot_archive, _)) = - snapshot_utils::get_highest_snapshot_archive_path( + return if let Some(snapshot_archive_info) = + snapshot_utils::get_highest_snapshot_archive_info( &snapshot_config.snapshot_package_output_path, ) { RpcRequestMiddleware::redirect(&format!( "/{}", - snapshot_archive + snapshot_archive_info + .path .file_name() .unwrap_or_else(|| std::ffi::OsStr::new("")) .to_str() diff --git a/runtime/src/snapshot_utils.rs b/runtime/src/snapshot_utils.rs index 60f14dcac6b405..3d379d935b50e8 100644 --- a/runtime/src/snapshot_utils.rs +++ b/runtime/src/snapshot_utils.rs @@ -38,6 +38,20 @@ use { thiserror::Error, }; +/// Information about a snapshot archive: its path, slot, hash, and archive format +pub struct SnapshotArchiveInfo { + /// Path to the snapshot archive file + pub path: PathBuf, + + /// Slot that the snapshot was made + pub slot: Slot, + + /// Hash of the accounts at this slot + pub hash: Hash, + + /// Archive format for the snapshot file + pub archive_format: ArchiveFormat, +} pub const SNAPSHOT_STATUS_CACHE_FILE_NAME: &str = "status_cache"; pub const MAX_SNAPSHOTS: usize = 8; // Save some snapshots but not too many @@ -47,6 +61,9 @@ const DEFAULT_SNAPSHOT_VERSION: SnapshotVersion = SnapshotVersion::V1_2_0; const TMP_SNAPSHOT_PREFIX: &str = "tmp-snapshot-"; pub const DEFAULT_MAX_SNAPSHOTS_TO_RETAIN: usize = 2; +pub const SNAPSHOT_ARCHIVE_FILENAME_REGEX: &str = + r"^snapshot-(\d+)-([[:alnum:]]+)\.(tar|tar\.bz2|tar\.zst|tar\.gz)$"; + #[derive(Copy, Clone, Eq, PartialEq, Debug)] pub enum SnapshotVersion { V1_2_0, @@ -237,6 +254,7 @@ pub fn remove_tmp_snapshot_archives(snapshot_path: &Path) { } } +/// Make a snapshot archive out of the AccountsPackage pub fn archive_snapshot_package( snapshot_package: &AccountsPackage, maximum_snapshots_to_retain: usize, @@ -593,9 +611,9 @@ fn serialize_status_cache( Ok(()) } +/// Remove the snapshot directory for this slot pub fn remove_snapshot>(slot: Slot, snapshot_path: P) -> Result<()> { let slot_snapshot_dir = get_bank_snapshot_dir(&snapshot_path, slot); - // Remove the snapshot directory for this slot fs::remove_dir_all(slot_snapshot_dir)?; Ok(()) } @@ -610,8 +628,9 @@ pub struct BankFromArchiveTimings { // From testing, 4 seems to be a sweet spot for ranges of 60M-360M accounts and 16-64 cores. This may need to be tuned later. pub const PARALLEL_UNTAR_READERS_DEFAULT: usize = 4; +/// Rebuild a bank from a snapshot archive #[allow(clippy::too_many_arguments)] -pub fn bank_from_archive + std::marker::Sync>( +pub fn bank_from_snapshot_archive

( account_paths: &[PathBuf], frozen_account_pubkeys: &[Pubkey], snapshot_path: &Path, @@ -625,7 +644,10 @@ pub fn bank_from_archive + std::marker::Sync>( limit_load_slot_count_from_snapshot: Option, shrink_ratio: AccountShrinkThreshold, test_hash_calculation: bool, -) -> Result<(Bank, BankFromArchiveTimings)> { +) -> Result<(Bank, BankFromArchiveTimings)> +where + P: AsRef + std::marker::Sync, +{ let unpack_dir = tempfile::Builder::new() .prefix(TMP_SNAPSHOT_PREFIX) .tempdir_in(snapshot_path)?; @@ -684,15 +706,18 @@ pub fn bank_from_archive + std::marker::Sync>( Ok((bank, timings)) } -pub fn get_snapshot_archive_path( +/// Build the snapshot archive path from its components: the snapshot archive output directory, the +/// snapshot slot, the accounts hash, and the archive format. +pub fn build_snapshot_archive_path( snapshot_output_dir: PathBuf, - snapshot_hash: &(Slot, Hash), + slot: Slot, + hash: &Hash, archive_format: ArchiveFormat, ) -> PathBuf { snapshot_output_dir.join(format!( "snapshot-{}-{}.{}", - snapshot_hash.0, - snapshot_hash.1, + slot, + hash, get_archive_ext(archive_format), )) } @@ -709,61 +734,84 @@ fn archive_format_from_str(archive_format: &str) -> Option { /// Parse a snapshot archive filename into its Slot, Hash, and Archive Format fn parse_snapshot_archive_filename(archive_filename: &str) -> Option<(Slot, Hash, ArchiveFormat)> { - let snapshot_archive_filename_regex = - Regex::new(r"^snapshot-(\d+)-([[:alnum:]]+)\.(tar|tar\.bz2|tar\.zst|tar\.gz)$"); - - snapshot_archive_filename_regex - .ok()? - .captures(archive_filename) - .and_then(|captures| { - let slot = captures.get(1).map(|x| x.as_str().parse::())?.ok()?; - let hash = captures.get(2).map(|x| x.as_str().parse::())?.ok()?; - let archive_format = captures - .get(3) - .map(|x| archive_format_from_str(x.as_str()))??; - - Some((slot, hash, archive_format)) - }) + let regex = Regex::new(SNAPSHOT_ARCHIVE_FILENAME_REGEX); + + regex.ok()?.captures(archive_filename).and_then(|captures| { + let slot = captures.get(1).map(|x| x.as_str().parse::())?.ok()?; + let hash = captures.get(2).map(|x| x.as_str().parse::())?.ok()?; + let archive_format = captures + .get(3) + .map(|x| archive_format_from_str(x.as_str()))??; + + Some((slot, hash, archive_format)) + }) } -/// Get a list of the snapshot archives in a directory, sorted by Slot in descending order -fn get_snapshot_archives>( - snapshot_output_dir: P, -) -> Vec<(PathBuf, (Slot, Hash, ArchiveFormat))> { +/// Get a list of the snapshot archives in a directory +pub fn get_snapshot_archives

(snapshot_output_dir: P) -> Vec +where + P: AsRef, +{ match fs::read_dir(&snapshot_output_dir) { Err(err) => { info!("Unable to read snapshot directory: {}", err); vec![] } - Ok(files) => { - let mut archives: Vec<_> = files - .filter_map(|entry| { - if let Ok(entry) = entry { - let path = entry.path(); - if path.is_file() { - if let Some(snapshot_hash) = parse_snapshot_archive_filename( - path.file_name().unwrap().to_str().unwrap(), - ) { - return Some((path, snapshot_hash)); - } + Ok(files) => files + .filter_map(|entry| { + if let Ok(entry) = entry { + let path = entry.path(); + if path.is_file() { + if let Some((slot, hash, archive_format)) = parse_snapshot_archive_filename( + path.file_name().unwrap().to_str().unwrap(), + ) { + return Some(SnapshotArchiveInfo { + path, + slot, + hash, + archive_format, + }); } } - None - }) - .collect(); - - archives.sort_by(|a, b| (b.1).0.cmp(&(a.1).0)); // reverse sort by slot - archives - } + } + None + }) + .collect(), } } -/// Get the snapshot archive with the highest Slot in a directory -pub fn get_highest_snapshot_archive_path>( - snapshot_output_dir: P, -) -> Option<(PathBuf, (Slot, Hash, ArchiveFormat))> { - let archives = get_snapshot_archives(snapshot_output_dir); - archives.into_iter().next() +/// Get a sorted list of the snapshot archives in a directory +fn get_sorted_snapshot_archives

(snapshot_output_dir: P) -> Vec +where + P: AsRef, +{ + let mut snapshot_archives = get_snapshot_archives(snapshot_output_dir); + sort_snapshot_archives(&mut snapshot_archives); + snapshot_archives +} + +/// Sort the list of snapshot archives by slot, in descending order +fn sort_snapshot_archives(snapshot_archives: &mut Vec) { + snapshot_archives.sort_unstable_by(|a, b| b.slot.cmp(&a.slot)); +} + +/// Get the highest slot of the snapshots in a directory +pub fn get_highest_snapshot_archive_slot

(snapshot_output_dir: P) -> Option +where + P: AsRef, +{ + get_highest_snapshot_archive_info(snapshot_output_dir) + .map(|snapshot_archive_info| snapshot_archive_info.slot) +} + +/// Get the path (and metadata) for the snapshot archive with the highest slot in a directory +pub fn get_highest_snapshot_archive_info

(snapshot_output_dir: P) -> Option +where + P: AsRef, +{ + get_sorted_snapshot_archives(snapshot_output_dir) + .into_iter() + .next() } pub fn purge_old_snapshot_archives>( @@ -775,12 +823,12 @@ pub fn purge_old_snapshot_archives>( snapshot_output_dir.as_ref(), maximum_snapshots_to_retain ); - let mut archives = get_snapshot_archives(snapshot_output_dir); + let mut archives = get_sorted_snapshot_archives(snapshot_output_dir); // Keep the oldest snapshot so we can always play the ledger from it. archives.pop(); let max_snaps = max(1, maximum_snapshots_to_retain); for old_archive in archives.into_iter().skip(max_snaps) { - fs::remove_file(old_archive.0) + fs::remove_file(old_archive.path) .unwrap_or_else(|err| info!("Failed to remove old snapshot: {:}", err)); } } @@ -993,7 +1041,7 @@ pub fn purge_old_snapshots(snapshot_path: &Path) { } } -// Gather the necessary elements for a snapshot of the given `root_bank` +/// Gather the necessary elements for a snapshot of the given `root_bank` pub fn snapshot_bank( root_bank: &Bank, status_cache_slot_deltas: Vec, @@ -1035,6 +1083,9 @@ pub fn snapshot_bank( /// Convenience function to create a snapshot archive out of any Bank, regardless of state. The /// Bank will be frozen during the process. +/// +/// Requires: +/// - `bank` is complete pub fn bank_to_snapshot_archive, Q: AsRef>( snapshot_path: P, bank: &Bank, @@ -1055,7 +1106,7 @@ pub fn bank_to_snapshot_archive, Q: AsRef>( let temp_dir = tempfile::tempdir_in(snapshot_path)?; - let storages: Vec<_> = bank.get_snapshot_storages(); + let storages = bank.get_snapshot_storages(); let slot_snapshot_paths = add_snapshot(&temp_dir, bank, &storages, snapshot_version)?; let package = package_snapshot( bank, @@ -1104,9 +1155,10 @@ pub fn process_accounts_package_pre( ("calculate_hash", time.as_us(), i64), ); - let tar_output_file = get_snapshot_archive_path( + let tar_output_file = build_snapshot_archive_path( accounts_package.snapshot_output_dir, - &(accounts_package.slot, hash), + accounts_package.slot, + &hash, accounts_package.archive_format, ); @@ -1128,6 +1180,10 @@ mod tests { use super::*; use assert_matches::assert_matches; use bincode::{deserialize_from, serialize_into}; + use solana_sdk::{ + genesis_config::create_genesis_config, + signature::{Keypair, Signer}, + }; use std::mem::size_of; #[test] @@ -1246,47 +1302,104 @@ mod tests { Some((43, Hash::default(), ArchiveFormat::TarZstd)) ); assert_eq!( - parse_snapshot_archive_filename(&format!("snapshot-42-{}.tar", Hash::default())), - Some((42, Hash::default(), ArchiveFormat::Tar)) + parse_snapshot_archive_filename(&format!("snapshot-44-{}.tar", Hash::default())), + Some((44, Hash::default(), ArchiveFormat::Tar)) ); assert!(parse_snapshot_archive_filename("invalid").is_none()); assert!(parse_snapshot_archive_filename("snapshot-bad!slot-bad!hash.bad!ext").is_none()); assert!(parse_snapshot_archive_filename("snapshot-12345678-bad!hash.bad!ext").is_none()); - assert!(parse_snapshot_archive_filename("snapshot-12345678-HASH1234.bad!ext").is_none()); + assert!(parse_snapshot_archive_filename(&format!( + "snapshot-12345678-{}.bad!ext", + Hash::new_unique() + )) + .is_none()); assert!(parse_snapshot_archive_filename("snapshot-12345678-bad!hash.tar").is_none()); - assert!(parse_snapshot_archive_filename("snapshot-bad!slot-HASH1234.bad!ext").is_none()); - assert!(parse_snapshot_archive_filename("snapshot-12345678-HASH1234.bad!ext").is_none()); - assert!(parse_snapshot_archive_filename("snapshot-bad!slot-HASH1234.tar").is_none()); + assert!(parse_snapshot_archive_filename(&format!( + "snapshot-bad!slot-{}.bad!ext", + Hash::new_unique() + )) + .is_none()); + assert!(parse_snapshot_archive_filename(&format!( + "snapshot-12345678-{}.bad!ext", + Hash::new_unique() + )) + .is_none()); + assert!(parse_snapshot_archive_filename(&format!( + "snapshot-bad!slot-{}.tar", + Hash::new_unique() + )) + .is_none()); assert!(parse_snapshot_archive_filename("snapshot-bad!slot-bad!hash.tar").is_none()); assert!(parse_snapshot_archive_filename("snapshot-12345678-bad!hash.tar").is_none()); - assert!(parse_snapshot_archive_filename("snapshot-bad!slot-HASH1234.tar").is_none()); + assert!(parse_snapshot_archive_filename(&format!( + "snapshot-bad!slot-{}.tar", + Hash::new_unique() + )) + .is_none()); } - #[test] - fn test_get_snapshot_archives() { - let temp_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); - - let min_slot = 123; - let max_slot = 456; - for slot in min_slot..max_slot { - let snapshot_filename = format!("snapshot-{}-{}.tar", slot, Hash::default()); - let snapshot_filepath = temp_snapshot_archives_dir.path().join(snapshot_filename); + /// A test helper function that creates snapshot archive files. Creates snapshot files in the + /// range (`min_snapshot_slot`, `max_snapshot_slot`]. Additionally, "bad" files are created + /// for snapshots to ensure the tests properly filter them out. + fn common_create_snapshot_archive_files( + snapshot_dir: &Path, + min_snapshot_slot: Slot, + max_snapshot_slot: Slot, + ) { + for snapshot_slot in min_snapshot_slot..max_snapshot_slot { + let snapshot_filename = format!("snapshot-{}-{}.tar", snapshot_slot, Hash::default()); + let snapshot_filepath = snapshot_dir.join(snapshot_filename); File::create(snapshot_filepath).unwrap(); } // Add in a snapshot with a bad filename and high slot to ensure filename are filtered and // sorted correctly - let bad_filename = format!("snapshot-{}-{}.bad!ext", max_slot + 1, Hash::default()); - let bad_filepath = temp_snapshot_archives_dir.path().join(bad_filename); + let bad_filename = format!("snapshot-{}-bad!hash.tar", max_snapshot_slot + 1); + let bad_filepath = snapshot_dir.join(bad_filename); File::create(bad_filepath).unwrap(); + } + + #[test] + fn test_get_snapshot_archives() { + solana_logger::setup(); + let temp_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); + let min_slot = 123; + let max_slot = 456; + common_create_snapshot_archive_files(temp_snapshot_archives_dir.path(), min_slot, max_slot); - let results = get_snapshot_archives(temp_snapshot_archives_dir); - assert_eq!(results.len(), max_slot - min_slot); - assert_eq!(results[0].1 .0 as usize, max_slot - 1); + let snapshot_archives = get_snapshot_archives(temp_snapshot_archives_dir); + assert_eq!(snapshot_archives.len() as Slot, max_slot - min_slot); + } + + #[test] + fn test_get_sorted_snapshot_archives() { + solana_logger::setup(); + let temp_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); + let min_slot = 12; + let max_slot = 45; + common_create_snapshot_archive_files(temp_snapshot_archives_dir.path(), min_slot, max_slot); + + let sorted_snapshot_archives = get_sorted_snapshot_archives(temp_snapshot_archives_dir); + assert_eq!(sorted_snapshot_archives.len() as Slot, max_slot - min_slot); + assert_eq!(sorted_snapshot_archives[0].slot, max_slot - 1); + } + + #[test] + fn test_get_highest_snapshot_archive_slot() { + solana_logger::setup(); + let temp_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); + let min_slot = 123; + let max_slot = 456; + common_create_snapshot_archive_files(temp_snapshot_archives_dir.path(), min_slot, max_slot); + + assert_eq!( + get_highest_snapshot_archive_slot(temp_snapshot_archives_dir.path()), + Some(max_slot - 1) + ); } fn common_test_purge_old_snapshot_archives( @@ -1339,4 +1452,140 @@ mod tests { let expected_snapshots = vec![&snap1_name, &snap2_name, &snap3_name]; common_test_purge_old_snapshot_archives(&snapshot_names, 2, &expected_snapshots); } + + /// Test roundtrip of bank to snapshot, then back again. This test creates the simplest bank + /// possible, so the contents of the snapshot archive will be quite minimal. + #[test] + fn test_roundtrip_bank_to_snapshot_to_bank_simple() { + solana_logger::setup(); + let genesis_config = GenesisConfig::default(); + let original_bank = Bank::new(&genesis_config); + + while !original_bank.is_complete() { + original_bank.register_tick(&Hash::new_unique()); + } + + let accounts_dir = tempfile::TempDir::new().unwrap(); + let snapshot_dir = tempfile::TempDir::new().unwrap(); + let snapshot_package_output_dir = tempfile::TempDir::new().unwrap(); + let snapshot_archive_format = ArchiveFormat::Tar; + + let snapshot_archive_path = bank_to_snapshot_archive( + snapshot_dir.path(), + &original_bank, + None, + snapshot_package_output_dir.path(), + snapshot_archive_format, + None, + 1, + ) + .unwrap(); + + let (roundtrip_bank, _) = bank_from_snapshot_archive( + &[PathBuf::from(accounts_dir.path())], + &[], + snapshot_dir.path(), + &snapshot_archive_path, + snapshot_archive_format, + &genesis_config, + None, + None, + AccountSecondaryIndexes::default(), + false, + None, + AccountShrinkThreshold::default(), + false, + ) + .unwrap(); + + assert_eq!(original_bank, roundtrip_bank); + } + + /// Test roundtrip of bank to snapshot, then back again. This test is more involved than the + /// simple version above; creating multiple banks over multiple slots and doing multiple + /// transfers. So this snapshot should contain more data. + #[test] + fn test_roundtrip_bank_to_snapshot_to_bank_complex() { + solana_logger::setup(); + let collector = Pubkey::new_unique(); + let key1 = Keypair::new(); + let key2 = Keypair::new(); + let key3 = Keypair::new(); + let key4 = Keypair::new(); + let key5 = Keypair::new(); + + let (genesis_config, mint_keypair) = create_genesis_config(1_000_000); + let bank0 = Arc::new(Bank::new(&genesis_config)); + bank0.transfer(1, &mint_keypair, &key1.pubkey()).unwrap(); + bank0.transfer(2, &mint_keypair, &key2.pubkey()).unwrap(); + bank0.transfer(3, &mint_keypair, &key3.pubkey()).unwrap(); + while !bank0.is_complete() { + bank0.register_tick(&Hash::new_unique()); + } + + let slot = 1; + let bank1 = Arc::new(Bank::new_from_parent(&bank0, &collector, slot)); + bank1.transfer(3, &mint_keypair, &key3.pubkey()).unwrap(); + bank1.transfer(4, &mint_keypair, &key4.pubkey()).unwrap(); + bank1.transfer(5, &mint_keypair, &key5.pubkey()).unwrap(); + while !bank1.is_complete() { + bank1.register_tick(&Hash::new_unique()); + } + + let slot = slot + 1; + let bank2 = Arc::new(Bank::new_from_parent(&bank1, &collector, slot)); + bank2.transfer(1, &mint_keypair, &key1.pubkey()).unwrap(); + while !bank2.is_complete() { + bank2.register_tick(&Hash::new_unique()); + } + + let slot = slot + 1; + let bank3 = Arc::new(Bank::new_from_parent(&bank2, &collector, slot)); + bank3.transfer(1, &mint_keypair, &key1.pubkey()).unwrap(); + while !bank3.is_complete() { + bank3.register_tick(&Hash::new_unique()); + } + + let slot = slot + 1; + let bank4 = Arc::new(Bank::new_from_parent(&bank3, &collector, slot)); + bank4.transfer(1, &mint_keypair, &key1.pubkey()).unwrap(); + while !bank4.is_complete() { + bank4.register_tick(&Hash::new_unique()); + } + + let accounts_dir = tempfile::TempDir::new().unwrap(); + let snapshot_dir = tempfile::TempDir::new().unwrap(); + let snapshot_package_output_dir = tempfile::TempDir::new().unwrap(); + let snapshot_archive_format = ArchiveFormat::Tar; + + let full_snapshot_archive_path = bank_to_snapshot_archive( + snapshot_dir.path(), + &bank4, + None, + snapshot_package_output_dir.path(), + snapshot_archive_format, + None, + std::usize::MAX, + ) + .unwrap(); + + let (roundtrip_bank, _) = bank_from_snapshot_archive( + &[PathBuf::from(accounts_dir.path())], + &[], + snapshot_dir.path(), + &full_snapshot_archive_path, + snapshot_archive_format, + &genesis_config, + None, + None, + AccountSecondaryIndexes::default(), + false, + None, + AccountShrinkThreshold::default(), + false, + ) + .unwrap(); + + assert_eq!(*bank4, roundtrip_bank); + } } diff --git a/validator/src/main.rs b/validator/src/main.rs index 7948c1fcac2a93..a2bf791cdfbcf2 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -48,10 +48,7 @@ use { }, hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, snapshot_config::SnapshotConfig, - snapshot_utils::{ - get_highest_snapshot_archive_path, ArchiveFormat, SnapshotVersion, - DEFAULT_MAX_SNAPSHOTS_TO_RETAIN, - }, + snapshot_utils::{self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_SNAPSHOTS_TO_RETAIN}, }, solana_sdk::{ clock::{Slot, DEFAULT_S_PER_SLOT}, @@ -471,8 +468,9 @@ fn get_rpc_node( blacklist_timeout = Instant::now(); let mut highest_snapshot_hash: Option<(Slot, Hash)> = - get_highest_snapshot_archive_path(snapshot_output_dir) - .map(|(_path, (slot, hash, _compression))| (slot, hash)); + snapshot_utils::get_highest_snapshot_archive_info(snapshot_output_dir).map( + |snapshot_archive_info| (snapshot_archive_info.slot, snapshot_archive_info.hash), + ); let eligible_rpc_peers = if snapshot_not_required { rpc_peers } else { @@ -853,8 +851,7 @@ fn rpc_bootstrap( let mut use_local_snapshot = false; if let Some(highest_local_snapshot_slot) = - get_highest_snapshot_archive_path(snapshot_output_dir) - .map(|(_path, (slot, _hash, _compression))| slot) + snapshot_utils::get_highest_snapshot_archive_slot(snapshot_output_dir) { if highest_local_snapshot_slot > snapshot_hash.0.saturating_sub(maximum_local_snapshot_age)