diff --git a/runtime/src/serde_snapshot.rs b/runtime/src/serde_snapshot.rs index 126b4c16c5e8c9..01fc59a0869960 100644 --- a/runtime/src/serde_snapshot.rs +++ b/runtime/src/serde_snapshot.rs @@ -471,24 +471,6 @@ where Ok((bank_fields, accounts_db_fields)) } -/// used by tests to compare contents of serialized bank fields -/// serialized format is not deterministic - likely due to randomness in structs like hashmaps -#[cfg(feature = "dev-context-only-utils")] -pub(crate) fn compare_two_serialized_banks( - path1: impl AsRef, - path2: impl AsRef, -) -> std::result::Result { - use std::fs::File; - let file1 = File::open(path1)?; - let mut stream1 = BufReader::new(file1); - let file2 = File::open(path2)?; - let mut stream2 = BufReader::new(file2); - - let fields1 = deserialize_bank_fields(&mut stream1)?; - let fields2 = deserialize_bank_fields(&mut stream2)?; - Ok(fields1 == fields2) -} - /// Get snapshot storage lengths from accounts_db_fields pub(crate) fn snapshot_storage_lengths_from_fields( accounts_db_fields: &AccountsDbFields, diff --git a/runtime/src/snapshot_utils.rs b/runtime/src/snapshot_utils.rs index cd8992cf74c72d..d6c85e8f260e6d 100644 --- a/runtime/src/snapshot_utils.rs +++ b/runtime/src/snapshot_utils.rs @@ -1,3 +1,5 @@ +#[cfg(feature = "dev-context-only-utils")] +use solana_accounts_db::utils::create_accounts_run_and_snapshot_dirs; use { crate::{ bank::{BankFieldsToDeserialize, BankFieldsToSerialize, BankHashStats, BankSlotDelta}, @@ -48,11 +50,6 @@ use { tempfile::TempDir, thiserror::Error, }; -#[cfg(feature = "dev-context-only-utils")] -use { - hardened_unpack::UnpackedAppendVecMap, - solana_accounts_db::utils::create_accounts_run_and_snapshot_dirs, -}; mod archive_format; mod snapshot_interval; @@ -2427,27 +2424,6 @@ pub fn purge_old_snapshot_archives( } } -#[cfg(feature = "dev-context-only-utils")] -fn unpack_snapshot_local( - snapshot_path: impl AsRef, - archive_format: ArchiveFormat, - ledger_dir: &Path, - account_paths: &[PathBuf], - num_threads: usize, -) -> Result { - assert!(num_threads > 0); - let archive_size = fs::metadata(&snapshot_path)?.len(); - let archive = Archive::new(decompressed_tar_reader( - archive_format, - snapshot_path, - archive_size, - )?); - let unpacked_append_vec_map = - hardened_unpack::unpack_snapshot(archive, archive_size, ledger_dir, account_paths)?; - - Ok(unpacked_append_vec_map) -} - pub fn verify_unpacked_snapshots_dir_and_version( unpacked_snapshots_dir_and_version: &UnpackedSnapshotsDirAndVersion, ) -> Result<(SnapshotVersion, BankSnapshotInfo)> { @@ -2499,100 +2475,6 @@ pub enum VerifyBank { NonDeterministic, } -#[cfg(feature = "dev-context-only-utils")] -pub fn verify_snapshot_archive( - snapshot_archive: impl AsRef, - snapshots_to_verify: impl AsRef, - archive_format: ArchiveFormat, - verify_bank: VerifyBank, - slot: Slot, -) { - let temp_dir = tempfile::TempDir::new().unwrap(); - let unpack_dir = temp_dir.path(); - let unpack_account_dir = create_accounts_run_and_snapshot_dirs(unpack_dir).unwrap().0; - unpack_snapshot_local( - snapshot_archive, - archive_format, - unpack_dir, - &[unpack_account_dir.clone()], - 1, - ) - .unwrap(); - - // Check snapshots are the same - let unpacked_snapshots = unpack_dir.join(BANK_SNAPSHOTS_DIR); - - // Since the unpack code collects all the appendvecs into one directory unpack_account_dir, we need to - // collect all the appendvecs in account_paths//snapshot/ into one directory for later comparison. - let storages_to_verify = unpack_dir.join("storages_to_verify"); - // Create the directory if it doesn't exist - fs::create_dir_all(&storages_to_verify).unwrap(); - - let slot = slot.to_string(); - let snapshot_slot_dir = snapshots_to_verify.as_ref().join(&slot); - - if let VerifyBank::NonDeterministic = verify_bank { - // file contents may be different, but deserialized structs should be equal - let p1 = snapshots_to_verify.as_ref().join(&slot).join(&slot); - let p2 = unpacked_snapshots.join(&slot).join(&slot); - assert!(crate::serde_snapshot::compare_two_serialized_banks(&p1, &p2).unwrap()); - fs::remove_file(p1).unwrap(); - fs::remove_file(p2).unwrap(); - } - - // The new the status_cache file is inside the slot directory together with the snapshot file. - // When unpacking an archive, the status_cache file from the archive is one-level up outside of - // the slot directory. - // The unpacked status_cache file need to be put back into the slot directory for the directory - // comparison to pass. - let existing_unpacked_status_cache_file = - unpacked_snapshots.join(SNAPSHOT_STATUS_CACHE_FILENAME); - let new_unpacked_status_cache_file = unpacked_snapshots - .join(&slot) - .join(SNAPSHOT_STATUS_CACHE_FILENAME); - fs::rename( - existing_unpacked_status_cache_file, - new_unpacked_status_cache_file, - ) - .unwrap(); - - let accounts_hardlinks_dir = snapshot_slot_dir.join(SNAPSHOT_ACCOUNTS_HARDLINKS); - if accounts_hardlinks_dir.is_dir() { - // This directory contain symlinks to all /snapshot/ directories. - for entry in fs::read_dir(&accounts_hardlinks_dir).unwrap() { - let link_dst_path = fs::read_link(entry.unwrap().path()).unwrap(); - // Copy all the files in dst_path into the storages_to_verify directory. - for entry in fs::read_dir(&link_dst_path).unwrap() { - let src_path = entry.unwrap().path(); - let dst_path = storages_to_verify.join(src_path.file_name().unwrap()); - fs::copy(src_path, dst_path).unwrap(); - } - } - fs::remove_dir_all(accounts_hardlinks_dir).unwrap(); - } - - let version_path = snapshot_slot_dir.join(SNAPSHOT_VERSION_FILENAME); - if version_path.is_file() { - fs::remove_file(version_path).unwrap(); - } - - let state_complete_path = snapshot_slot_dir.join(SNAPSHOT_STATE_COMPLETE_FILENAME); - if state_complete_path.is_file() { - fs::remove_file(state_complete_path).unwrap(); - } - - assert!(!dir_diff::is_different(&snapshots_to_verify, unpacked_snapshots).unwrap()); - - // In the unarchiving case, there is an extra empty "accounts" directory. The account - // files in the archive accounts/ have been expanded to [account_paths]. - // Remove the empty "accounts" directory for the directory comparison below. - // In some test cases the directory to compare do not come from unarchiving. - // Ignore the error when this directory does not exist. - _ = fs::remove_dir(unpack_account_dir.join("accounts")); - // Check the account entries are the same - assert!(!dir_diff::is_different(&storages_to_verify, unpack_account_dir).unwrap()); -} - /// Purges all bank snapshots pub fn purge_all_bank_snapshots(bank_snapshots_dir: impl AsRef) { let bank_snapshots = get_bank_snapshots(&bank_snapshots_dir);