Correct missing entry handling to avoid bad warns (bp #8339)

core/src/snapshot_packager_service.rs

@@ -156,7 +156,7 @@ mod tests {
             5,
             vec![],
             link_snapshots_dir,
-            storage_entries.clone(),
+            vec![storage_entries],
             output_tar_path.clone(),
             Hash::default(),
         );

core/tests/bank_forks.rs

@@ -131,6 +131,7 @@ mod tests {
         }
         // Generate a snapshot package for last bank
         let last_bank = bank_forks.get(last_slot).unwrap();
+        let storages: Vec<_> = last_bank.get_snapshot_storages();
         let slot_snapshot_paths =
             snapshot_utils::get_snapshot_paths(&snapshot_config.snapshot_path);
         let snapshot_package = snapshot_utils::package_snapshot(
@@ -143,6 +144,7 @@ mod tests {
             ),
             &snapshot_config.snapshot_path,
             &last_bank.src.roots(),
+            storages,
         )
         .unwrap();
 
@@ -202,7 +204,8 @@ mod tests {
 
         // Take snapshot of zeroth bank
         let bank0 = bank_forks.get(0).unwrap();
-        snapshot_utils::add_snapshot(&snapshot_config.snapshot_path, bank0).unwrap();
+        let storages: Vec<_> = bank0.get_snapshot_storages();
+        snapshot_utils::add_snapshot(&snapshot_config.snapshot_path, bank0, &storages).unwrap();
 
         // Set up snapshotting channels
         let (sender, receiver) = channel();

ledger-tool/src/main.rs

@@ -903,14 +903,16 @@ fn main() {
                         exit(1);
                     });
 
-                    snapshot_utils::add_snapshot(&temp_dir, &bank)
+                    let storages: Vec<_> = bank.get_snapshot_storages();
+                    snapshot_utils::add_snapshot(&temp_dir, &bank, &storages)
                         .and_then(|slot_snapshot_paths| {
                             snapshot_utils::package_snapshot(
                                 &bank,
                                 &slot_snapshot_paths,
                                 snapshot_utils::get_snapshot_archive_path(output_directory),
                                 &temp_dir,
                                 &bank.src.roots(),
+                                storages,
                             )
                         })
                         .and_then(|package| {

ledger/src/bank_forks.rs

@@ -251,8 +251,9 @@ impl BankForks {
             .cloned()
             .expect("root must exist in BankForks");
 
+        let storages: Vec<_> = bank.get_snapshot_storages();
         let mut add_snapshot_time = Measure::start("add-snapshot-ms");
-        snapshot_utils::add_snapshot(&config.snapshot_path, &bank)?;
+        snapshot_utils::add_snapshot(&config.snapshot_path, &bank, &storages)?;
         add_snapshot_time.stop();
         inc_new_counter_info!("add-snapshot-ms", add_snapshot_time.as_ms() as usize);
 
@@ -269,6 +270,7 @@ impl BankForks {
             tar_output_file,
             &config.snapshot_path,
             slots_to_snapshot,
+            storages,
         )?;
 
         // Send the package to the packaging thread

ledger/src/snapshot_package.rs

@@ -1,12 +1,9 @@
-use solana_runtime::{accounts_db::AccountStorageEntry, bank::BankSlotDelta};
+use solana_runtime::{accounts_db::SnapshotStorages, bank::BankSlotDelta};
 use solana_sdk::clock::Slot;
 use solana_sdk::hash::Hash;
 use std::{
     path::PathBuf,
-    sync::{
-        mpsc::{Receiver, SendError, Sender},
-        Arc,
-    },
+    sync::mpsc::{Receiver, SendError, Sender},
 };
 use tempfile::TempDir;
 
@@ -19,7 +16,7 @@ pub struct SnapshotPackage {
     pub root: Slot,
     pub slot_deltas: Vec<BankSlotDelta>,
     pub snapshot_links: TempDir,
-    pub storage_entries: Vec<Arc<AccountStorageEntry>>,
+    pub storages: SnapshotStorages,
     pub tar_output_file: PathBuf,
     pub hash: Hash,
 }
@@ -29,15 +26,15 @@ impl SnapshotPackage {
         root: Slot,
         slot_deltas: Vec<BankSlotDelta>,
         snapshot_links: TempDir,
-        storage_entries: Vec<Arc<AccountStorageEntry>>,
+        storages: SnapshotStorages,
         tar_output_file: PathBuf,
         hash: Hash,
     ) -> Self {
         Self {
             root,
             slot_deltas,
             snapshot_links,
-            storage_entries,
+            storages,
             tar_output_file,
             hash,
         }

ledger/src/snapshot_utils.rs

@@ -4,8 +4,12 @@ use bzip2::bufread::BzDecoder;
 use fs_extra::dir::CopyOptions;
 use log::*;
 use solana_measure::measure::Measure;
-use solana_runtime::bank::{
-    self, deserialize_from_snapshot, Bank, BankSlotDelta, MAX_SNAPSHOT_DATA_FILE_SIZE,
+use solana_runtime::{
+    accounts_db::{SnapshotStorage, SnapshotStorages},
+    bank::{
+        self, deserialize_from_snapshot, Bank, BankRcSerialize, BankSlotDelta,
+        MAX_SNAPSHOT_DATA_FILE_SIZE,
+    },
 };
 use solana_sdk::clock::Slot;
 use std::{
@@ -80,23 +84,16 @@ pub fn package_snapshot<P: AsRef<Path>, Q: AsRef<Path>>(
     snapshot_package_output_file: P,
     snapshot_path: Q,
     slots_to_snapshot: &[Slot],
+    snapshot_storages: SnapshotStorages,
 ) -> Result<SnapshotPackage> {
     // Hard link all the snapshots we need for this package
     let snapshot_hard_links_dir = tempfile::tempdir_in(snapshot_path)?;
 
-    // Get a reference to all the relevant AccountStorageEntries
-    let account_storage_entries: Vec<_> = bank
-        .rc
-        .get_rooted_storage_entries()
-        .into_iter()
-        .filter(|x| x.slot_id() <= bank.slot())
-        .collect();
-
     // Create a snapshot package
     info!(
         "Snapshot for bank: {} has {} account storage entries",
         bank.slot(),
-        account_storage_entries.len()
+        snapshot_storages.len()
     );
 
     // Any errors from this point on will cause the above SnapshotPackage to drop, clearing
@@ -107,7 +104,7 @@ pub fn package_snapshot<P: AsRef<Path>, Q: AsRef<Path>>(
         bank.slot(),
         bank.src.slot_deltas(slots_to_snapshot),
         snapshot_hard_links_dir,
-        account_storage_entries,
+        snapshot_storages,
         snapshot_package_output_file.as_ref().to_path_buf(),
         bank.hash(),
     );
@@ -149,7 +146,7 @@ pub fn archive_snapshot_package(snapshot_package: &SnapshotPackage) -> Result<()
     )?;
 
     // Add the AppendVecs into the compressible list
-    for storage in &snapshot_package.storage_entries {
+    for storage in snapshot_package.storages.iter().flatten() {
         storage.flush()?;
         let storage_path = storage.get_path();
         let output_path = staging_accounts_dir.join(
@@ -321,7 +318,11 @@ where
     Ok(ret)
 }
 
-pub fn add_snapshot<P: AsRef<Path>>(snapshot_path: P, bank: &Bank) -> Result<SlotSnapshotPaths> {
+pub fn add_snapshot<P: AsRef<Path>>(
+    snapshot_path: P,
+    bank: &Bank,
+    snapshot_storages: &[SnapshotStorage],
+) -> Result<SlotSnapshotPaths> {
     bank.purge_zero_lamport_accounts();
     let slot = bank.slot();
     // snapshot_path/slot
@@ -341,7 +342,13 @@ pub fn add_snapshot<P: AsRef<Path>>(snapshot_path: P, bank: &Bank) -> Result<Slo
         MAX_SNAPSHOT_DATA_FILE_SIZE,
         |stream| {
             serialize_into(stream.by_ref(), &*bank)?;
-            serialize_into(stream.by_ref(), &bank.rc)?;
+            serialize_into(
+                stream.by_ref(),
+                &BankRcSerialize {
+                    bank_rc: &bank.rc,
+                    snapshot_storages,
+                },
+            )?;
             Ok(())
         },
     )?;

runtime/src/accounts.rs

@@ -1333,7 +1333,11 @@ mod tests {
         let mut writer = Cursor::new(vec![]);
         serialize_into(
             &mut writer,
-            &AccountsDBSerialize::new(&*accounts.accounts_db, 0),
+            &AccountsDBSerialize::new(
+                &*accounts.accounts_db,
+                0,
+                &accounts.accounts_db.get_snapshot_storages(0),
+            ),
         )
         .unwrap();