perf(serde_snapshot): build versioned epoch stakes in thread

runtime/src/bank.rs

@@ -457,7 +457,9 @@ pub struct BankFieldsToDeserialize {
     pub(crate) epoch_schedule: EpochSchedule,
     pub(crate) inflation: Inflation,
     pub(crate) stakes: DeserializableStakes<Delegation>,
-    pub(crate) versioned_epoch_stakes: HashMap<Epoch, DeserializableVersionedEpochStakes>,
+    /// Transformed into `HashMap<Epoch, VersionedEpochStakes>` in `serde_snapshot` and passed to
+    /// `Bank::new_from_snapshot` as separate parameter for performance (conversion is time consuming)
+    pub(crate) versioned_epoch_stakes: Vec<(Epoch, DeserializableVersionedEpochStakes)>,
     pub(crate) is_delta: bool,
     pub(crate) accounts_data_len: u64,
     pub(crate) accounts_lt_hash: AccountsLtHash,
@@ -1816,6 +1818,7 @@ impl Bank {
         fields: BankFieldsToDeserialize,
         debug_keys: Option<Arc<HashSet<Pubkey>>>,
         accounts_data_size_initial: u64,
+        epoch_stakes: HashMap<Epoch, VersionedEpochStakes>,
     ) -> Self {
         let now = Instant::now();
         let ancestors = Ancestors::from(&fields.ancestors);
@@ -1843,6 +1846,14 @@ impl Bank {
              snapshot or bugs in cached accounts or accounts-db.",
         ));
         info!("Loading Stakes took: {stakes_time}");
+        assert!(
+            fields.versioned_epoch_stakes.is_empty(),
+            "should be already converted and passed in epoch_stakes parameter"
+        );
+        assert!(
+            !epoch_stakes.is_empty(),
+            "should be populated (from fields.versioned_epoch_stakes)"
+        );
         let stakes_accounts_load_duration = now.elapsed();
         let mut bank = Self {
             rc: bank_rc,
@@ -1879,11 +1890,7 @@ impl Bank {
             epoch_schedule: fields.epoch_schedule,
             inflation: Arc::new(RwLock::new(fields.inflation)),
             stakes_cache: StakesCache::new(stakes),
-            epoch_stakes: fields
-                .versioned_epoch_stakes
-                .into_iter()
-                .map(|(k, v)| (k, v.into()))
-                .collect(),
+            epoch_stakes,
             is_delta: AtomicBool::new(fields.is_delta),
             rewards: RwLock::new(vec![]),
             cluster_type: Some(genesis_config.cluster_type),

runtime/src/serde_snapshot.rs

@@ -53,6 +53,7 @@ use {
             atomic::{AtomicBool, AtomicUsize, Ordering},
             Arc,
         },
+        thread,
         time::Instant,
     },
     storage::SerializableStorage,
@@ -222,7 +223,7 @@ impl From<DeserializableVersionedBank> for BankFieldsToDeserialize {
             inflation: dvb.inflation,
             stakes: dvb.stakes,
             is_delta: dvb.is_delta,
-            versioned_epoch_stakes: HashMap::default(), // populated from ExtraFieldsToDeserialize
+            versioned_epoch_stakes: vec![], // populated from ExtraFieldsToDeserialize
             accounts_lt_hash: AccountsLtHash(LT_HASH_CANARY), // populated from ExtraFieldsToDeserialize
             bank_hash_stats: BankHashStats::default(),        // populated from AccountsDbFields
         }
@@ -425,7 +426,7 @@ struct ExtraFieldsToDeserialize {
     #[serde(deserialize_with = "default_on_eof")]
     _obsolete_epoch_accounts_hash: Option<Hash>,
     #[serde(deserialize_with = "default_on_eof")]
-    versioned_epoch_stakes: HashMap<u64, DeserializableVersionedEpochStakes>,
+    versioned_epoch_stakes: Vec<(u64, DeserializableVersionedEpochStakes)>,
     #[serde(deserialize_with = "default_on_eof")]
     accounts_lt_hash: Option<SerdeAccountsLtHash>,
 }
@@ -799,6 +800,16 @@ where
     E: SerializableStorage + std::marker::Sync,
 {
     let mut bank_fields = bank_fields.collapse_into();
+    // Epoch stakes take several seconds to reconstruct, do it in parallel with loading accountsdb
+    let deserializable_epoch_stakes = std::mem::take(&mut bank_fields.versioned_epoch_stakes);
+    let epoch_stakes_handle = thread::Builder::new()
+        .name("solRctEpochStk".into())
+        .spawn(|| {
+            deserializable_epoch_stakes
+                .into_iter()
+                .map(|(epoch, stakes)| (epoch, stakes.into()))
+                .collect()
+        })?;
     let (accounts_db, reconstructed_accounts_db_info) = reconstruct_accountsdb_from_fields(
         snapshot_accounts_db_fields,
         account_paths,
@@ -813,6 +824,7 @@ where
 
     let bank_rc = BankRc::new(Accounts::new(Arc::new(accounts_db)));
     let runtime_config = Arc::new(runtime_config.clone());
+    let epoch_stakes = epoch_stakes_handle.join().expect("calculate epoch stakes");
 
     let bank = Bank::new_from_snapshot(
         bank_rc,
@@ -821,6 +833,7 @@ where
         bank_fields,
         debug_keys,
         reconstructed_accounts_db_info.accounts_data_len,
+        epoch_stakes,
     );
 
     info!("rent_collector: {:?}", bank.rent_collector());