Add "newer" snapshot serde style with accounts_data_len (and more!)

core/tests/snapshots.rs

@@ -103,6 +103,11 @@ mod tests {
     DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_2_0, Testnet, V1_2_0_Testnet);
     DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_2_0, MainnetBeta, V1_2_0_MainnetBeta);
 
+    DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_3_0, Development, V1_3_0_Development);
+    DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_3_0, Devnet, V1_3_0_Devnet);
+    DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_3_0, Testnet, V1_3_0_Testnet);
+    DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_3_0, MainnetBeta, V1_3_0_MainnetBeta);
+
     struct SnapshotTestConfig {
         accounts_dir: TempDir,
         bank_snapshots_dir: TempDir,

runtime/src/bank.rs

@@ -864,8 +864,6 @@ pub(crate) struct BankFieldsToDeserialize {
     pub(crate) ns_per_slot: u128,
     pub(crate) genesis_creation_time: UnixTimestamp,
     pub(crate) slots_per_year: f64,
-    #[allow(dead_code)]
-    pub(crate) unused: u64,
     pub(crate) slot: Slot,
     pub(crate) epoch: Epoch,
     pub(crate) block_height: u64,
@@ -880,6 +878,7 @@ pub(crate) struct BankFieldsToDeserialize {
     pub(crate) stakes: Stakes,
     pub(crate) epoch_stakes: HashMap<Epoch, EpochStakes>,
     pub(crate) is_delta: bool,
+    pub(crate) accounts_data_len: Option<u64>, // TODO: Remove Option wrapper once "newer" snapshot serde style is the only one supported
 }
 
 // Bank's common fields shared by all supported snapshot versions for serialization.
@@ -904,7 +903,6 @@ pub(crate) struct BankFieldsToSerialize<'a> {
     pub(crate) ns_per_slot: u128,
     pub(crate) genesis_creation_time: UnixTimestamp,
     pub(crate) slots_per_year: f64,
-    pub(crate) unused: u64,
     pub(crate) slot: Slot,
     pub(crate) epoch: Epoch,
     pub(crate) block_height: u64,
@@ -919,6 +917,7 @@ pub(crate) struct BankFieldsToSerialize<'a> {
     pub(crate) stakes: &'a StakesCache,
     pub(crate) epoch_stakes: &'a HashMap<Epoch, EpochStakes>,
     pub(crate) is_delta: bool,
+    pub(crate) accounts_data_len: u64,
 }
 
 // Can't derive PartialEq because RwLock doesn't implement PartialEq
@@ -943,7 +942,6 @@ impl PartialEq for Bank {
             && self.ns_per_slot == other.ns_per_slot
             && self.genesis_creation_time == other.genesis_creation_time
             && self.slots_per_year == other.slots_per_year
-            && self.unused == other.unused
             && self.slot == other.slot
             && self.epoch == other.epoch
             && self.block_height == other.block_height
@@ -1095,9 +1093,6 @@ pub struct Bank {
     /// The number of slots per year, used for inflation
     slots_per_year: f64,
 
-    /// Unused
-    unused: u64,
-
     /// Bank slot (i.e. block)
     slot: Slot,
 
@@ -1300,7 +1295,6 @@ impl Bank {
             ns_per_slot: u128::default(),
             genesis_creation_time: UnixTimestamp::default(),
             slots_per_year: f64::default(),
-            unused: u64::default(),
             slot: Slot::default(),
             bank_id: BankId::default(),
             epoch: Epoch::default(),
@@ -1531,7 +1525,6 @@ impl Bank {
             ticks_per_slot: parent.ticks_per_slot,
             ns_per_slot: parent.ns_per_slot,
             genesis_creation_time: parent.genesis_creation_time,
-            unused: parent.unused,
             slots_per_year: parent.slots_per_year,
             epoch_schedule,
             collected_rent: AtomicU64::new(0),
@@ -1711,7 +1704,7 @@ impl Bank {
         debug_keys: Option<Arc<HashSet<Pubkey>>>,
         additional_builtins: Option<&Builtins>,
         debug_do_not_add_builtins: bool,
-        accounts_data_len: u64,
+        computed_accounts_data_len: u64, // TODO: Remove this parameter once "newer" snapshot serde style is the only one supported
     ) -> Self {
         fn new<T: Default>() -> T {
             T::default()
@@ -1738,7 +1731,6 @@ impl Bank {
             ns_per_slot: fields.ns_per_slot,
             genesis_creation_time: fields.genesis_creation_time,
             slots_per_year: fields.slots_per_year,
-            unused: genesis_config.unused,
             slot: fields.slot,
             bank_id: 0,
             epoch: fields.epoch,
@@ -1775,8 +1767,13 @@ impl Bank {
             vote_only_bank: false,
             cost_tracker: RwLock::new(CostTracker::default()),
             sysvar_cache: RwLock::new(SysvarCache::default()),
-            accounts_data_len: AtomicU64::new(accounts_data_len),
+            accounts_data_len: AtomicU64::new(
+                fields
+                    .accounts_data_len
+                    .map_or(computed_accounts_data_len, |len| len),
+            ),
         };
+
         bank.finish_init(
             genesis_config,
             additional_builtins,
@@ -1798,7 +1795,6 @@ impl Bank {
                 * genesis_config.ticks_per_slot as u128
         );
         assert_eq!(bank.genesis_creation_time, genesis_config.creation_time);
-        assert_eq!(bank.unused, genesis_config.unused);
         assert_eq!(bank.max_tick_height, (bank.slot + 1) * bank.ticks_per_slot);
         assert_eq!(
             bank.slots_per_year,
@@ -1843,7 +1839,6 @@ impl Bank {
             ns_per_slot: self.ns_per_slot,
             genesis_creation_time: self.genesis_creation_time,
             slots_per_year: self.slots_per_year,
-            unused: self.unused,
             slot: self.slot,
             epoch: self.epoch,
             block_height: self.block_height,
@@ -1858,6 +1853,7 @@ impl Bank {
             stakes: &self.stakes_cache,
             epoch_stakes: &self.epoch_stakes,
             is_delta: self.is_delta.load(Relaxed),
+            accounts_data_len: self.load_accounts_data_len(),
         }
     }
 
@@ -1980,11 +1976,6 @@ impl Bank {
         )
     }
 
-    /// Unused conversion
-    pub fn get_unused_from_slot(rooted_slot: Slot, unused: u64) -> u64 {
-        (rooted_slot + (unused - 1)) / unused
-    }
-
     pub fn clock(&self) -> sysvar::clock::Clock {
         from_account(&self.get_account(&sysvar::clock::id()).unwrap_or_default())
             .unwrap_or_default()
@@ -2910,7 +2901,6 @@ impl Bank {
         self.ticks_per_slot = genesis_config.ticks_per_slot();
         self.ns_per_slot = genesis_config.ns_per_slot();
         self.genesis_creation_time = genesis_config.creation_time;
-        self.unused = genesis_config.unused;
         self.max_tick_height = (self.slot + 1) * self.ticks_per_slot;
         self.slots_per_year = genesis_config.slots_per_year();
 

runtime/src/serde_snapshot.rs

@@ -44,8 +44,8 @@ use {
     },
 };
 
-mod common;
 mod newer;
+mod older;
 mod storage;
 mod tests;
 mod utils;
@@ -59,6 +59,7 @@ pub(crate) use tests::reconstruct_accounts_db_via_serialization;
 #[derive(Copy, Clone, Eq, PartialEq)]
 pub(crate) enum SerdeStyle {
     Newer,
+    Older,
 }
 
 const MAX_STREAM_SIZE: u64 = 32 * 1024 * 1024 * 1024;
@@ -239,6 +240,7 @@ where
     }
     match serde_style {
         SerdeStyle::Newer => INTO!(newer),
+        SerdeStyle::Older => INTO!(older),
     }
     .map_err(|err| {
         warn!("bankrc_from_stream error: {:?}", err);
@@ -269,6 +271,7 @@ where
     }
     match serde_style {
         SerdeStyle::Newer => INTO!(newer),
+        SerdeStyle::Older => INTO!(older),
     }
     .map_err(|err| {
         warn!("bankrc_to_stream error: {:?}", err);

runtime/src/serde_snapshot/newer.rs

@@ -1,6 +1,5 @@
 use {
     super::{
-        common::UnusedAccounts,
         storage::SerializableAccountStorageEntry,
         utils::{serialize_iter_as_map, serialize_iter_as_seq},
         *,
@@ -33,7 +32,6 @@ struct DeserializableVersionedBank {
     ns_per_slot: u128,
     genesis_creation_time: UnixTimestamp,
     slots_per_year: f64,
-    unused: u64,
     slot: Slot,
     epoch: Epoch,
     block_height: u64,
@@ -46,10 +44,9 @@ struct DeserializableVersionedBank {
     epoch_schedule: EpochSchedule,
     inflation: Inflation,
     stakes: Stakes,
-    #[allow(dead_code)]
-    unused_accounts: UnusedAccounts,
     epoch_stakes: HashMap<Epoch, EpochStakes>,
     is_delta: bool,
+    accounts_data_len: u64,
 }
 
 impl From<DeserializableVersionedBank> for BankFieldsToDeserialize {
@@ -71,7 +68,6 @@ impl From<DeserializableVersionedBank> for BankFieldsToDeserialize {
             ns_per_slot: dvb.ns_per_slot,
             genesis_creation_time: dvb.genesis_creation_time,
             slots_per_year: dvb.slots_per_year,
-            unused: dvb.unused,
             slot: dvb.slot,
             epoch: dvb.epoch,
             block_height: dvb.block_height,
@@ -86,6 +82,7 @@ impl From<DeserializableVersionedBank> for BankFieldsToDeserialize {
             stakes: dvb.stakes,
             epoch_stakes: dvb.epoch_stakes,
             is_delta: dvb.is_delta,
+            accounts_data_len: Some(dvb.accounts_data_len),
         }
     }
 }
@@ -110,7 +107,6 @@ struct SerializableVersionedBank<'a> {
     ns_per_slot: u128,
     genesis_creation_time: UnixTimestamp,
     slots_per_year: f64,
-    unused: u64,
     slot: Slot,
     epoch: Epoch,
     block_height: u64,
@@ -123,16 +119,13 @@ struct SerializableVersionedBank<'a> {
     epoch_schedule: EpochSchedule,
     inflation: Inflation,
     stakes: &'a StakesCache,
-    unused_accounts: UnusedAccounts,
     epoch_stakes: &'a HashMap<Epoch, EpochStakes>,
     is_delta: bool,
+    accounts_data_len: u64,
 }
 
 impl<'a> From<crate::bank::BankFieldsToSerialize<'a>> for SerializableVersionedBank<'a> {
     fn from(rhs: crate::bank::BankFieldsToSerialize<'a>) -> Self {
-        fn new<T: Default>() -> T {
-            T::default()
-        }
         Self {
             blockhash_queue: rhs.blockhash_queue,
             ancestors: rhs.ancestors,
@@ -150,7 +143,6 @@ impl<'a> From<crate::bank::BankFieldsToSerialize<'a>> for SerializableVersionedB
             ns_per_slot: rhs.ns_per_slot,
             genesis_creation_time: rhs.genesis_creation_time,
             slots_per_year: rhs.slots_per_year,
-            unused: rhs.unused,
             slot: rhs.slot,
             epoch: rhs.epoch,
             block_height: rhs.block_height,
@@ -163,9 +155,9 @@ impl<'a> From<crate::bank::BankFieldsToSerialize<'a>> for SerializableVersionedB
             epoch_schedule: rhs.epoch_schedule,
             inflation: rhs.inflation,
             stakes: rhs.stakes,
-            unused_accounts: new(),
             epoch_stakes: rhs.epoch_stakes,
             is_delta: rhs.is_delta,
+            accounts_data_len: rhs.accounts_data_len,
         }
     }
 }