From 59e0af126bef639493531a1a479880d3165a83c2 Mon Sep 17 00:00:00 2001 From: Kristofer Peterson Date: Sun, 14 Jun 2020 23:19:00 +0200 Subject: [PATCH 1/4] Move bank (de)serialisation logic from bank and snapshot_utils to serde_snapshot. Add sanity assertions between genesis config and bank fields on deserialisation. Atomically update atomic bool in quote_for_specialization_detection(). Use same genesis config when restoring snapshots in test cases. --- core/tests/bank_forks.rs | 11 +- programs/librapay/src/lib.rs | 4 +- runtime/src/accounts.rs | 32 +--- runtime/src/accounts_db.rs | 4 +- runtime/src/bank.rs | 234 +++++++++++++++++++++++---- runtime/src/rent_collector.rs | 2 +- runtime/src/serde_snapshot.rs | 155 +++++++++++++----- runtime/src/serde_snapshot/common.rs | 171 ++++++++++++++++++++ runtime/src/serde_snapshot/future.rs | 42 +++-- runtime/src/serde_snapshot/legacy.rs | 49 ++++-- runtime/src/serde_snapshot/tests.rs | 80 +++++---- runtime/src/snapshot_utils.rs | 40 ++--- sdk/macro-frozen-abi/src/lib.rs | 7 +- 13 files changed, 628 insertions(+), 203 deletions(-) create mode 100644 runtime/src/serde_snapshot/common.rs diff --git a/core/tests/bank_forks.rs b/core/tests/bank_forks.rs index 1ef9914b5f3..4ffb56d3828 100644 --- a/core/tests/bank_forks.rs +++ b/core/tests/bank_forks.rs @@ -110,6 +110,7 @@ mod tests { fn restore_from_snapshot( old_bank_forks: &BankForks, old_last_slot: Slot, + old_genesis_config: &GenesisConfig, account_paths: &[PathBuf], ) { let (snapshot_path, snapshot_package_output_path) = old_bank_forks @@ -134,7 +135,7 @@ mod tests { &CompressionType::Bzip2, ), CompressionType::Bzip2, - &GenesisConfig::default(), + old_genesis_config, ) .unwrap(); @@ -169,7 +170,6 @@ mod tests { let mut snapshot_test_config = SnapshotTestConfig::new(snapshot_version, 1); let bank_forks = &mut snapshot_test_config.bank_forks; - let accounts_dir = &snapshot_test_config.accounts_dir; let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair; let (s, _r) = channel(); @@ -185,6 +185,7 @@ mod tests { bank_forks.set_root(bank.slot(), &sender, None); } } + // Generate a snapshot package for last bank let last_bank = bank_forks.get(last_slot).unwrap(); let snapshot_config = &snapshot_test_config.snapshot_config; @@ -203,10 +204,12 @@ mod tests { snapshot_version, ) .unwrap(); - snapshot_utils::archive_snapshot_package(&snapshot_package).unwrap(); - restore_from_snapshot(bank_forks, last_slot, &[accounts_dir.path().to_path_buf()]); + // Restore bank from snapshot + let account_paths = &[snapshot_test_config.accounts_dir.path().to_path_buf()]; + let genesis_config = &snapshot_test_config.genesis_config_info.genesis_config; + restore_from_snapshot(bank_forks, last_slot, genesis_config, account_paths); } fn run_test_bank_forks_snapshot_n(snapshot_version: SnapshotVersion) { diff --git a/programs/librapay/src/lib.rs b/programs/librapay/src/lib.rs index e24a199f70a..68efbfa6cb2 100644 --- a/programs/librapay/src/lib.rs +++ b/programs/librapay/src/lib.rs @@ -38,7 +38,9 @@ pub fn create_genesis(from: &Keypair, client: &T, amount: u64) -> Key let instruction = librapay_instruction::genesis(&genesis.pubkey(), amount); let message = Message::new(&[instruction], Some(&from.pubkey())); - client.send_and_confirm_message(&[from, &genesis], message).unwrap(); + client + .send_and_confirm_message(&[from, &genesis], message) + .unwrap(); genesis } diff --git a/runtime/src/accounts.rs b/runtime/src/accounts.rs index 6b8342821d8..2eab90fd000 100644 --- a/runtime/src/accounts.rs +++ b/runtime/src/accounts.rs @@ -65,23 +65,19 @@ pub enum AccountAddressFilter { } impl Accounts { - pub(crate) fn new_empty(accounts_db: AccountsDB) -> Self { + pub fn new(paths: Vec) -> Self { Self { - accounts_db: Arc::new(accounts_db), + slot: 0, + accounts_db: Arc::new(AccountsDB::new(paths)), account_locks: Mutex::new(HashSet::new()), readonly_locks: Arc::new(RwLock::new(Some(HashMap::new()))), - ..Self::default() } } - pub fn new(paths: Vec) -> Self { - Self::new_with_frozen_accounts(paths, &HashMap::default(), &[]) - } - pub fn new_from_parent(parent: &Accounts, slot: Slot, parent_slot: Slot) -> Self { let accounts_db = parent.accounts_db.clone(); accounts_db.set_hash(slot, parent_slot); - Accounts { + Self { slot, accounts_db, account_locks: Mutex::new(HashSet::new()), @@ -89,25 +85,13 @@ impl Accounts { } } - pub fn new_with_frozen_accounts( - paths: Vec, - ancestors: &Ancestors, - frozen_account_pubkeys: &[Pubkey], - ) -> Self { - let mut accounts = Accounts { + pub(crate) fn new_empty(accounts_db: AccountsDB) -> Self { + Self { slot: 0, - accounts_db: Arc::new(AccountsDB::new(paths)), + accounts_db: Arc::new(accounts_db), account_locks: Mutex::new(HashSet::new()), readonly_locks: Arc::new(RwLock::new(Some(HashMap::new()))), - }; - accounts.freeze_accounts(ancestors, frozen_account_pubkeys); - accounts - } - - pub fn freeze_accounts(&mut self, ancestors: &Ancestors, frozen_account_pubkeys: &[Pubkey]) { - Arc::get_mut(&mut self.accounts_db) - .unwrap() - .freeze_accounts(ancestors, frozen_account_pubkeys); + } } /// Return true if the slice has any duplicate elements diff --git a/runtime/src/accounts_db.rs b/runtime/src/accounts_db.rs index b24169d2d59..e34e407a086 100644 --- a/runtime/src/accounts_db.rs +++ b/runtime/src/accounts_db.rs @@ -1782,7 +1782,7 @@ impl AccountsDB { hashes } - pub fn freeze_accounts(&mut self, ancestors: &Ancestors, account_pubkeys: &[Pubkey]) { + pub(crate) fn freeze_accounts(&mut self, ancestors: &Ancestors, account_pubkeys: &[Pubkey]) { for account_pubkey in account_pubkeys { if let Some((account, _slot)) = self.load_slow(ancestors, &account_pubkey) { let frozen_account_info = FrozenAccountInfo { @@ -1976,7 +1976,7 @@ impl AccountsDB { } } - pub fn print_accounts_stats(&self, label: &'static str) { + pub(crate) fn print_accounts_stats(&self, label: &'static str) { self.print_index(label); self.print_count_and_status(label); } diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 062edfd9f57..6037f99c097 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -25,7 +25,6 @@ use crate::{ use byteorder::{ByteOrder, LittleEndian}; use itertools::Itertools; use log::*; -use serde::{Deserialize, Serialize}; use solana_measure::measure::Measure; use solana_metrics::{ datapoint_debug, inc_new_counter_debug, inc_new_counter_error, inc_new_counter_info, @@ -108,20 +107,6 @@ pub struct BankRc { pub(crate) slot: Slot, } -#[cfg(RUSTC_WITH_SPECIALIZATION)] -use solana_sdk::abi_example::AbiExample; -#[cfg(RUSTC_WITH_SPECIALIZATION)] -impl AbiExample for BankRc { - fn example() -> Self { - BankRc { - // Set parent to None to cut the recursion into another Bank - parent: RwLock::new(None), - accounts: AbiExample::example(), - slot: AbiExample::example(), - } - } -} - impl BankRc { pub(crate) fn new(accounts: Accounts, slot: Slot) -> Self { Self { @@ -203,22 +188,81 @@ impl HashAgeKind { } } -#[derive(Default, Clone, PartialEq, Debug, Deserialize, Serialize, AbiExample)] -struct UnusedAccounts { - unused1: HashSet, - unused2: HashSet, - unused3: HashMap, +#[derive(Clone, Default)] +pub(crate) struct BankFields { + pub(crate) blockhash_queue: BlockhashQueue, + pub(crate) ancestors: Ancestors, + pub(crate) hash: Hash, + pub(crate) parent_hash: Hash, + pub(crate) parent_slot: Slot, + pub(crate) hard_forks: HardForks, + pub(crate) transaction_count: u64, + pub(crate) tick_height: u64, + pub(crate) signature_count: u64, + pub(crate) capitalization: u64, + pub(crate) max_tick_height: u64, + pub(crate) hashes_per_tick: Option, + pub(crate) ticks_per_slot: u64, + pub(crate) ns_per_slot: u128, + pub(crate) genesis_creation_time: UnixTimestamp, + pub(crate) slots_per_year: f64, + pub(crate) unused: u64, + pub(crate) slot: Slot, + pub(crate) epoch: Epoch, + pub(crate) block_height: u64, + pub(crate) collector_id: Pubkey, + pub(crate) collector_fees: u64, + pub(crate) fee_calculator: FeeCalculator, + pub(crate) fee_rate_governor: FeeRateGovernor, + pub(crate) collected_rent: u64, + pub(crate) rent_collector: RentCollector, + pub(crate) epoch_schedule: EpochSchedule, + pub(crate) inflation: Inflation, + pub(crate) stakes: Stakes, + pub(crate) epoch_stakes: HashMap, + pub(crate) is_delta: bool, +} + +pub(crate) struct RefBankFields<'a> { + pub(crate) blockhash_queue: &'a RwLock, + pub(crate) ancestors: &'a Ancestors, + pub(crate) hash: Hash, + pub(crate) parent_hash: Hash, + pub(crate) parent_slot: Slot, + pub(crate) hard_forks: &'a RwLock, + pub(crate) transaction_count: u64, + pub(crate) tick_height: u64, + pub(crate) signature_count: u64, + pub(crate) capitalization: u64, + pub(crate) max_tick_height: u64, + pub(crate) hashes_per_tick: Option, + pub(crate) ticks_per_slot: u64, + pub(crate) ns_per_slot: u128, + pub(crate) genesis_creation_time: UnixTimestamp, + pub(crate) slots_per_year: f64, + pub(crate) unused: u64, + pub(crate) slot: Slot, + pub(crate) epoch: Epoch, + pub(crate) block_height: u64, + pub(crate) collector_id: Pubkey, + pub(crate) collector_fees: u64, + pub(crate) fee_calculator: FeeCalculator, + pub(crate) fee_rate_governor: FeeRateGovernor, + pub(crate) collected_rent: u64, + pub(crate) rent_collector: RentCollector, + pub(crate) epoch_schedule: EpochSchedule, + pub(crate) inflation: Inflation, + pub(crate) stakes: &'a RwLock, + pub(crate) epoch_stakes: &'a HashMap, + pub(crate) is_delta: bool, } /// Manager for the state of all accounts and programs after processing its entries. -#[frozen_abi(digest = "GsvisJfTaHxmTX4s6gQs2bZtxVggB7F325XDdXTA573p")] -#[derive(Default, Deserialize, Serialize, AbiExample)] +#[derive(Default)] pub struct Bank { /// References to accounts, parent and signature status - #[serde(skip)] pub rc: BankRc, - #[serde(skip)] pub src: StatusCacheRc, /// FIFO queue of `recent_blockhash` items @@ -308,9 +352,6 @@ pub struct Bank { /// cache of vote_account and stake_account state for this fork stakes: RwLock, - /// unused - unused_accounts: RwLock, - /// staked nodes on epoch boundaries, saved off when a bank.slot() is at /// a leader schedule calculation boundary epoch_stakes: HashMap, @@ -324,24 +365,18 @@ pub struct Bank { /// Callback to be notified when a bank enters a new Epoch /// (used to adjust cluster features over time) - #[serde(skip)] entered_epoch_callback: Arc>>, /// Last time when the cluster info vote listener has synced with this bank - #[serde(skip)] pub last_vote_sync: AtomicU64, /// Rewards that were paid out immediately after this bank was created - #[serde(skip)] pub rewards: Option>, - #[serde(skip)] pub skip_drop: AtomicBool, - #[serde(skip)] pub operating_mode: Option, - #[serde(skip)] pub lazy_rent_collection: AtomicBool, } @@ -370,7 +405,7 @@ impl Bank { bank.finish_init(); // Freeze accounts after process_genesis_config creates the initial append vecs - Arc::get_mut(&mut bank.rc.accounts) + Arc::get_mut(&mut Arc::get_mut(&mut bank.rc.accounts).unwrap().accounts_db) .unwrap() .freeze_accounts(&bank.ancestors, frozen_account_pubkeys); @@ -440,7 +475,6 @@ impl Bank { transaction_count: AtomicU64::new(parent.transaction_count()), stakes: RwLock::new(parent.stakes.read().unwrap().clone_with_epoch(epoch)), epoch_stakes: parent.epoch_stakes.clone(), - unused_accounts: RwLock::new(parent.unused_accounts.read().unwrap().clone()), parent_hash: parent.hash(), parent_slot: parent.slot(), collector_id: *collector_id, @@ -514,6 +548,136 @@ impl Bank { new } + /// Create a bank from explicit arguments and deserialized fields from snapshot + #[allow(clippy::float_cmp)] + pub(crate) fn new_from_fields( + bank_rc: BankRc, + genesis_config: &GenesisConfig, + fields: BankFields, + ) -> Self { + fn new() -> T { + T::default() + } + let mut bank = Self { + rc: bank_rc, + src: new(), + blockhash_queue: RwLock::new(fields.blockhash_queue), + ancestors: fields.ancestors, + hash: RwLock::new(fields.hash), + parent_hash: fields.parent_hash, + parent_slot: fields.parent_slot, + hard_forks: Arc::new(RwLock::new(fields.hard_forks)), + transaction_count: AtomicU64::new(fields.transaction_count), + tick_height: AtomicU64::new(fields.tick_height), + signature_count: AtomicU64::new(fields.signature_count), + capitalization: AtomicU64::new(fields.capitalization), + max_tick_height: fields.max_tick_height, + hashes_per_tick: fields.hashes_per_tick, + ticks_per_slot: fields.ticks_per_slot, + ns_per_slot: fields.ns_per_slot, + genesis_creation_time: fields.genesis_creation_time, + slots_per_year: fields.slots_per_year, + unused: genesis_config.unused, + slot: fields.slot, + epoch: fields.epoch, + block_height: fields.block_height, + collector_id: fields.collector_id, + collector_fees: AtomicU64::new(fields.collector_fees), + fee_calculator: fields.fee_calculator, + fee_rate_governor: fields.fee_rate_governor, + collected_rent: AtomicU64::new(fields.collected_rent), + rent_collector: fields.rent_collector, + epoch_schedule: fields.epoch_schedule, + inflation: Arc::new(RwLock::new(fields.inflation)), + stakes: RwLock::new(fields.stakes), + epoch_stakes: fields.epoch_stakes, + is_delta: AtomicBool::new(fields.is_delta), + message_processor: new(), + entered_epoch_callback: new(), + last_vote_sync: new(), + rewards: new(), + skip_drop: new(), + operating_mode: Some(genesis_config.operating_mode), + lazy_rent_collection: new(), + }; + bank.finish_init(); + + // Sanity assertions between bank snapshot and genesis config + // Consider removing from serializable bank state ([Ref]BankFields) and initializing + // from the passed in genesis_config instead (as new()/new_with_paths() already do) + assert_eq!( + bank.hashes_per_tick, + genesis_config.poh_config.hashes_per_tick + ); + assert_eq!(bank.ticks_per_slot, genesis_config.ticks_per_slot); + assert_eq!( + bank.ns_per_slot, + genesis_config.poh_config.target_tick_duration.as_nanos() + * genesis_config.ticks_per_slot as u128 + ); + assert_eq!(bank.genesis_creation_time, genesis_config.creation_time); + assert_eq!(bank.unused, genesis_config.unused); + assert_eq!(bank.max_tick_height, (bank.slot + 1) * bank.ticks_per_slot); + assert_eq!( + bank.slots_per_year, + years_as_slots( + 1.0, + &genesis_config.poh_config.target_tick_duration, + bank.ticks_per_slot, + ) + ); + assert_eq!(bank.epoch_schedule, genesis_config.epoch_schedule); + assert_eq!(bank.epoch, bank.epoch_schedule.get_epoch(bank.slot)); + assert_eq!( + bank.rent_collector, + RentCollector::new( + bank.epoch, + &bank.epoch_schedule, + bank.slots_per_year, + &genesis_config.rent, + ) + ); + + bank + } + + /// Return subset of bank fields representing serializable state + pub(crate) fn get_ref_fields(&self) -> RefBankFields { + RefBankFields { + blockhash_queue: &self.blockhash_queue, + ancestors: &self.ancestors, + hash: *self.hash.read().unwrap(), + parent_hash: self.parent_hash, + parent_slot: self.parent_slot, + hard_forks: &*self.hard_forks, + transaction_count: self.transaction_count.load(Ordering::Relaxed), + tick_height: self.tick_height.load(Ordering::Relaxed), + signature_count: self.signature_count.load(Ordering::Relaxed), + capitalization: self.capitalization.load(Ordering::Relaxed), + max_tick_height: self.max_tick_height, + hashes_per_tick: self.hashes_per_tick, + ticks_per_slot: self.ticks_per_slot, + ns_per_slot: self.ns_per_slot, + genesis_creation_time: self.genesis_creation_time, + slots_per_year: self.slots_per_year, + unused: self.unused, + slot: self.slot, + epoch: self.epoch, + block_height: self.block_height, + collector_id: self.collector_id, + collector_fees: self.collector_fees.load(Ordering::Relaxed), + fee_calculator: self.fee_calculator.clone(), + fee_rate_governor: self.fee_rate_governor.clone(), + collected_rent: self.collected_rent.load(Ordering::Relaxed), + rent_collector: self.rent_collector.clone(), + epoch_schedule: self.epoch_schedule, + inflation: *self.inflation.read().unwrap(), + stakes: &self.stakes, + epoch_stakes: &self.epoch_stakes, + is_delta: self.is_delta.load(Ordering::Relaxed), + } + } + pub fn collector_id(&self) -> &Pubkey { &self.collector_id } diff --git a/runtime/src/rent_collector.rs b/runtime/src/rent_collector.rs index 375b61f8c19..0b42d617a49 100644 --- a/runtime/src/rent_collector.rs +++ b/runtime/src/rent_collector.rs @@ -4,7 +4,7 @@ use solana_sdk::{ rent::Rent, sysvar, }; -#[derive(Default, Serialize, Deserialize, Clone, AbiExample)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Debug, AbiExample)] pub struct RentCollector { pub epoch: Epoch, pub epoch_schedule: EpochSchedule, diff --git a/runtime/src/serde_snapshot.rs b/runtime/src/serde_snapshot.rs index a09021c4103..61321075641 100644 --- a/runtime/src/serde_snapshot.rs +++ b/runtime/src/serde_snapshot.rs @@ -4,10 +4,17 @@ use { accounts_db::{ AccountStorageEntry, AccountStorageStatus, AccountsDB, AppendVecId, BankHashInfo, }, + accounts_index::Ancestors, append_vec::AppendVec, - bank::BankRc, + bank::{Bank, BankFields, BankRc}, + blockhash_queue::BlockhashQueue, + epoch_stakes::EpochStakes, + message_processor::MessageProcessor, + rent_collector::RentCollector, + stakes::Stakes, }, - bincode::{deserialize_from, serialize_into, Error}, + bincode, + bincode::{config::Options, serialize_into, Error}, fs_extra::dir::CopyOptions, log::{info, warn}, rand::{thread_rng, Rng}, @@ -15,7 +22,16 @@ use { de::{DeserializeOwned, Visitor}, Deserialize, Deserializer, Serialize, Serializer, }, - solana_sdk::clock::Slot, + solana_sdk::{ + clock::{Epoch, Slot, UnixTimestamp}, + epoch_schedule::EpochSchedule, + fee_calculator::{FeeCalculator, FeeRateGovernor}, + genesis_config::GenesisConfig, + hard_forks::HardForks, + hash::Hash, + inflation::Inflation, + pubkey::Pubkey, + }, std::{ cmp::min, collections::HashMap, @@ -28,6 +44,10 @@ use { }, }; +#[cfg(RUSTC_WITH_SPECIALIZATION)] +use solana_sdk::abi_example::IgnoreAsHelper; + +mod common; mod future; mod legacy; mod tests; @@ -42,28 +62,28 @@ use utils::{serialize_iter_as_map, serialize_iter_as_seq, serialize_iter_as_tupl #[cfg(test)] pub(crate) use self::tests::reconstruct_accounts_db_via_serialization; -pub use crate::accounts_db::{SnapshotStorage, SnapshotStorages}; +pub(crate) use crate::accounts_db::{SnapshotStorage, SnapshotStorages}; #[derive(Copy, Clone, Eq, PartialEq)] -pub enum SerdeStyle { +pub(crate) enum SerdeStyle { NEWER, OLDER, } -const MAX_ACCOUNTS_DB_STREAM_SIZE: u64 = 32 * 1024 * 1024 * 1024; +const MAX_STREAM_SIZE: u64 = 32 * 1024 * 1024 * 1024; -#[derive(Clone, Debug, Default, Deserialize, Serialize)] -pub struct AccountDBFields(HashMap>, u64, Slot, BankHashInfo); +#[derive(Clone, Debug, Default, Deserialize, Serialize, AbiExample)] +struct AccountsDbFields(HashMap>, u64, Slot, BankHashInfo); -pub trait TypeContext<'a> { +trait TypeContext<'a> { type SerializableAccountStorageEntry: Serialize + DeserializeOwned + From<&'a AccountStorageEntry> + Into; - fn serialize_bank_rc_fields( + fn serialize_bank_fields( serializer: S, - serializable_bank: &SerializableBankRc<'a, Self>, + serializable_bank: &SerializableBank<'a, Self>, ) -> std::result::Result where Self: std::marker::Sized; @@ -75,37 +95,63 @@ pub trait TypeContext<'a> { where Self: std::marker::Sized; + fn deserialize_bank_fields( + stream: &mut BufReader, + ) -> Result< + ( + BankFields, + AccountsDbFields, + ), + Error, + > + where + R: Read; + fn deserialize_accounts_db_fields( stream: &mut BufReader, - ) -> Result, Error> + ) -> Result, Error> where R: Read; +} - // we might define fn (de)serialize_bank(...) -> Result for versionized bank serialization in the future +fn deserialize_from(reader: R) -> bincode::Result +where + R: Read, + T: DeserializeOwned, +{ + bincode::options() + .with_limit(MAX_STREAM_SIZE) + .with_fixint_encoding() + .allow_trailing_bytes() + .deserialize_from::(reader) } -pub fn bankrc_from_stream( +pub(crate) fn bank_from_stream( serde_style: SerdeStyle, - account_paths: &[PathBuf], - slot: Slot, stream: &mut BufReader, - stream_append_vecs_path: P, -) -> std::result::Result + append_vecs_path: P, + account_paths: &[PathBuf], + genesis_config: &GenesisConfig, + frozen_account_pubkeys: &[Pubkey], +) -> std::result::Result where R: Read, P: AsRef, { macro_rules! INTO { - ($x:ident) => { - Ok(BankRc::new( - Accounts::new_empty(context_accountsdb_from_fields::<$x, P>( - $x::deserialize_accounts_db_fields(stream)?, - account_paths, - stream_append_vecs_path, - )?), - slot, - )) - }; + ($x:ident) => {{ + let (bank_fields, accounts_db_fields) = $x::deserialize_bank_fields(stream)?; + + let bank = reconstruct_bank_from_fields( + bank_fields, + accounts_db_fields, + genesis_config, + frozen_account_pubkeys, + account_paths, + append_vecs_path, + )?; + Ok(bank) + }}; } match serde_style { SerdeStyle::NEWER => INTO!(TypeContextFuture), @@ -117,10 +163,10 @@ where }) } -pub fn bankrc_to_stream( +pub(crate) fn bank_to_stream( serde_style: SerdeStyle, stream: &mut BufWriter, - bank_rc: &BankRc, + bank: &Bank, snapshot_storages: &[SnapshotStorage], ) -> Result<(), Error> where @@ -128,10 +174,10 @@ where { macro_rules! INTO { ($x:ident) => { - serialize_into( + bincode::serialize_into( stream, - &SerializableBankRc::<$x> { - bank_rc, + &SerializableBank::<$x> { + bank, snapshot_storages, phantom: std::marker::PhantomData::default(), }, @@ -148,22 +194,22 @@ where }) } -pub struct SerializableBankRc<'a, C> { - bank_rc: &'a BankRc, +struct SerializableBank<'a, C> { + bank: &'a Bank, snapshot_storages: &'a [SnapshotStorage], phantom: std::marker::PhantomData, } -impl<'a, C: TypeContext<'a>> Serialize for SerializableBankRc<'a, C> { +impl<'a, C: TypeContext<'a>> Serialize for SerializableBank<'a, C> { fn serialize(&self, serializer: S) -> std::result::Result where S: serde::ser::Serializer, { - C::serialize_bank_rc_fields(serializer, self) + C::serialize_bank_fields(serializer, self) } } -pub struct SerializableAccountsDB<'a, C> { +struct SerializableAccountsDB<'a, C> { accounts_db: &'a AccountsDB, slot: Slot, account_storage_entries: &'a [SnapshotStorage], @@ -179,18 +225,43 @@ impl<'a, C: TypeContext<'a>> Serialize for SerializableAccountsDB<'a, C> { } } -fn context_accountsdb_from_fields<'a, C, P>( - account_db_fields: AccountDBFields, +#[cfg(RUSTC_WITH_SPECIALIZATION)] +impl<'a, C> IgnoreAsHelper for SerializableAccountsDB<'a, C> {} + +fn reconstruct_bank_from_fields( + bank_fields: BankFields, + accounts_db_fields: AccountsDbFields, + genesis_config: &GenesisConfig, + frozen_account_pubkeys: &[Pubkey], + account_paths: &[PathBuf], + append_vecs_path: P, +) -> Result +where + E: Into, + P: AsRef, +{ + let mut accounts_db = + reconstruct_accountsdb_from_fields(accounts_db_fields, account_paths, append_vecs_path)?; + accounts_db.freeze_accounts(&bank_fields.ancestors, frozen_account_pubkeys); + + let bank_rc = BankRc::new(Accounts::new_empty(accounts_db), bank_fields.slot); + let bank = Bank::new_from_fields(bank_rc, genesis_config, bank_fields); + + Ok(bank) +} + +fn reconstruct_accountsdb_from_fields( + accounts_db_fields: AccountsDbFields, account_paths: &[PathBuf], stream_append_vecs_path: P, ) -> Result where - C: TypeContext<'a>, + E: Into, P: AsRef, { let accounts_db = AccountsDB::new(account_paths.to_vec()); - let AccountDBFields(storage, version, slot, bank_hash_info) = account_db_fields; + let AccountsDbFields(storage, version, slot, bank_hash_info) = accounts_db_fields; // convert to two level map of slot -> id -> account storage entry let storage = { diff --git a/runtime/src/serde_snapshot/common.rs b/runtime/src/serde_snapshot/common.rs new file mode 100644 index 00000000000..8e27b3f7355 --- /dev/null +++ b/runtime/src/serde_snapshot/common.rs @@ -0,0 +1,171 @@ +use super::*; +use crate::bank::RefBankFields; +use std::collections::HashSet; +use std::sync::RwLock; + +#[derive(Default, Clone, PartialEq, Debug, Deserialize, Serialize, AbiExample)] +pub(crate) struct UnusedAccounts { + unused1: HashSet, + unused2: HashSet, + unused3: HashMap, +} + +// Deserializable version of Bank for snapshot format +#[derive(Clone, Serialize, Deserialize, AbiExample)] +pub(crate) struct DeserializableBankFields { + pub(crate) blockhash_queue: BlockhashQueue, + pub(crate) ancestors: Ancestors, + pub(crate) hash: Hash, + pub(crate) parent_hash: Hash, + pub(crate) parent_slot: Slot, + pub(crate) hard_forks: HardForks, + pub(crate) transaction_count: u64, + pub(crate) tick_height: u64, + pub(crate) signature_count: u64, + pub(crate) capitalization: u64, + pub(crate) max_tick_height: u64, + pub(crate) hashes_per_tick: Option, + pub(crate) ticks_per_slot: u64, + pub(crate) ns_per_slot: u128, + pub(crate) genesis_creation_time: UnixTimestamp, + pub(crate) slots_per_year: f64, + pub(crate) unused: u64, + pub(crate) slot: Slot, + pub(crate) epoch: Epoch, + pub(crate) block_height: u64, + pub(crate) collector_id: Pubkey, + pub(crate) collector_fees: u64, + pub(crate) fee_calculator: FeeCalculator, + pub(crate) fee_rate_governor: FeeRateGovernor, + pub(crate) collected_rent: u64, + pub(crate) rent_collector: RentCollector, + pub(crate) epoch_schedule: EpochSchedule, + pub(crate) inflation: Inflation, + pub(crate) stakes: Stakes, + pub(crate) unused_accounts: UnusedAccounts, + pub(crate) epoch_stakes: HashMap, + pub(crate) is_delta: bool, + pub(crate) message_processor: MessageProcessor, +} + +impl Into for DeserializableBankFields { + fn into(self) -> BankFields { + BankFields { + blockhash_queue: self.blockhash_queue, + ancestors: self.ancestors, + hash: self.hash, + parent_hash: self.parent_hash, + parent_slot: self.parent_slot, + hard_forks: self.hard_forks, + transaction_count: self.transaction_count, + tick_height: self.tick_height, + signature_count: self.signature_count, + capitalization: self.capitalization, + max_tick_height: self.max_tick_height, + hashes_per_tick: self.hashes_per_tick, + ticks_per_slot: self.ticks_per_slot, + ns_per_slot: self.ns_per_slot, + genesis_creation_time: self.genesis_creation_time, + slots_per_year: self.slots_per_year, + unused: self.unused, + slot: self.slot, + epoch: self.epoch, + block_height: self.block_height, + collector_id: self.collector_id, + collector_fees: self.collector_fees, + fee_calculator: self.fee_calculator, + fee_rate_governor: self.fee_rate_governor, + collected_rent: self.collected_rent, + rent_collector: self.rent_collector, + epoch_schedule: self.epoch_schedule, + inflation: self.inflation, + stakes: self.stakes, + epoch_stakes: self.epoch_stakes, + is_delta: self.is_delta, + } + } +} + +// Serializable version of Bank for snapshot format +#[derive(Serialize)] +pub(crate) struct SerializableBankFields<'a> { + pub(crate) blockhash_queue: &'a RwLock, + pub(crate) ancestors: &'a Ancestors, + pub(crate) hash: Hash, + pub(crate) parent_hash: Hash, + pub(crate) parent_slot: Slot, + pub(crate) hard_forks: &'a RwLock, + pub(crate) transaction_count: u64, + pub(crate) tick_height: u64, + pub(crate) signature_count: u64, + pub(crate) capitalization: u64, + pub(crate) max_tick_height: u64, + pub(crate) hashes_per_tick: Option, + pub(crate) ticks_per_slot: u64, + pub(crate) ns_per_slot: u128, + pub(crate) genesis_creation_time: UnixTimestamp, + pub(crate) slots_per_year: f64, + pub(crate) unused: u64, + pub(crate) slot: Slot, + pub(crate) epoch: Epoch, + pub(crate) block_height: u64, + pub(crate) collector_id: Pubkey, + pub(crate) collector_fees: u64, + pub(crate) fee_calculator: FeeCalculator, + pub(crate) fee_rate_governor: FeeRateGovernor, + pub(crate) collected_rent: u64, + pub(crate) rent_collector: RentCollector, + pub(crate) epoch_schedule: EpochSchedule, + pub(crate) inflation: Inflation, + pub(crate) stakes: &'a RwLock, + pub(crate) unused_accounts: UnusedAccounts, + pub(crate) epoch_stakes: &'a HashMap, + pub(crate) is_delta: bool, + pub(crate) message_processor: MessageProcessor, +} + +impl<'a> From> for SerializableBankFields<'a> { + fn from(rhs: RefBankFields<'a>) -> Self { + fn new() -> T { + T::default() + } + Self { + blockhash_queue: rhs.blockhash_queue, + ancestors: rhs.ancestors, + hash: rhs.hash, + parent_hash: rhs.parent_hash, + parent_slot: rhs.parent_slot, + hard_forks: rhs.hard_forks, + transaction_count: rhs.transaction_count, + tick_height: rhs.tick_height, + signature_count: rhs.signature_count, + capitalization: rhs.capitalization, + max_tick_height: rhs.max_tick_height, + hashes_per_tick: rhs.hashes_per_tick, + ticks_per_slot: rhs.ticks_per_slot, + ns_per_slot: rhs.ns_per_slot, + genesis_creation_time: rhs.genesis_creation_time, + slots_per_year: rhs.slots_per_year, + unused: rhs.unused, + slot: rhs.slot, + epoch: rhs.epoch, + block_height: rhs.block_height, + collector_id: rhs.collector_id, + collector_fees: rhs.collector_fees, + fee_calculator: rhs.fee_calculator, + fee_rate_governor: rhs.fee_rate_governor, + collected_rent: rhs.collected_rent, + rent_collector: rhs.rent_collector, + epoch_schedule: rhs.epoch_schedule, + inflation: rhs.inflation, + stakes: rhs.stakes, + unused_accounts: new(), + epoch_stakes: rhs.epoch_stakes, + is_delta: rhs.is_delta, + message_processor: new(), + } + } +} + +#[cfg(RUSTC_WITH_SPECIALIZATION)] +impl<'a> IgnoreAsHelper for SerializableBankFields<'a> {} diff --git a/runtime/src/serde_snapshot/future.rs b/runtime/src/serde_snapshot/future.rs index ce12527e327..92e4868182e 100644 --- a/runtime/src/serde_snapshot/future.rs +++ b/runtime/src/serde_snapshot/future.rs @@ -1,5 +1,8 @@ +use super::common::{DeserializableBankFields, SerializableBankFields}; use {super::*, solana_measure::measure::Measure, std::cell::RefCell}; +type AccountsDbFields = super::AccountsDbFields; + // Serializable version of AccountStorageEntry for snapshot format #[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] pub(super) struct SerializableAccountStorageEntry { @@ -29,21 +32,23 @@ pub(super) struct Context {} impl<'a> TypeContext<'a> for Context { type SerializableAccountStorageEntry = SerializableAccountStorageEntry; - fn serialize_bank_rc_fields( + fn serialize_bank_fields( serializer: S, - serializable_bank: &SerializableBankRc<'a, Self>, + serializable_bank: &SerializableBank<'a, Self>, ) -> std::result::Result where Self: std::marker::Sized, { - let accounts_db_serialize = SerializableAccountsDB::<'a, Self> { - accounts_db: &*serializable_bank.bank_rc.accounts.accounts_db, - slot: serializable_bank.bank_rc.slot, - account_storage_entries: serializable_bank.snapshot_storages, - phantom: std::marker::PhantomData::default(), - }; - - accounts_db_serialize.serialize(serializer) + ( + SerializableBankFields::from(serializable_bank.bank.get_ref_fields()), + SerializableAccountsDB::<'a, Self> { + accounts_db: &*serializable_bank.bank.rc.accounts.accounts_db, + slot: serializable_bank.bank.rc.slot, + account_storage_entries: serializable_bank.snapshot_storages, + phantom: std::marker::PhantomData::default(), + }, + ) + .serialize(serializer) } fn serialize_accounts_db_fields( @@ -93,12 +98,23 @@ impl<'a> TypeContext<'a> for Context { result } - fn deserialize_accounts_db_fields( + fn deserialize_bank_fields( mut stream: &mut BufReader, - ) -> Result, Error> + ) -> Result<(BankFields, AccountsDbFields), Error> + where + R: Read, + { + let bank_fields = deserialize_from::<_, DeserializableBankFields>(&mut stream)?.into(); + let accounts_db_fields = Self::deserialize_accounts_db_fields(stream)?; + Ok((bank_fields, accounts_db_fields)) + } + + fn deserialize_accounts_db_fields( + stream: &mut BufReader, + ) -> Result where R: Read, { - deserialize_from(&mut stream) + deserialize_from(stream) } } diff --git a/runtime/src/serde_snapshot/legacy.rs b/runtime/src/serde_snapshot/legacy.rs index 62f477355dc..1c34e5a5d64 100644 --- a/runtime/src/serde_snapshot/legacy.rs +++ b/runtime/src/serde_snapshot/legacy.rs @@ -1,7 +1,12 @@ +use super::common::{DeserializableBankFields, SerializableBankFields}; #[cfg(all(test, RUSTC_WITH_SPECIALIZATION))] use solana_sdk::abi_example::IgnoreAsHelper; use {super::*, bincode::config::Options, solana_measure::measure::Measure, std::cell::RefCell}; +const MAX_ACCOUNTS_DB_STREAM_SIZE: u64 = MAX_STREAM_SIZE; + +type AccountsDbFields = super::AccountsDbFields; + // Serializable version of AccountStorageEntry for snapshot format #[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] pub(super) struct SerializableAccountStorageEntry { @@ -52,7 +57,7 @@ impl Into for SerializableAppendVec { } } -// Serialization of AppendVec requires serialization of u64 to +// Serialization of AppendVec requires serialization of u64 to // eight byte vector which is then itself serialized to the stream impl Serialize for SerializableAppendVec { fn serialize(&self, serializer: S) -> Result @@ -67,7 +72,7 @@ impl Serialize for SerializableAppendVec { } } -// Deserialization of AppendVec requires deserialization +// Deserialization of AppendVec requires deserialization // of eight byte vector from which u64 is then deserialized impl<'de> Deserialize<'de> for SerializableAppendVec { fn deserialize(deserializer: D) -> Result @@ -103,23 +108,23 @@ pub(super) struct Context {} impl<'a> TypeContext<'a> for Context { type SerializableAccountStorageEntry = SerializableAccountStorageEntry; - fn serialize_bank_rc_fields( + fn serialize_bank_fields( serializer: S, - serializable_bank: &SerializableBankRc<'a, Self>, + serializable_bank: &SerializableBank<'a, Self>, ) -> std::result::Result where Self: std::marker::Sized, { - // as there is no deserialize_bank_rc_fields(), do not emit the u64 - // size field here and have serialize_accounts_db_fields() emit two - // u64 size fields instead - SerializableAccountsDB::<'a, Self> { - accounts_db: &*serializable_bank.bank_rc.accounts.accounts_db, - slot: serializable_bank.bank_rc.slot, - account_storage_entries: serializable_bank.snapshot_storages, - phantom: std::marker::PhantomData::default(), - } - .serialize(serializer) + ( + SerializableBankFields::from(serializable_bank.bank.get_ref_fields()), + SerializableAccountsDB::<'a, Self> { + accounts_db: &*serializable_bank.bank.rc.accounts.accounts_db, + slot: serializable_bank.bank.rc.slot, + account_storage_entries: serializable_bank.snapshot_storages, + phantom: std::marker::PhantomData::default(), + }, + ) + .serialize(serializer) } fn serialize_accounts_db_fields( @@ -161,6 +166,7 @@ impl<'a> TypeContext<'a> for Context { .clone(), ); + // as there is no deserialize_bank_rc_fields(), emit two u64 size fields here instead let mut serialize_account_storage_timer = Measure::start("serialize_account_storage_ms"); let result = ( &MAX_ACCOUNTS_DB_STREAM_SIZE, @@ -179,9 +185,20 @@ impl<'a> TypeContext<'a> for Context { result } + fn deserialize_bank_fields( + mut stream: &mut BufReader, + ) -> Result<(BankFields, AccountsDbFields), Error> + where + R: Read, + { + let bank_fields = deserialize_from::<_, DeserializableBankFields>(&mut stream)?.into(); + let accounts_db_fields = Self::deserialize_accounts_db_fields(stream)?; + Ok((bank_fields, accounts_db_fields)) + } + fn deserialize_accounts_db_fields( mut stream: &mut BufReader, - ) -> Result, Error> + ) -> Result where R: Read, { @@ -203,6 +220,6 @@ impl<'a> TypeContext<'a> for Context { // (3rd of 3 elements) read in (slot, bank hashes) pair let (slot, bank_hash_info): (Slot, BankHashInfo) = deserialize_from(&mut stream)?; - Ok(AccountDBFields(storage, version, slot, bank_hash_info)) + Ok(AccountsDbFields(storage, version, slot, bank_hash_info)) } } diff --git a/runtime/src/serde_snapshot/tests.rs b/runtime/src/serde_snapshot/tests.rs index 14a9d53a780..9262ace1f9c 100644 --- a/runtime/src/serde_snapshot/tests.rs +++ b/runtime/src/serde_snapshot/tests.rs @@ -64,7 +64,7 @@ where P: AsRef, { // read and deserialise the accounts database directly from the stream - context_accountsdb_from_fields::( + reconstruct_accountsdb_from_fields( C::deserialize_accounts_db_fields(stream)?, account_paths, stream_append_vecs_path, @@ -198,17 +198,15 @@ fn test_bank_serialize_style(serde_style: SerdeStyle) { let snapshot_storages = bank2.get_snapshot_storages(); let mut buf = vec![]; let mut writer = Cursor::new(&mut buf); - serialize_into(&mut writer, &bank2).unwrap(); - crate::serde_snapshot::bankrc_to_stream( + crate::serde_snapshot::bank_to_stream( serde_style, &mut std::io::BufWriter::new(&mut writer), - &bank2.rc, + &bank2, &snapshot_storages, ) .unwrap(); - let mut rdr = Cursor::new(&buf[..]); - let mut dbank: Bank = bincode::deserialize_from(&mut rdr).unwrap(); + let rdr = Cursor::new(&buf[..]); let mut reader = std::io::BufReader::new(&buf[rdr.position() as usize..]); // Create a new set of directories for this bank's accounts @@ -218,17 +216,16 @@ fn test_bank_serialize_style(serde_style: SerdeStyle) { // Create a directory to simulate AppendVecs unpackaged from a snapshot tar let copied_accounts = TempDir::new().unwrap(); copy_append_vecs(&bank2.rc.accounts.accounts_db, copied_accounts.path()).unwrap(); - dbank.set_bank_rc( - crate::serde_snapshot::bankrc_from_stream( - serde_style, - &dbank_paths, - dbank.slot(), - &mut reader, - copied_accounts.path(), - ) - .unwrap(), - ref_sc, - ); + let mut dbank = crate::serde_snapshot::bank_from_stream( + serde_style, + &mut reader, + copied_accounts.path(), + &dbank_paths, + &genesis_config, + &[], + ) + .unwrap(); + dbank.src = ref_sc; assert_eq!(dbank.get_balance(&key1.pubkey()), 0); assert_eq!(dbank.get_balance(&key2.pubkey()), 10); assert_eq!(dbank.get_balance(&key3.pubkey()), 0); @@ -280,45 +277,58 @@ fn test_bank_serialize_older() { } #[cfg(all(test, RUSTC_WITH_SPECIALIZATION))] -mod test_bank_rc_serialize { +mod test_bank_serialize { use super::*; + use solana_sdk::abi_example::AbiExample; // These some what long test harness is required to freeze the ABI of - // BankRc's serialization due to versioned nature - #[frozen_abi(digest = "HfCP74JKqPdeAccNJEj7KEoNxtsmX3zRqc2rpTy1NC7H")] - #[derive(Serialize, AbiExample)] - pub struct BandRcAbiTestWrapperFuture { + // Bank's serialization due to versioned nature + #[frozen_abi(digest = "9gwmJf65VCq3cVfqkSdMQXnMT3Y81MGq4PwWLEutq7v9")] + #[derive(Default, Serialize)] + pub struct BankAbiTestWrapperFuture { #[serde(serialize_with = "wrapper_future")] - bank_rc: BankRc, + bank: Bank, + } + + impl AbiExample for BankAbiTestWrapperFuture { + fn example() -> Self { + Self::default() + } } - pub fn wrapper_future(bank_rc: &BankRc, s: S) -> std::result::Result + pub fn wrapper_future(bank: &Bank, s: S) -> std::result::Result where S: serde::Serializer, { - let snapshot_storages = bank_rc.accounts.accounts_db.get_snapshot_storages(0); - (SerializableBankRc:: { - bank_rc, + let snapshot_storages = bank.rc.accounts.accounts_db.get_snapshot_storages(0); + (SerializableBank:: { + bank, snapshot_storages: &snapshot_storages, phantom: std::marker::PhantomData::default(), }) .serialize(s) } - #[frozen_abi(digest = "43niyekyWwreLALcdEeFFpd7h8U6pgSXGqfKBRw8H7Vy")] - #[derive(Serialize, AbiExample)] - pub struct BandRcAbiTestWrapperLegacy { + #[frozen_abi(digest = "9dKCAhXxpjEK63wdWPcGMhw9yK43nSwi72FpqpHX5A9s")] + #[derive(Default, Serialize)] + pub struct BankAbiTestWrapperLegacy { #[serde(serialize_with = "wrapper_legacy")] - bank_rc: BankRc, + bank: Bank, + } + + impl AbiExample for BankAbiTestWrapperLegacy { + fn example() -> Self { + Self::default() + } } - pub fn wrapper_legacy(bank_rc: &BankRc, s: S) -> std::result::Result + pub fn wrapper_legacy(bank: &Bank, s: S) -> std::result::Result where S: serde::Serializer, { - let snapshot_storages = bank_rc.accounts.accounts_db.get_snapshot_storages(0); - (SerializableBankRc:: { - bank_rc, + let snapshot_storages = bank.rc.accounts.accounts_db.get_snapshot_storages(0); + (SerializableBank:: { + bank, snapshot_storages: &snapshot_storages, phantom: std::marker::PhantomData::default(), }) diff --git a/runtime/src/snapshot_utils.rs b/runtime/src/snapshot_utils.rs index d417fa9c7e3..e552aeca73f 100644 --- a/runtime/src/snapshot_utils.rs +++ b/runtime/src/snapshot_utils.rs @@ -3,7 +3,7 @@ use crate::{ bank_forks::CompressionType, hardened_unpack::{unpack_snapshot, UnpackError}, serde_snapshot::{ - bankrc_from_stream, bankrc_to_stream, SerdeStyle, SnapshotStorage, SnapshotStorages, + bank_from_stream, bank_to_stream, SerdeStyle, SnapshotStorage, SnapshotStorages, }, snapshot_package::AccountsPackage, }; @@ -23,7 +23,6 @@ use std::{ path::{Path, PathBuf}, process::ExitStatus, str::FromStr, - sync::Arc, }; use tar::Archive; use tempfile::TempDir; @@ -462,8 +461,7 @@ pub fn add_snapshot>( SnapshotVersion::V1_1_0 => SerdeStyle::OLDER, SnapshotVersion::V1_2_0 => SerdeStyle::NEWER, }; - serialize_into(stream.by_ref(), bank)?; - bankrc_to_stream(serde_style, stream.by_ref(), &bank.rc, snapshot_storages)?; + bank_to_stream(serde_style, stream.by_ref(), bank, snapshot_storages)?; Ok(()) }; let consumed_size = @@ -723,38 +721,24 @@ where info!("Loading bank from {:?}", &root_paths.snapshot_file_path); let bank = deserialize_snapshot_data_file(&root_paths.snapshot_file_path, |mut stream| { - let mut bank: Bank = bincode::options() - .with_limit(MAX_SNAPSHOT_DATA_FILE_SIZE) - .with_fixint_encoding() - .allow_trailing_bytes() - .deserialize_from(&mut stream)?; - - info!("Rebuilding accounts..."); - - let mut bankrc = match snapshot_version_enum { - SnapshotVersion::V1_1_0 => bankrc_from_stream( + Ok(match snapshot_version_enum { + SnapshotVersion::V1_1_0 => bank_from_stream( SerdeStyle::OLDER, - account_paths, - bank.slot(), &mut stream, &append_vecs_path, + account_paths, + genesis_config, + frozen_account_pubkeys, ), - SnapshotVersion::V1_2_0 => bankrc_from_stream( + SnapshotVersion::V1_2_0 => bank_from_stream( SerdeStyle::NEWER, - account_paths, - bank.slot(), &mut stream, &append_vecs_path, + account_paths, + genesis_config, + frozen_account_pubkeys, ), - }?; - Arc::get_mut(&mut Arc::get_mut(&mut bankrc.accounts).unwrap().accounts_db) - .unwrap() - .freeze_accounts(&bank.ancestors, frozen_account_pubkeys); - - bank.rc = bankrc; - bank.operating_mode = Some(genesis_config.operating_mode); - bank.finish_init(); - Ok(bank) + }?) })?; let status_cache_path = unpacked_snapshots_dir.join(SNAPSHOT_STATUS_CACHE_FILE_NAME); diff --git a/sdk/macro-frozen-abi/src/lib.rs b/sdk/macro-frozen-abi/src/lib.rs index e53171dc1d5..876400c9b3c 100644 --- a/sdk/macro-frozen-abi/src/lib.rs +++ b/sdk/macro-frozen-abi/src/lib.rs @@ -78,8 +78,11 @@ fn quote_for_specialization_detection() -> TokenStream2 { std::sync::atomic::AtomicBool::new(false); } - if !SPECIALIZATION_DETECTOR_INJECTED.load(std::sync::atomic::Ordering::Relaxed) { - SPECIALIZATION_DETECTOR_INJECTED.store(true, std::sync::atomic::Ordering::Relaxed); + if !SPECIALIZATION_DETECTOR_INJECTED.compare_and_swap( + false, + true, + std::sync::atomic::Ordering::AcqRel, + ) { quote! { mod specialization_detector { trait SpecializedTrait { From e53b1ab8ef1f16a84fe15d02f68bb66a2cab9d31 Mon Sep 17 00:00:00 2001 From: Ryo Onodera Date: Wed, 1 Jul 2020 23:55:31 +0900 Subject: [PATCH 2/4] Tidy up namings and duplicate structs to version --- runtime/src/bank.rs | 17 ++- runtime/src/serde_snapshot.rs | 18 +-- runtime/src/serde_snapshot/common.rs | 162 ----------------------- runtime/src/serde_snapshot/future.rs | 177 +++++++++++++++++++++++++- runtime/src/serde_snapshot/legacy.rs | 184 +++++++++++++++++++++++++-- 5 files changed, 367 insertions(+), 191 deletions(-) diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 6037f99c097..9aff1d52c4f 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -188,8 +188,11 @@ impl HashAgeKind { } } +// Bank's common fields shared by all supported snapshot versions for deserialization. +// Sync fields with BankFieldsToSerialize! This is paired with it. +// All members are made public to remain Bank's members private and to make versioned deserializer workable on this #[derive(Clone, Default)] -pub(crate) struct BankFields { +pub(crate) struct BankFieldsToDeserialize { pub(crate) blockhash_queue: BlockhashQueue, pub(crate) ancestors: Ancestors, pub(crate) hash: Hash, @@ -223,7 +226,11 @@ pub(crate) struct BankFields { pub(crate) is_delta: bool, } -pub(crate) struct RefBankFields<'a> { +// Bank's common fields shared by all supported snapshot versions for serialization. +// This is separated from BankFieldsToDeserialize to avoid cloning by using refs. +// So, sync fields with BankFieldsToDeserialize! +// all members are made public to remain Bank private and to make versioned serializer workable on this +pub(crate) struct BankFieldsToSerialize<'a> { pub(crate) blockhash_queue: &'a RwLock, pub(crate) ancestors: &'a Ancestors, pub(crate) hash: Hash, @@ -553,7 +560,7 @@ impl Bank { pub(crate) fn new_from_fields( bank_rc: BankRc, genesis_config: &GenesisConfig, - fields: BankFields, + fields: BankFieldsToDeserialize, ) -> Self { fn new() -> T { T::default() @@ -642,8 +649,8 @@ impl Bank { } /// Return subset of bank fields representing serializable state - pub(crate) fn get_ref_fields(&self) -> RefBankFields { - RefBankFields { + pub(crate) fn get_fields_to_serialize(&self) -> BankFieldsToSerialize { + BankFieldsToSerialize { blockhash_queue: &self.blockhash_queue, ancestors: &self.ancestors, hash: *self.hash.read().unwrap(), diff --git a/runtime/src/serde_snapshot.rs b/runtime/src/serde_snapshot.rs index 61321075641..fac72d12d12 100644 --- a/runtime/src/serde_snapshot.rs +++ b/runtime/src/serde_snapshot.rs @@ -6,7 +6,7 @@ use { }, accounts_index::Ancestors, append_vec::AppendVec, - bank::{Bank, BankFields, BankRc}, + bank::{Bank, BankFieldsToDeserialize, BankRc}, blockhash_queue::BlockhashQueue, epoch_stakes::EpochStakes, message_processor::MessageProcessor, @@ -81,9 +81,9 @@ trait TypeContext<'a> { + From<&'a AccountStorageEntry> + Into; - fn serialize_bank_fields( + fn serialize_bank_and_storage( serializer: S, - serializable_bank: &SerializableBank<'a, Self>, + serializable_bank: &SerializableBankAndStorage<'a, Self>, ) -> std::result::Result where Self: std::marker::Sized; @@ -99,7 +99,7 @@ trait TypeContext<'a> { stream: &mut BufReader, ) -> Result< ( - BankFields, + BankFieldsToDeserialize, AccountsDbFields, ), Error, @@ -176,7 +176,7 @@ where ($x:ident) => { bincode::serialize_into( stream, - &SerializableBank::<$x> { + &SerializableBankAndStorage::<$x> { bank, snapshot_storages, phantom: std::marker::PhantomData::default(), @@ -194,18 +194,18 @@ where }) } -struct SerializableBank<'a, C> { +struct SerializableBankAndStorage<'a, C> { bank: &'a Bank, snapshot_storages: &'a [SnapshotStorage], phantom: std::marker::PhantomData, } -impl<'a, C: TypeContext<'a>> Serialize for SerializableBank<'a, C> { +impl<'a, C: TypeContext<'a>> Serialize for SerializableBankAndStorage<'a, C> { fn serialize(&self, serializer: S) -> std::result::Result where S: serde::ser::Serializer, { - C::serialize_bank_fields(serializer, self) + C::serialize_bank_and_storage(serializer, self) } } @@ -229,7 +229,7 @@ impl<'a, C: TypeContext<'a>> Serialize for SerializableAccountsDB<'a, C> { impl<'a, C> IgnoreAsHelper for SerializableAccountsDB<'a, C> {} fn reconstruct_bank_from_fields( - bank_fields: BankFields, + bank_fields: BankFieldsToDeserialize, accounts_db_fields: AccountsDbFields, genesis_config: &GenesisConfig, frozen_account_pubkeys: &[Pubkey], diff --git a/runtime/src/serde_snapshot/common.rs b/runtime/src/serde_snapshot/common.rs index 8e27b3f7355..23cdf45eecf 100644 --- a/runtime/src/serde_snapshot/common.rs +++ b/runtime/src/serde_snapshot/common.rs @@ -1,7 +1,5 @@ use super::*; -use crate::bank::RefBankFields; use std::collections::HashSet; -use std::sync::RwLock; #[derive(Default, Clone, PartialEq, Debug, Deserialize, Serialize, AbiExample)] pub(crate) struct UnusedAccounts { @@ -9,163 +7,3 @@ pub(crate) struct UnusedAccounts { unused2: HashSet, unused3: HashMap, } - -// Deserializable version of Bank for snapshot format -#[derive(Clone, Serialize, Deserialize, AbiExample)] -pub(crate) struct DeserializableBankFields { - pub(crate) blockhash_queue: BlockhashQueue, - pub(crate) ancestors: Ancestors, - pub(crate) hash: Hash, - pub(crate) parent_hash: Hash, - pub(crate) parent_slot: Slot, - pub(crate) hard_forks: HardForks, - pub(crate) transaction_count: u64, - pub(crate) tick_height: u64, - pub(crate) signature_count: u64, - pub(crate) capitalization: u64, - pub(crate) max_tick_height: u64, - pub(crate) hashes_per_tick: Option, - pub(crate) ticks_per_slot: u64, - pub(crate) ns_per_slot: u128, - pub(crate) genesis_creation_time: UnixTimestamp, - pub(crate) slots_per_year: f64, - pub(crate) unused: u64, - pub(crate) slot: Slot, - pub(crate) epoch: Epoch, - pub(crate) block_height: u64, - pub(crate) collector_id: Pubkey, - pub(crate) collector_fees: u64, - pub(crate) fee_calculator: FeeCalculator, - pub(crate) fee_rate_governor: FeeRateGovernor, - pub(crate) collected_rent: u64, - pub(crate) rent_collector: RentCollector, - pub(crate) epoch_schedule: EpochSchedule, - pub(crate) inflation: Inflation, - pub(crate) stakes: Stakes, - pub(crate) unused_accounts: UnusedAccounts, - pub(crate) epoch_stakes: HashMap, - pub(crate) is_delta: bool, - pub(crate) message_processor: MessageProcessor, -} - -impl Into for DeserializableBankFields { - fn into(self) -> BankFields { - BankFields { - blockhash_queue: self.blockhash_queue, - ancestors: self.ancestors, - hash: self.hash, - parent_hash: self.parent_hash, - parent_slot: self.parent_slot, - hard_forks: self.hard_forks, - transaction_count: self.transaction_count, - tick_height: self.tick_height, - signature_count: self.signature_count, - capitalization: self.capitalization, - max_tick_height: self.max_tick_height, - hashes_per_tick: self.hashes_per_tick, - ticks_per_slot: self.ticks_per_slot, - ns_per_slot: self.ns_per_slot, - genesis_creation_time: self.genesis_creation_time, - slots_per_year: self.slots_per_year, - unused: self.unused, - slot: self.slot, - epoch: self.epoch, - block_height: self.block_height, - collector_id: self.collector_id, - collector_fees: self.collector_fees, - fee_calculator: self.fee_calculator, - fee_rate_governor: self.fee_rate_governor, - collected_rent: self.collected_rent, - rent_collector: self.rent_collector, - epoch_schedule: self.epoch_schedule, - inflation: self.inflation, - stakes: self.stakes, - epoch_stakes: self.epoch_stakes, - is_delta: self.is_delta, - } - } -} - -// Serializable version of Bank for snapshot format -#[derive(Serialize)] -pub(crate) struct SerializableBankFields<'a> { - pub(crate) blockhash_queue: &'a RwLock, - pub(crate) ancestors: &'a Ancestors, - pub(crate) hash: Hash, - pub(crate) parent_hash: Hash, - pub(crate) parent_slot: Slot, - pub(crate) hard_forks: &'a RwLock, - pub(crate) transaction_count: u64, - pub(crate) tick_height: u64, - pub(crate) signature_count: u64, - pub(crate) capitalization: u64, - pub(crate) max_tick_height: u64, - pub(crate) hashes_per_tick: Option, - pub(crate) ticks_per_slot: u64, - pub(crate) ns_per_slot: u128, - pub(crate) genesis_creation_time: UnixTimestamp, - pub(crate) slots_per_year: f64, - pub(crate) unused: u64, - pub(crate) slot: Slot, - pub(crate) epoch: Epoch, - pub(crate) block_height: u64, - pub(crate) collector_id: Pubkey, - pub(crate) collector_fees: u64, - pub(crate) fee_calculator: FeeCalculator, - pub(crate) fee_rate_governor: FeeRateGovernor, - pub(crate) collected_rent: u64, - pub(crate) rent_collector: RentCollector, - pub(crate) epoch_schedule: EpochSchedule, - pub(crate) inflation: Inflation, - pub(crate) stakes: &'a RwLock, - pub(crate) unused_accounts: UnusedAccounts, - pub(crate) epoch_stakes: &'a HashMap, - pub(crate) is_delta: bool, - pub(crate) message_processor: MessageProcessor, -} - -impl<'a> From> for SerializableBankFields<'a> { - fn from(rhs: RefBankFields<'a>) -> Self { - fn new() -> T { - T::default() - } - Self { - blockhash_queue: rhs.blockhash_queue, - ancestors: rhs.ancestors, - hash: rhs.hash, - parent_hash: rhs.parent_hash, - parent_slot: rhs.parent_slot, - hard_forks: rhs.hard_forks, - transaction_count: rhs.transaction_count, - tick_height: rhs.tick_height, - signature_count: rhs.signature_count, - capitalization: rhs.capitalization, - max_tick_height: rhs.max_tick_height, - hashes_per_tick: rhs.hashes_per_tick, - ticks_per_slot: rhs.ticks_per_slot, - ns_per_slot: rhs.ns_per_slot, - genesis_creation_time: rhs.genesis_creation_time, - slots_per_year: rhs.slots_per_year, - unused: rhs.unused, - slot: rhs.slot, - epoch: rhs.epoch, - block_height: rhs.block_height, - collector_id: rhs.collector_id, - collector_fees: rhs.collector_fees, - fee_calculator: rhs.fee_calculator, - fee_rate_governor: rhs.fee_rate_governor, - collected_rent: rhs.collected_rent, - rent_collector: rhs.rent_collector, - epoch_schedule: rhs.epoch_schedule, - inflation: rhs.inflation, - stakes: rhs.stakes, - unused_accounts: new(), - epoch_stakes: rhs.epoch_stakes, - is_delta: rhs.is_delta, - message_processor: new(), - } - } -} - -#[cfg(RUSTC_WITH_SPECIALIZATION)] -impl<'a> IgnoreAsHelper for SerializableBankFields<'a> {} diff --git a/runtime/src/serde_snapshot/future.rs b/runtime/src/serde_snapshot/future.rs index 92e4868182e..da1c6fdd36a 100644 --- a/runtime/src/serde_snapshot/future.rs +++ b/runtime/src/serde_snapshot/future.rs @@ -1,4 +1,6 @@ -use super::common::{DeserializableBankFields, SerializableBankFields}; +use super::common::UnusedAccounts; +#[cfg(all(test, RUSTC_WITH_SPECIALIZATION))] +use solana_sdk::abi_example::IgnoreAsHelper; use {super::*, solana_measure::measure::Measure, std::cell::RefCell}; type AccountsDbFields = super::AccountsDbFields; @@ -28,19 +30,182 @@ impl Into for SerializableAccountStorageEntry { } } +use std::sync::RwLock; +// Deserializable version of Bank which need not be serializable, +// because it's handled by SerializableVersionedBank. +// So, sync fields with it! +#[derive(Clone, Deserialize)] +pub(crate) struct DeserializableVersionedBank { + pub(crate) blockhash_queue: BlockhashQueue, + pub(crate) ancestors: Ancestors, + pub(crate) hash: Hash, + pub(crate) parent_hash: Hash, + pub(crate) parent_slot: Slot, + pub(crate) hard_forks: HardForks, + pub(crate) transaction_count: u64, + pub(crate) tick_height: u64, + pub(crate) signature_count: u64, + pub(crate) capitalization: u64, + pub(crate) max_tick_height: u64, + pub(crate) hashes_per_tick: Option, + pub(crate) ticks_per_slot: u64, + pub(crate) ns_per_slot: u128, + pub(crate) genesis_creation_time: UnixTimestamp, + pub(crate) slots_per_year: f64, + pub(crate) unused: u64, + pub(crate) slot: Slot, + pub(crate) epoch: Epoch, + pub(crate) block_height: u64, + pub(crate) collector_id: Pubkey, + pub(crate) collector_fees: u64, + pub(crate) fee_calculator: FeeCalculator, + pub(crate) fee_rate_governor: FeeRateGovernor, + pub(crate) collected_rent: u64, + pub(crate) rent_collector: RentCollector, + pub(crate) epoch_schedule: EpochSchedule, + pub(crate) inflation: Inflation, + pub(crate) stakes: Stakes, + pub(crate) unused_accounts: UnusedAccounts, + pub(crate) epoch_stakes: HashMap, + pub(crate) is_delta: bool, + pub(crate) message_processor: MessageProcessor, +} + +impl Into for DeserializableVersionedBank { + fn into(self) -> BankFieldsToDeserialize { + BankFieldsToDeserialize { + blockhash_queue: self.blockhash_queue, + ancestors: self.ancestors, + hash: self.hash, + parent_hash: self.parent_hash, + parent_slot: self.parent_slot, + hard_forks: self.hard_forks, + transaction_count: self.transaction_count, + tick_height: self.tick_height, + signature_count: self.signature_count, + capitalization: self.capitalization, + max_tick_height: self.max_tick_height, + hashes_per_tick: self.hashes_per_tick, + ticks_per_slot: self.ticks_per_slot, + ns_per_slot: self.ns_per_slot, + genesis_creation_time: self.genesis_creation_time, + slots_per_year: self.slots_per_year, + unused: self.unused, + slot: self.slot, + epoch: self.epoch, + block_height: self.block_height, + collector_id: self.collector_id, + collector_fees: self.collector_fees, + fee_calculator: self.fee_calculator, + fee_rate_governor: self.fee_rate_governor, + collected_rent: self.collected_rent, + rent_collector: self.rent_collector, + epoch_schedule: self.epoch_schedule, + inflation: self.inflation, + stakes: self.stakes, + epoch_stakes: self.epoch_stakes, + is_delta: self.is_delta, + } + } +} +// Serializable version of Bank, not Deserializable to avoid cloning by using refs. +// Sync fields with DeserializableVersionedBank! +#[derive(Serialize)] +pub(crate) struct SerializableVersionedBank<'a> { + pub(crate) blockhash_queue: &'a RwLock, + pub(crate) ancestors: &'a Ancestors, + pub(crate) hash: Hash, + pub(crate) parent_hash: Hash, + pub(crate) parent_slot: Slot, + pub(crate) hard_forks: &'a RwLock, + pub(crate) transaction_count: u64, + pub(crate) tick_height: u64, + pub(crate) signature_count: u64, + pub(crate) capitalization: u64, + pub(crate) max_tick_height: u64, + pub(crate) hashes_per_tick: Option, + pub(crate) ticks_per_slot: u64, + pub(crate) ns_per_slot: u128, + pub(crate) genesis_creation_time: UnixTimestamp, + pub(crate) slots_per_year: f64, + pub(crate) unused: u64, + pub(crate) slot: Slot, + pub(crate) epoch: Epoch, + pub(crate) block_height: u64, + pub(crate) collector_id: Pubkey, + pub(crate) collector_fees: u64, + pub(crate) fee_calculator: FeeCalculator, + pub(crate) fee_rate_governor: FeeRateGovernor, + pub(crate) collected_rent: u64, + pub(crate) rent_collector: RentCollector, + pub(crate) epoch_schedule: EpochSchedule, + pub(crate) inflation: Inflation, + pub(crate) stakes: &'a RwLock, + pub(crate) unused_accounts: UnusedAccounts, + pub(crate) epoch_stakes: &'a HashMap, + pub(crate) is_delta: bool, + pub(crate) message_processor: MessageProcessor, +} + +impl<'a> From> for SerializableVersionedBank<'a> { + fn from(rhs: crate::bank::BankFieldsToSerialize<'a>) -> Self { + fn new() -> T { + T::default() + } + Self { + blockhash_queue: rhs.blockhash_queue, + ancestors: rhs.ancestors, + hash: rhs.hash, + parent_hash: rhs.parent_hash, + parent_slot: rhs.parent_slot, + hard_forks: rhs.hard_forks, + transaction_count: rhs.transaction_count, + tick_height: rhs.tick_height, + signature_count: rhs.signature_count, + capitalization: rhs.capitalization, + max_tick_height: rhs.max_tick_height, + hashes_per_tick: rhs.hashes_per_tick, + ticks_per_slot: rhs.ticks_per_slot, + ns_per_slot: rhs.ns_per_slot, + genesis_creation_time: rhs.genesis_creation_time, + slots_per_year: rhs.slots_per_year, + unused: rhs.unused, + slot: rhs.slot, + epoch: rhs.epoch, + block_height: rhs.block_height, + collector_id: rhs.collector_id, + collector_fees: rhs.collector_fees, + fee_calculator: rhs.fee_calculator, + fee_rate_governor: rhs.fee_rate_governor, + collected_rent: rhs.collected_rent, + rent_collector: rhs.rent_collector, + epoch_schedule: rhs.epoch_schedule, + inflation: rhs.inflation, + stakes: rhs.stakes, + unused_accounts: new(), + epoch_stakes: rhs.epoch_stakes, + is_delta: rhs.is_delta, + message_processor: new(), + } + } +} + +#[cfg(RUSTC_WITH_SPECIALIZATION)] +impl<'a> IgnoreAsHelper for SerializableVersionedBank<'a> {} + pub(super) struct Context {} impl<'a> TypeContext<'a> for Context { type SerializableAccountStorageEntry = SerializableAccountStorageEntry; - fn serialize_bank_fields( + fn serialize_bank_and_storage( serializer: S, - serializable_bank: &SerializableBank<'a, Self>, + serializable_bank: &SerializableBankAndStorage<'a, Self>, ) -> std::result::Result where Self: std::marker::Sized, { ( - SerializableBankFields::from(serializable_bank.bank.get_ref_fields()), + SerializableVersionedBank::from(serializable_bank.bank.get_fields_to_serialize()), SerializableAccountsDB::<'a, Self> { accounts_db: &*serializable_bank.bank.rc.accounts.accounts_db, slot: serializable_bank.bank.rc.slot, @@ -100,11 +265,11 @@ impl<'a> TypeContext<'a> for Context { fn deserialize_bank_fields( mut stream: &mut BufReader, - ) -> Result<(BankFields, AccountsDbFields), Error> + ) -> Result<(BankFieldsToDeserialize, AccountsDbFields), Error> where R: Read, { - let bank_fields = deserialize_from::<_, DeserializableBankFields>(&mut stream)?.into(); + let bank_fields = deserialize_from::<_, DeserializableVersionedBank>(&mut stream)?.into(); let accounts_db_fields = Self::deserialize_accounts_db_fields(stream)?; Ok((bank_fields, accounts_db_fields)) } diff --git a/runtime/src/serde_snapshot/legacy.rs b/runtime/src/serde_snapshot/legacy.rs index 1c34e5a5d64..0d37bb915ca 100644 --- a/runtime/src/serde_snapshot/legacy.rs +++ b/runtime/src/serde_snapshot/legacy.rs @@ -1,4 +1,4 @@ -use super::common::{DeserializableBankFields, SerializableBankFields}; +use super::common::UnusedAccounts; #[cfg(all(test, RUSTC_WITH_SPECIALIZATION))] use solana_sdk::abi_example::IgnoreAsHelper; use {super::*, bincode::config::Options, solana_measure::measure::Measure, std::cell::RefCell}; @@ -104,23 +104,189 @@ impl<'de> Deserialize<'de> for SerializableAppendVec { } } +// Deserializable version of Bank which need not be serializable, +// because it's handled by SerializableVersionedBank. +// So, sync fields with it! +#[derive(Clone, Deserialize)] +pub(crate) struct DeserializableVersionedBank { + pub(crate) blockhash_queue: BlockhashQueue, + pub(crate) ancestors: Ancestors, + pub(crate) hash: Hash, + pub(crate) parent_hash: Hash, + pub(crate) parent_slot: Slot, + pub(crate) hard_forks: HardForks, + pub(crate) transaction_count: u64, + pub(crate) tick_height: u64, + pub(crate) signature_count: u64, + pub(crate) capitalization: u64, + pub(crate) max_tick_height: u64, + pub(crate) hashes_per_tick: Option, + pub(crate) ticks_per_slot: u64, + pub(crate) ns_per_slot: u128, + pub(crate) genesis_creation_time: UnixTimestamp, + pub(crate) slots_per_year: f64, + pub(crate) unused: u64, + pub(crate) slot: Slot, + pub(crate) epoch: Epoch, + pub(crate) block_height: u64, + pub(crate) collector_id: Pubkey, + pub(crate) collector_fees: u64, + pub(crate) fee_calculator: FeeCalculator, + pub(crate) fee_rate_governor: FeeRateGovernor, + pub(crate) collected_rent: u64, + pub(crate) rent_collector: RentCollector, + pub(crate) epoch_schedule: EpochSchedule, + pub(crate) inflation: Inflation, + pub(crate) stakes: Stakes, + pub(crate) unused_accounts: UnusedAccounts, + pub(crate) epoch_stakes: HashMap, + pub(crate) is_delta: bool, + pub(crate) message_processor: MessageProcessor, +} + +impl Into for DeserializableVersionedBank { + fn into(self) -> BankFieldsToDeserialize { + BankFieldsToDeserialize { + blockhash_queue: self.blockhash_queue, + ancestors: self.ancestors, + hash: self.hash, + parent_hash: self.parent_hash, + parent_slot: self.parent_slot, + hard_forks: self.hard_forks, + transaction_count: self.transaction_count, + tick_height: self.tick_height, + signature_count: self.signature_count, + capitalization: self.capitalization, + max_tick_height: self.max_tick_height, + hashes_per_tick: self.hashes_per_tick, + ticks_per_slot: self.ticks_per_slot, + ns_per_slot: self.ns_per_slot, + genesis_creation_time: self.genesis_creation_time, + slots_per_year: self.slots_per_year, + unused: self.unused, + slot: self.slot, + epoch: self.epoch, + block_height: self.block_height, + collector_id: self.collector_id, + collector_fees: self.collector_fees, + fee_calculator: self.fee_calculator, + fee_rate_governor: self.fee_rate_governor, + collected_rent: self.collected_rent, + rent_collector: self.rent_collector, + epoch_schedule: self.epoch_schedule, + inflation: self.inflation, + stakes: self.stakes, + epoch_stakes: self.epoch_stakes, + is_delta: self.is_delta, + } + } +} +// Serializable version of Bank, not Deserializable to avoid cloning by using refs. +// Sync fields with DeserializableVersionedBank! +#[derive(Serialize)] +pub(crate) struct SerializableVersionedBank<'a> { + pub(crate) blockhash_queue: &'a RwLock, + pub(crate) ancestors: &'a Ancestors, + pub(crate) hash: Hash, + pub(crate) parent_hash: Hash, + pub(crate) parent_slot: Slot, + pub(crate) hard_forks: &'a RwLock, + pub(crate) transaction_count: u64, + pub(crate) tick_height: u64, + pub(crate) signature_count: u64, + pub(crate) capitalization: u64, + pub(crate) max_tick_height: u64, + pub(crate) hashes_per_tick: Option, + pub(crate) ticks_per_slot: u64, + pub(crate) ns_per_slot: u128, + pub(crate) genesis_creation_time: UnixTimestamp, + pub(crate) slots_per_year: f64, + pub(crate) unused: u64, + pub(crate) slot: Slot, + pub(crate) epoch: Epoch, + pub(crate) block_height: u64, + pub(crate) collector_id: Pubkey, + pub(crate) collector_fees: u64, + pub(crate) fee_calculator: FeeCalculator, + pub(crate) fee_rate_governor: FeeRateGovernor, + pub(crate) collected_rent: u64, + pub(crate) rent_collector: RentCollector, + pub(crate) epoch_schedule: EpochSchedule, + pub(crate) inflation: Inflation, + pub(crate) stakes: &'a RwLock, + pub(crate) unused_accounts: UnusedAccounts, + pub(crate) epoch_stakes: &'a HashMap, + pub(crate) is_delta: bool, + pub(crate) message_processor: MessageProcessor, +} + +use std::sync::RwLock; + +impl<'a> From> for SerializableVersionedBank<'a> { + fn from(rhs: crate::bank::BankFieldsToSerialize<'a>) -> Self { + fn new() -> T { + T::default() + } + Self { + blockhash_queue: rhs.blockhash_queue, + ancestors: rhs.ancestors, + hash: rhs.hash, + parent_hash: rhs.parent_hash, + parent_slot: rhs.parent_slot, + hard_forks: rhs.hard_forks, + transaction_count: rhs.transaction_count, + tick_height: rhs.tick_height, + signature_count: rhs.signature_count, + capitalization: rhs.capitalization, + max_tick_height: rhs.max_tick_height, + hashes_per_tick: rhs.hashes_per_tick, + ticks_per_slot: rhs.ticks_per_slot, + ns_per_slot: rhs.ns_per_slot, + genesis_creation_time: rhs.genesis_creation_time, + slots_per_year: rhs.slots_per_year, + unused: rhs.unused, + slot: rhs.slot, + epoch: rhs.epoch, + block_height: rhs.block_height, + collector_id: rhs.collector_id, + collector_fees: rhs.collector_fees, + fee_calculator: rhs.fee_calculator, + fee_rate_governor: rhs.fee_rate_governor, + collected_rent: rhs.collected_rent, + rent_collector: rhs.rent_collector, + epoch_schedule: rhs.epoch_schedule, + inflation: rhs.inflation, + stakes: rhs.stakes, + unused_accounts: new(), + epoch_stakes: rhs.epoch_stakes, + is_delta: rhs.is_delta, + message_processor: new(), + } + } +} + +#[cfg(RUSTC_WITH_SPECIALIZATION)] +impl<'a> IgnoreAsHelper for SerializableVersionedBank<'a> {} + pub(super) struct Context {} impl<'a> TypeContext<'a> for Context { type SerializableAccountStorageEntry = SerializableAccountStorageEntry; - fn serialize_bank_fields( + fn serialize_bank_and_storage( serializer: S, - serializable_bank: &SerializableBank<'a, Self>, + serializable_bank_and_storage: &SerializableBankAndStorage<'a, Self>, ) -> std::result::Result where Self: std::marker::Sized, { ( - SerializableBankFields::from(serializable_bank.bank.get_ref_fields()), + SerializableVersionedBank::from( + serializable_bank_and_storage.bank.get_fields_to_serialize(), + ), SerializableAccountsDB::<'a, Self> { - accounts_db: &*serializable_bank.bank.rc.accounts.accounts_db, - slot: serializable_bank.bank.rc.slot, - account_storage_entries: serializable_bank.snapshot_storages, + accounts_db: &*serializable_bank_and_storage.bank.rc.accounts.accounts_db, + slot: serializable_bank_and_storage.bank.rc.slot, + account_storage_entries: serializable_bank_and_storage.snapshot_storages, phantom: std::marker::PhantomData::default(), }, ) @@ -187,11 +353,11 @@ impl<'a> TypeContext<'a> for Context { fn deserialize_bank_fields( mut stream: &mut BufReader, - ) -> Result<(BankFields, AccountsDbFields), Error> + ) -> Result<(BankFieldsToDeserialize, AccountsDbFields), Error> where R: Read, { - let bank_fields = deserialize_from::<_, DeserializableBankFields>(&mut stream)?.into(); + let bank_fields = deserialize_from::<_, DeserializableVersionedBank>(&mut stream)?.into(); let accounts_db_fields = Self::deserialize_accounts_db_fields(stream)?; Ok((bank_fields, accounts_db_fields)) } From 57480ec57476604abbb080f4adb05d0d8eca3c9e Mon Sep 17 00:00:00 2001 From: Kristofer Peterson Date: Thu, 9 Jul 2020 14:17:02 +0100 Subject: [PATCH 3/4] Apply struct renames to tests --- runtime/src/serde_snapshot/legacy.rs | 2 +- runtime/src/serde_snapshot/tests.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/runtime/src/serde_snapshot/legacy.rs b/runtime/src/serde_snapshot/legacy.rs index 0d37bb915ca..226ec44119a 100644 --- a/runtime/src/serde_snapshot/legacy.rs +++ b/runtime/src/serde_snapshot/legacy.rs @@ -260,7 +260,7 @@ impl<'a> From> for SerializableVersionedB unused_accounts: new(), epoch_stakes: rhs.epoch_stakes, is_delta: rhs.is_delta, - message_processor: new(), + message_processor: new(), } } } diff --git a/runtime/src/serde_snapshot/tests.rs b/runtime/src/serde_snapshot/tests.rs index 9262ace1f9c..25b54bd7162 100644 --- a/runtime/src/serde_snapshot/tests.rs +++ b/runtime/src/serde_snapshot/tests.rs @@ -301,7 +301,7 @@ mod test_bank_serialize { S: serde::Serializer, { let snapshot_storages = bank.rc.accounts.accounts_db.get_snapshot_storages(0); - (SerializableBank:: { + (SerializableBankAndStorage:: { bank, snapshot_storages: &snapshot_storages, phantom: std::marker::PhantomData::default(), @@ -327,7 +327,7 @@ mod test_bank_serialize { S: serde::Serializer, { let snapshot_storages = bank.rc.accounts.accounts_db.get_snapshot_storages(0); - (SerializableBank:: { + (SerializableBankAndStorage:: { bank, snapshot_storages: &snapshot_storages, phantom: std::marker::PhantomData::default(), From 243023f219cf090339fc73175624698b9e99ab35 Mon Sep 17 00:00:00 2001 From: Kristofer Peterson Date: Thu, 9 Jul 2020 15:10:15 +0100 Subject: [PATCH 4/4] Update abi hashes --- runtime/src/serde_snapshot/tests.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/src/serde_snapshot/tests.rs b/runtime/src/serde_snapshot/tests.rs index 25b54bd7162..32d677148b0 100644 --- a/runtime/src/serde_snapshot/tests.rs +++ b/runtime/src/serde_snapshot/tests.rs @@ -283,7 +283,7 @@ mod test_bank_serialize { // These some what long test harness is required to freeze the ABI of // Bank's serialization due to versioned nature - #[frozen_abi(digest = "9gwmJf65VCq3cVfqkSdMQXnMT3Y81MGq4PwWLEutq7v9")] + #[frozen_abi(digest = "9BGkhttaVsELn1zoHMKXLvi3Qty51nY1yz584Fao2Ev9")] #[derive(Default, Serialize)] pub struct BankAbiTestWrapperFuture { #[serde(serialize_with = "wrapper_future")] @@ -309,7 +309,7 @@ mod test_bank_serialize { .serialize(s) } - #[frozen_abi(digest = "9dKCAhXxpjEK63wdWPcGMhw9yK43nSwi72FpqpHX5A9s")] + #[frozen_abi(digest = "HYmXta1fhiHe6hZLGJriMqKx9N2U8YRJY51AGZmxzM5m")] #[derive(Default, Serialize)] pub struct BankAbiTestWrapperLegacy { #[serde(serialize_with = "wrapper_legacy")]