From 59f5d67960a0d30add68d84fb3d94bfeb8d323a6 Mon Sep 17 00:00:00 2001 From: Ashwin Sekar Date: Mon, 6 Jun 2022 13:07:02 -0700 Subject: [PATCH] NOTE: this is a modified backport that only loads the extra field. This allows us to deserialize v1.11 snapshots Original: Serialize lamports per signature (#25364) * Serialize lamports per signature * Add full snapshot archive test, enable features in previous tests (cherry picked from commit 8caced68ce8dff7b18566ed90b94bc077bd6f370) --- runtime/src/bank.rs | 4 +- runtime/src/genesis_utils.rs | 22 +++++---- runtime/src/serde_snapshot.rs | 72 +++++++++++++++++++++++++++++ runtime/src/serde_snapshot/newer.rs | 35 +++++++++++++- runtime/src/serde_snapshot/tests.rs | 60 ++++++++++++++++++++++++ sdk/program/src/fee_calculator.rs | 7 +++ 6 files changed, 188 insertions(+), 12 deletions(-) diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index f2a804546ff..5a3f4a93727 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -1189,10 +1189,10 @@ pub struct Bank { /// Deprecated, do not use /// Latest transaction fees for transactions processed by this bank - fee_calculator: FeeCalculator, + pub(crate) fee_calculator: FeeCalculator, /// Track cluster signature throughput and adjust fee rate - fee_rate_governor: FeeRateGovernor, + pub(crate) fee_rate_governor: FeeRateGovernor, /// Rent that has been collected collected_rent: AtomicU64, diff --git a/runtime/src/genesis_utils.rs b/runtime/src/genesis_utils.rs index b9dd64d7881..1e789e05ab6 100644 --- a/runtime/src/genesis_utils.rs +++ b/runtime/src/genesis_utils.rs @@ -188,18 +188,22 @@ pub fn create_genesis_config_with_leader( pub fn activate_all_features(genesis_config: &mut GenesisConfig) { // Activate all features at genesis in development mode for feature_id in FeatureSet::default().inactive { - genesis_config.accounts.insert( - feature_id, - Account::from(feature::create_account( - &Feature { - activated_at: Some(0), - }, - std::cmp::max(genesis_config.rent.minimum_balance(Feature::size_of()), 1), - )), - ); + activate_feature(genesis_config, feature_id); } } +pub fn activate_feature(genesis_config: &mut GenesisConfig, feature_id: Pubkey) { + genesis_config.accounts.insert( + feature_id, + Account::from(feature::create_account( + &Feature { + activated_at: Some(0), + }, + std::cmp::max(genesis_config.rent.minimum_balance(Feature::size_of()), 1), + )), + ); +} + #[allow(clippy::too_many_arguments)] pub fn create_genesis_config_with_leader_ex( mint_lamports: u64, diff --git a/runtime/src/serde_snapshot.rs b/runtime/src/serde_snapshot.rs index 343e66ad6e3..e2bf03809fe 100644 --- a/runtime/src/serde_snapshot.rs +++ b/runtime/src/serde_snapshot.rs @@ -150,6 +150,14 @@ trait TypeContext<'a> { where Self: std::marker::Sized; + #[cfg(test)] + fn serialize_bank_and_storage_without_extra_fields( + serializer: S, + serializable_bank: &SerializableBankAndStorageNoExtra<'a, Self>, + ) -> std::result::Result + where + Self: std::marker::Sized; + fn serialize_accounts_db_fields( serializer: S, serializable_db: &SerializableAccountsDb<'a, Self>, @@ -285,6 +293,37 @@ where }) } +#[cfg(test)] +pub(crate) fn bank_to_stream_no_extra_fields( + serde_style: SerdeStyle, + stream: &mut BufWriter, + bank: &Bank, + snapshot_storages: &[SnapshotStorage], +) -> Result<(), Error> +where + W: Write, +{ + macro_rules! INTO { + ($style:ident) => { + bincode::serialize_into( + stream, + &SerializableBankAndStorageNoExtra::<$style::Context> { + bank, + snapshot_storages, + phantom: std::marker::PhantomData::default(), + }, + ) + }; + } + match serde_style { + SerdeStyle::Newer => INTO!(newer), + } + .map_err(|err| { + warn!("bankrc_to_stream error: {:?}", err); + err + }) +} + struct SerializableBankAndStorage<'a, C> { bank: &'a Bank, snapshot_storages: &'a [SnapshotStorage], @@ -300,6 +339,39 @@ impl<'a, C: TypeContext<'a>> Serialize for SerializableBankAndStorage<'a, C> { } } +#[cfg(test)] +struct SerializableBankAndStorageNoExtra<'a, C> { + bank: &'a Bank, + snapshot_storages: &'a [SnapshotStorage], + phantom: std::marker::PhantomData, +} + +#[cfg(test)] +impl<'a, C: TypeContext<'a>> Serialize for SerializableBankAndStorageNoExtra<'a, C> { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::ser::Serializer, + { + C::serialize_bank_and_storage_without_extra_fields(serializer, self) + } +} + +#[cfg(test)] +impl<'a, C> From> for SerializableBankAndStorage<'a, C> { + fn from(s: SerializableBankAndStorageNoExtra<'a, C>) -> SerializableBankAndStorage<'a, C> { + let SerializableBankAndStorageNoExtra { + bank, + snapshot_storages, + phantom, + } = s; + SerializableBankAndStorage { + bank, + snapshot_storages, + phantom, + } + } +} + struct SerializableAccountsDb<'a, C> { accounts_db: &'a AccountsDb, slot: Slot, diff --git a/runtime/src/serde_snapshot/newer.rs b/runtime/src/serde_snapshot/newer.rs index eb44af3525b..d1aac9d015d 100644 --- a/runtime/src/serde_snapshot/newer.rs +++ b/runtime/src/serde_snapshot/newer.rs @@ -185,6 +185,29 @@ impl<'a> TypeContext<'a> for Context { serializer: S, serializable_bank: &SerializableBankAndStorage<'a, Self>, ) -> std::result::Result + where + Self: std::marker::Sized, + { + // ONLY FOR THE BACKPORT, extra field is serialized on master + let ancestors = HashMap::from(&serializable_bank.bank.ancestors); + let fields = serializable_bank.bank.get_fields_to_serialize(&ancestors); + ( + SerializableVersionedBank::from(fields), + SerializableAccountsDb::<'a, Self> { + accounts_db: &*serializable_bank.bank.rc.accounts.accounts_db, + slot: serializable_bank.bank.rc.slot, + account_storage_entries: serializable_bank.snapshot_storages, + phantom: std::marker::PhantomData::default(), + }, + ) + .serialize(serializer) + } + + #[cfg(test)] + fn serialize_bank_and_storage_without_extra_fields( + serializer: S, + serializable_bank: &SerializableBankAndStorageNoExtra<'a, Self>, + ) -> std::result::Result where Self: std::marker::Sized, { @@ -255,8 +278,18 @@ impl<'a> TypeContext<'a> for Context { where R: Read, { - let bank_fields = deserialize_from::<_, DeserializableVersionedBank>(&mut stream)?.into(); + let mut bank_fields: BankFieldsToDeserialize = + deserialize_from::<_, DeserializableVersionedBank>(&mut stream)?.into(); let accounts_db_fields = Self::deserialize_accounts_db_fields(stream)?; + // Process extra fields + let lamports_per_signature: u64 = match deserialize_from(stream) { + Err(err) if err.to_string() == "io error: unexpected end of file" => Ok(0), + Err(err) if err.to_string() == "io error: failed to fill whole buffer" => Ok(0), + result => result, + }?; + bank_fields.fee_rate_governor = bank_fields + .fee_rate_governor + .clone_with_lamports_per_signature(lamports_per_signature); Ok((bank_fields, accounts_db_fields)) } diff --git a/runtime/src/serde_snapshot/tests.rs b/runtime/src/serde_snapshot/tests.rs index 4d1118567fd..c38ef6cdcfd 100644 --- a/runtime/src/serde_snapshot/tests.rs +++ b/runtime/src/serde_snapshot/tests.rs @@ -5,6 +5,7 @@ use { accounts::{create_test_accounts, Accounts}, accounts_db::{get_temp_accounts_paths, AccountShrinkThreshold}, bank::{Bank, StatusCacheRc}, + genesis_utils::activate_feature, hardened_unpack::UnpackedAppendVecMap, }, bincode::serialize_into, @@ -12,6 +13,7 @@ use { solana_sdk::{ account::{AccountSharedData, ReadableAccount}, clock::Slot, + feature_set::disable_fee_calculator, genesis_config::{create_genesis_config, ClusterType}, pubkey::Pubkey, signature::{Keypair, Signer}, @@ -299,6 +301,64 @@ fn test_bank_serialize_newer() { test_bank_serialize_style(SerdeStyle::Newer) } +#[test] +fn test_blank_extra_fields() { + solana_logger::setup(); + let (mut genesis_config, _) = create_genesis_config(500); + activate_feature(&mut genesis_config, disable_fee_calculator::id()); + + let bank0 = Arc::new(Bank::new_for_tests(&genesis_config)); + bank0.squash(); + let mut bank = Bank::new_from_parent(&bank0, &Pubkey::default(), 1); + + // Set extra fields + bank.fee_rate_governor.lamports_per_signature = 7000; + + // Serialize, but don't serialize the extra fields + let snapshot_storages = bank.get_snapshot_storages(None); + let mut buf = vec![]; + let mut writer = Cursor::new(&mut buf); + crate::serde_snapshot::bank_to_stream_no_extra_fields( + SerdeStyle::Newer, + &mut std::io::BufWriter::new(&mut writer), + &bank, + &snapshot_storages, + ) + .unwrap(); + + // Deserialize + let rdr = Cursor::new(&buf[..]); + let mut reader = std::io::BufReader::new(&buf[rdr.position() as usize..]); + let mut snapshot_streams = SnapshotStreams { + full_snapshot_stream: &mut reader, + incremental_snapshot_stream: None, + }; + let (_accounts_dir, dbank_paths) = get_temp_accounts_paths(4).unwrap(); + let copied_accounts = TempDir::new().unwrap(); + let unpacked_append_vec_map = + copy_append_vecs(&bank.rc.accounts.accounts_db, copied_accounts.path()).unwrap(); + let dbank = crate::serde_snapshot::bank_from_streams( + SerdeStyle::Newer, + &mut snapshot_streams, + &dbank_paths, + unpacked_append_vec_map, + &genesis_config, + None, + None, + AccountSecondaryIndexes::default(), + false, + None, + AccountShrinkThreshold::default(), + false, + Some(crate::accounts_db::ACCOUNTS_DB_CONFIG_FOR_TESTING), + None, + ) + .unwrap(); + + // Defaults to 0 + assert_eq!(0, dbank.fee_rate_governor.lamports_per_signature); +} + #[cfg(RUSTC_WITH_SPECIALIZATION)] mod test_bank_serialize { use super::*; diff --git a/sdk/program/src/fee_calculator.rs b/sdk/program/src/fee_calculator.rs index b190f050c46..042fca202e1 100644 --- a/sdk/program/src/fee_calculator.rs +++ b/sdk/program/src/fee_calculator.rs @@ -161,6 +161,13 @@ impl FeeRateGovernor { me } + pub fn clone_with_lamports_per_signature(&self, lamports_per_signature: u64) -> Self { + Self { + lamports_per_signature, + ..*self + } + } + /// calculate unburned fee from a fee total, returns (unburned, burned) pub fn burn(&self, fees: u64) -> (u64, u64) { let burned = fees * u64::from(self.burn_percent) / 100;