diff --git a/accounts-db/src/tiered_storage/byte_block.rs b/accounts-db/src/tiered_storage/byte_block.rs index 6fc7dec611e9a9..0a9969010d8e27 100644 --- a/accounts-db/src/tiered_storage/byte_block.rs +++ b/accounts-db/src/tiered_storage/byte_block.rs @@ -95,6 +95,9 @@ impl ByteBlockWriter { if let Some(rent_epoch) = opt_fields.rent_epoch { size += self.write_pod(&rent_epoch)?; } + if let Some(lamports) = opt_fields.lamports { + size += self.write_pod(&lamports)?; + } debug_assert_eq!(size, opt_fields.size()); @@ -341,6 +344,7 @@ mod tests { fn write_optional_fields(format: AccountBlockFormat) { let mut test_epoch = 5432312; + let mut test_lamports = 2314312321321; let mut writer = ByteBlockWriter::new(format); let mut opt_fields_vec = vec![]; @@ -349,10 +353,17 @@ mod tests { // prepare a vector of optional fields that contains all combinations // of Some and None. for rent_epoch in [None, Some(test_epoch)] { - some_count += rent_epoch.iter().count(); - - opt_fields_vec.push(AccountMetaOptionalFields { rent_epoch }); - test_epoch += 1; + for lamports in [None, Some(test_lamports)] { + some_count += rent_epoch.map_or(0, |_| 1); + some_count += lamports.map_or(0, |_| 1); + + opt_fields_vec.push(AccountMetaOptionalFields { + rent_epoch, + lamports, + }); + test_epoch += 1; + test_lamports += 1; + } } // write all the combinations of the optional fields @@ -383,6 +394,12 @@ mod tests { verified_count += 1; offset += std::mem::size_of::(); } + if let Some(expected_lamports) = opt_fields.lamports { + let lamports = read_pod::(&decoded_buffer, offset).unwrap(); + assert_eq!(*lamports, expected_lamports); + verified_count += 1; + offset += std::mem::size_of::(); + } } // make sure the number of Some fields matches the number of fields we diff --git a/accounts-db/src/tiered_storage/error.rs b/accounts-db/src/tiered_storage/error.rs index 145334574b4ea3..b0b7281602affc 100644 --- a/accounts-db/src/tiered_storage/error.rs +++ b/accounts-db/src/tiered_storage/error.rs @@ -31,4 +31,7 @@ pub enum TieredStorageError { #[error("OffsetAlignmentError: offset {0} must be multiple of {1}")] OffsetAlignmentError(usize, usize), + + #[error("UnsupportedAccountMetaFormat: format is unsupported.")] + UnsupportedAccountMetaFormat, } diff --git a/accounts-db/src/tiered_storage/footer.rs b/accounts-db/src/tiered_storage/footer.rs index 1eb4fbdb3ff2ec..6302c8f620648f 100644 --- a/accounts-db/src/tiered_storage/footer.rs +++ b/accounts-db/src/tiered_storage/footer.rs @@ -57,8 +57,7 @@ impl Default for TieredStorageMagicNumber { pub enum AccountMetaFormat { #[default] Hot = 0, - // Temporarily comment out to avoid unimplemented!() block - // Cold = 1, + HotPacked = 1, } #[repr(u16)] @@ -336,7 +335,7 @@ mod tests { fn test_footer() { let path = get_append_vec_path("test_file_footer"); let expected_footer = TieredStorageFooter { - account_meta_format: AccountMetaFormat::Hot, + account_meta_format: AccountMetaFormat::HotPacked, owners_block_format: OwnersBlockFormat::AddressesOnly, index_block_format: IndexBlockFormat::AddressesThenOffsets, account_block_format: AccountBlockFormat::AlignedRaw, diff --git a/accounts-db/src/tiered_storage/hot.rs b/accounts-db/src/tiered_storage/hot.rs index 5448c9b0f8a5ee..5b3df51db660cb 100644 --- a/accounts-db/src/tiered_storage/hot.rs +++ b/accounts-db/src/tiered_storage/hot.rs @@ -5,6 +5,7 @@ use { account_storage::meta::{StoredAccountInfo, StoredAccountMeta}, accounts_file::MatchAccountOwnerError, accounts_hash::AccountHash, + accounts_index::ZeroLamport, tiered_storage::{ byte_block, file::TieredStorageFile, @@ -29,7 +30,7 @@ use { pub const HOT_FORMAT: TieredStorageFormat = TieredStorageFormat { meta_entry_size: std::mem::size_of::(), - account_meta_format: AccountMetaFormat::Hot, + account_meta_format: AccountMetaFormat::HotPacked, owners_block_format: OwnersBlockFormat::AddressesOnly, index_block_format: IndexBlockFormat::AddressesThenOffsets, account_block_format: AccountBlockFormat::AlignedRaw, @@ -141,8 +142,6 @@ impl HotAccountOffset { #[derive(Debug, Clone, Copy, PartialEq, Eq, Pod, Zeroable)] #[repr(C)] pub struct HotAccountMeta { - /// The balance of this account. - lamports: u64, /// Stores important fields in a packed struct. packed_fields: HotMetaPackedFields, /// Stores boolean flags and existence of each optional field. @@ -150,24 +149,17 @@ pub struct HotAccountMeta { } // Ensure there are no implicit padding bytes -const _: () = assert!(std::mem::size_of::() == 8 + 4 + 4); +const _: () = assert!(std::mem::size_of::() == 4 + 4); impl TieredAccountMeta for HotAccountMeta { /// Construct a HotAccountMeta instance. fn new() -> Self { HotAccountMeta { - lamports: 0, packed_fields: HotMetaPackedFields::default(), flags: AccountMetaFlags::new(), } } - /// A builder function that initializes lamports. - fn with_lamports(mut self, lamports: u64) -> Self { - self.lamports = lamports; - self - } - /// A builder function that initializes the number of padding bytes /// for the account data associated with the current meta. fn with_account_data_padding(mut self, padding: u8) -> Self { @@ -201,9 +193,25 @@ impl TieredAccountMeta for HotAccountMeta { self } - /// Returns the balance of the lamports associated with the account. - fn lamports(&self) -> u64 { - self.lamports + /// Returns the balance of the lamports associated with the account + /// from the TieredAccountMeta, or None if the lamports is stored + /// inside the optional field. + fn lamports_from_meta(&self) -> Option { + self.flags.lamports() + } + + /// Returns the balance of the lamports associated with the account + /// from the optional fields, or None if the lamports is stored + /// inside the TieredAccountMeta. + fn lamports_from_optional_fields(&self, account_block: &[u8]) -> Option { + self.flags + .has_optional_lamports_field() + .then(|| { + let offset = self.optional_fields_offset(account_block) + + AccountMetaOptionalFields::lamports_offset(self.flags()); + byte_block::read_pod::(account_block, offset).copied() + }) + .flatten() } /// Returns the number of padding bytes for the associated account data @@ -263,6 +271,13 @@ impl TieredAccountMeta for HotAccountMeta { } } +impl ZeroLamport for HotAccountMeta { + /// Whether the account has zero lamports. + fn is_zero_lamport(&self) -> bool { + self.flags.is_zero_lamport() + } +} + /// The struct that offers read APIs for accessing a hot account. #[derive(PartialEq, Eq, Debug)] pub struct HotAccount<'accounts_file, M: TieredAccountMeta> { @@ -299,7 +314,15 @@ impl<'accounts_file, M: TieredAccountMeta> HotAccount<'accounts_file, M> { impl<'accounts_file, M: TieredAccountMeta> ReadableAccount for HotAccount<'accounts_file, M> { /// Returns the balance of the lamports of this account. fn lamports(&self) -> u64 { - self.meta.lamports() + if let Some(lamports) = self.meta.lamports_from_meta() { + return lamports; + } + + // If the meta does not have the lamports, then it must inside the optional + // fields. + self.meta + .lamports_from_optional_fields(self.account_block) + .unwrap() } /// Returns the address of the owner of this account. @@ -375,17 +398,22 @@ impl HotStorageReader { &self, account_offset: HotAccountOffset, ) -> TieredStorageResult<&HotAccountMeta> { - let offset = account_offset.offset(); - - assert!( - offset.saturating_add(std::mem::size_of::()) - <= self.footer.index_block_offset as usize, - "reading HotAccountOffset ({}) would exceed accounts blocks offset boundary ({}).", - offset, - self.footer.index_block_offset, - ); - let (meta, _) = get_pod::(&self.mmap, offset)?; - Ok(meta) + match self.footer.account_meta_format { + AccountMetaFormat::HotPacked => { + let offset = account_offset.offset(); + + assert!( + offset.saturating_add(std::mem::size_of::()) + <= self.footer.index_block_offset as usize, + "reading HotAccountOffset ({}) would exceed accounts blocks offset boundary ({}).", + offset, + self.footer.index_block_offset, + ); + let (meta, _) = get_pod::(&self.mmap, offset)?; + Ok(meta) + } + _ => Err(TieredStorageError::UnsupportedAccountMetaFormat), + } } /// Returns the offset to the account given the specified index. @@ -431,7 +459,7 @@ impl HotStorageReader { .get_account_meta_from_offset(account_offset) .map_err(|_| MatchAccountOwnerError::UnableToLoad)?; - if account_meta.lamports() == 0 { + if account_meta.is_zero_lamport() { Err(MatchAccountOwnerError::NoMatch) } else { let account_owner = self @@ -549,6 +577,9 @@ fn write_optional_fields( if let Some(rent_epoch) = opt_fields.rent_epoch { size += file.write_pod(&rent_epoch)?; } + if let Some(lamports) = opt_fields.lamports { + size += file.write_pod(&lamports)?; + } debug_assert_eq!(size, opt_fields.size()); @@ -579,14 +610,16 @@ impl HotStorageWriter { executable: bool, rent_epoch: Option, ) -> TieredStorageResult { - let optional_fields = AccountMetaOptionalFields { rent_epoch }; + let optional_fields = AccountMetaOptionalFields { + rent_epoch, + lamports: AccountMetaFlags::get_optional_lamports_field(lamports), + }; - let mut flags = AccountMetaFlags::new_from(&optional_fields); + let mut flags = AccountMetaFlags::new_from(&optional_fields, lamports); flags.set_executable(executable); let padding_len = padding_bytes(account_data.len()); let meta = HotAccountMeta::new() - .with_lamports(lamports) .with_owner_offset(owner_offset) .with_account_data_size(account_data.len() as u64) .with_account_data_padding(padding_len) @@ -710,7 +743,10 @@ pub mod tests { footer::{AccountBlockFormat, AccountMetaFormat, TieredStorageFooter, FOOTER_SIZE}, hot::{HotAccountMeta, HotStorageReader}, index::{AccountIndexWriterEntry, IndexBlockFormat, IndexOffset}, - meta::{AccountMetaFlags, AccountMetaOptionalFields, TieredAccountMeta}, + meta::{ + AccountMetaFlags, AccountMetaOptionalFields, TieredAccountMeta, + LAMPORTS_INFO_MAX_BALANCE, + }, owners::{OwnersBlockFormat, OwnersTable}, test_utils::{create_test_account, verify_test_account}, }, @@ -726,10 +762,9 @@ pub mod tests { #[test] fn test_hot_account_meta_layout() { - assert_eq!(offset_of!(HotAccountMeta, lamports), 0x00); - assert_eq!(offset_of!(HotAccountMeta, packed_fields), 0x08); - assert_eq!(offset_of!(HotAccountMeta, flags), 0x0C); - assert_eq!(std::mem::size_of::(), 16); + assert_eq!(offset_of!(HotAccountMeta, packed_fields), 0x00); + assert_eq!(offset_of!(HotAccountMeta, flags), 0x04); + assert_eq!(std::mem::size_of::(), 8); } #[test] @@ -796,30 +831,44 @@ pub mod tests { HotAccountMeta::new().with_owner_offset(OwnerOffset(MAX_HOT_OWNER_OFFSET.0 + 1)); } - #[test] - fn test_hot_account_meta() { - const TEST_LAMPORTS: u64 = 2314232137; + /// A helper function to test hot account lamports. + fn hot_account_lamports_test(lamports: u64, expected_lamports: Option) { const TEST_PADDING: u8 = 5; const TEST_OWNER_OFFSET: OwnerOffset = OwnerOffset(0x1fef_1234); const TEST_RENT_EPOCH: Epoch = 7; let optional_fields = AccountMetaOptionalFields { rent_epoch: Some(TEST_RENT_EPOCH), + lamports: AccountMetaFlags::get_optional_lamports_field(lamports), }; - let flags = AccountMetaFlags::new_from(&optional_fields); + let flags = AccountMetaFlags::new_from(&optional_fields, lamports); let meta = HotAccountMeta::new() - .with_lamports(TEST_LAMPORTS) .with_account_data_padding(TEST_PADDING) .with_owner_offset(TEST_OWNER_OFFSET) .with_flags(&flags); - assert_eq!(meta.lamports(), TEST_LAMPORTS); + assert_eq!(meta.lamports_from_meta(), expected_lamports); assert_eq!(meta.account_data_padding(), TEST_PADDING); assert_eq!(meta.owner_offset(), TEST_OWNER_OFFSET); assert_eq!(*meta.flags(), flags); } + #[test] + fn test_hot_account_zero_lamports() { + hot_account_lamports_test(0, Some(0)); + } + + #[test] + fn test_hot_account_lamports_in_meta() { + hot_account_lamports_test(LAMPORTS_INFO_MAX_BALANCE, Some(LAMPORTS_INFO_MAX_BALANCE)); + } + + #[test] + fn test_hot_account_lamports_not_fit() { + hot_account_lamports_test(LAMPORTS_INFO_MAX_BALANCE + 1, None); + } + #[test] fn test_hot_account_meta_full() { let account_data = [11u8; 83]; @@ -828,14 +877,18 @@ pub mod tests { const TEST_LAMPORT: u64 = 2314232137; const OWNER_OFFSET: u32 = 0x1fef_1234; const TEST_RENT_EPOCH: Epoch = 7; + const TEST_BIG_LAMPORTS: u64 = u64::MAX; let optional_fields = AccountMetaOptionalFields { rent_epoch: Some(TEST_RENT_EPOCH), + lamports: AccountMetaFlags::get_optional_lamports_field(TEST_BIG_LAMPORTS), }; + // Since the lamports is too big to fit into meta, it is expected + // to be in the optional fields. + assert_eq!(optional_fields.lamports, Some(TEST_BIG_LAMPORTS)); - let flags = AccountMetaFlags::new_from(&optional_fields); + let flags = AccountMetaFlags::new_from(&optional_fields, TEST_BIG_LAMPORTS); let expected_meta = HotAccountMeta::new() - .with_lamports(TEST_LAMPORT) .with_account_data_padding(padding.len().try_into().unwrap()) .with_owner_offset(OwnerOffset(OWNER_OFFSET)) .with_flags(&flags); @@ -854,6 +907,7 @@ pub mod tests { assert_eq!(expected_meta, *meta); assert!(meta.flags().has_rent_epoch()); assert_eq!(meta.account_data_padding() as usize, padding.len()); + assert!(meta.flags().has_optional_lamports_field()); let account_block = &buffer[std::mem::size_of::()..]; assert_eq!( @@ -865,6 +919,10 @@ pub mod tests { assert_eq!(account_data.len(), meta.account_data_size(account_block)); assert_eq!(account_data, meta.account_data(account_block)); assert_eq!(meta.rent_epoch(account_block), optional_fields.rent_epoch); + assert_eq!( + meta.lamports_from_optional_fields(account_block), + optional_fields.lamports + ); } #[test] @@ -873,7 +931,7 @@ pub mod tests { let temp_dir = TempDir::new().unwrap(); let path = temp_dir.path().join("test_hot_storage_footer"); let expected_footer = TieredStorageFooter { - account_meta_format: AccountMetaFormat::Hot, + account_meta_format: AccountMetaFormat::HotPacked, owners_block_format: OwnersBlockFormat::AddressesOnly, index_block_format: IndexBlockFormat::AddressesThenOffsets, account_block_format: AccountBlockFormat::AlignedRaw, @@ -915,15 +973,13 @@ pub mod tests { let hot_account_metas: Vec<_> = (0..NUM_ACCOUNTS) .map(|_| { - HotAccountMeta::new() - .with_lamports(rng.gen_range(0..u64::MAX)) - .with_owner_offset(OwnerOffset(rng.gen_range(0..NUM_ACCOUNTS))) + HotAccountMeta::new().with_owner_offset(OwnerOffset(rng.gen_range(0..NUM_ACCOUNTS))) }) .collect(); let account_offsets: Vec<_>; let mut footer = TieredStorageFooter { - account_meta_format: AccountMetaFormat::Hot, + account_meta_format: AccountMetaFormat::HotPacked, account_entry_count: NUM_ACCOUNTS, ..TieredStorageFooter::default() }; @@ -965,7 +1021,7 @@ pub mod tests { .join("test_get_acount_meta_from_offset_out_of_bounds"); let footer = TieredStorageFooter { - account_meta_format: AccountMetaFormat::Hot, + account_meta_format: AccountMetaFormat::HotPacked, index_block_offset: 160, ..TieredStorageFooter::default() }; @@ -1008,7 +1064,7 @@ pub mod tests { .collect(); let mut footer = TieredStorageFooter { - account_meta_format: AccountMetaFormat::Hot, + account_meta_format: AccountMetaFormat::HotPacked, account_entry_count: NUM_ACCOUNTS, // Set index_block_offset to 0 as we didn't write any account // meta/data in this test @@ -1052,7 +1108,7 @@ pub mod tests { .collect(); let footer = TieredStorageFooter { - account_meta_format: AccountMetaFormat::Hot, + account_meta_format: AccountMetaFormat::HotPacked, // meta/data nor index block in this test owners_block_offset: 0, ..TieredStorageFooter::default() @@ -1103,14 +1159,17 @@ pub mod tests { let hot_account_metas: Vec<_> = std::iter::repeat_with({ || { HotAccountMeta::new() - .with_lamports(rng.gen_range(1..u64::MAX)) .with_owner_offset(OwnerOffset(rng.gen_range(0..NUM_OWNERS))) + .with_flags(&AccountMetaFlags::new_from( + &AccountMetaOptionalFields::default(), + rng.gen_range(0..LAMPORTS_INFO_MAX_BALANCE), + )) } }) .take(NUM_ACCOUNTS as usize) .collect(); let mut footer = TieredStorageFooter { - account_meta_format: AccountMetaFormat::Hot, + account_meta_format: AccountMetaFormat::HotPacked, account_entry_count: NUM_ACCOUNTS, owner_count: NUM_OWNERS, ..TieredStorageFooter::default() @@ -1218,7 +1277,6 @@ pub mod tests { let account_metas: Vec<_> = (0..NUM_ACCOUNTS) .map(|i| { HotAccountMeta::new() - .with_lamports(rng.gen_range(0..u64::MAX)) .with_owner_offset(OwnerOffset(rng.gen_range(0..NUM_OWNERS) as u32)) .with_account_data_padding(padding_bytes(account_datas[i].len())) }) @@ -1230,7 +1288,7 @@ pub mod tests { .collect(); let mut footer = TieredStorageFooter { - account_meta_format: AccountMetaFormat::Hot, + account_meta_format: AccountMetaFormat::HotPacked, account_entry_count: NUM_ACCOUNTS as u32, owner_count: NUM_OWNERS as u32, ..TieredStorageFooter::default() @@ -1288,7 +1346,6 @@ pub mod tests { .get_account(IndexOffset(i as u32)) .unwrap() .unwrap(); - assert_eq!(stored_meta.lamports(), account_metas[i].lamports()); assert_eq!(stored_meta.data().len(), account_datas[i].len()); assert_eq!(stored_meta.data(), account_datas[i]); assert_eq!( diff --git a/accounts-db/src/tiered_storage/meta.rs b/accounts-db/src/tiered_storage/meta.rs index 2aa53e5a4de1ed..3513da96e270d9 100644 --- a/accounts-db/src/tiered_storage/meta.rs +++ b/accounts-db/src/tiered_storage/meta.rs @@ -1,12 +1,30 @@ //! The account meta and related structs for the tiered storage. use { - crate::tiered_storage::owners::OwnerOffset, + crate::{accounts_index::ZeroLamport, tiered_storage::owners::OwnerOffset}, bytemuck::{Pod, Zeroable}, modular_bitfield::prelude::*, solana_sdk::stake_history::Epoch, }; +/// The number of bits used in lamports_info field. +/// Note that this value must match the bits in AccountMetaFlags::lamports_info. +pub const LAMPORTS_INFO_BITS: u64 = 30; +/// The max lamports balance that the lamports_info field can handle. +/// Any lamports beyond this value will be stored separately in optional fields. +pub const LAMPORTS_INFO_MAX_BALANCE: u64 = + ((1u64 << LAMPORTS_INFO_BITS) - 1) - LAMPORTS_INFO_RESERVED_VALUES; + +/// The number of special values inside lamports_info. +/// This const MUST be updated when adding new reserved values. +pub const LAMPORTS_INFO_RESERVED_VALUES: u64 = 2; + +/// A reserved lamports_info value indicating zero-lamports balance. +pub const LAMPORTS_INFO_IS_ZERO_BALANCE: u32 = 0; +/// A reserved lamports_info value indicating the lamports balance is stored +/// in optional fields. +pub const LAMPORTS_INFO_HAS_OPTIONAL_FIELD: u32 = 1; + /// The struct that handles the account meta flags. #[bitfield(bits = 32)] #[repr(C)] @@ -16,8 +34,13 @@ pub struct AccountMetaFlags { pub has_rent_epoch: bool, /// whether the account is executable pub executable: bool, - /// the reserved bits. - reserved: B30, + /// this fewer-than-u64 lamports info stores lamports that can fit + /// within its limitation, or a bit indicating the lamport is stored + /// separately as an optional field. + /// + /// Note that the number of bits using in this field must match + /// the const LAMPORTS_INFO_BITS. + pub lamports_info: B30, } // Ensure there are no implicit padding bytes @@ -25,13 +48,10 @@ const _: () = assert!(std::mem::size_of::() == 4); /// A trait that allows different implementations of the account meta that /// support different tiers of the accounts storage. -pub trait TieredAccountMeta: Sized { +pub trait TieredAccountMeta: Sized + ZeroLamport { /// Constructs a TieredAcountMeta instance. fn new() -> Self; - /// A builder function that initializes lamports. - fn with_lamports(self, lamports: u64) -> Self; - /// A builder function that initializes the number of padding bytes /// for the account data associated with the current meta. fn with_account_data_padding(self, padding: u8) -> Self; @@ -47,8 +67,15 @@ pub trait TieredAccountMeta: Sized { /// meta. fn with_flags(self, flags: &AccountMetaFlags) -> Self; - /// Returns the balance of the lamports associated with the account. - fn lamports(&self) -> u64; + /// Returns the balance of the lamports associated with the account + /// from the TieredAccountMeta, or None if the lamports is stored + /// inside the optional field. + fn lamports_from_meta(&self) -> Option; + + /// Returns the balance of the lamports associated with the account + /// from the optional fields, or None if the lamports is stored + /// inside the TieredAccountMeta. + fn lamports_from_optional_fields(&self, _account_block: &[u8]) -> Option; /// Returns the number of padding bytes for the associated account data fn account_data_padding(&self) -> u8; @@ -82,28 +109,64 @@ pub trait TieredAccountMeta: Sized { } impl AccountMetaFlags { - pub fn new_from(optional_fields: &AccountMetaOptionalFields) -> Self { + pub fn new_from(optional_fields: &AccountMetaOptionalFields, lamports: u64) -> Self { let mut flags = AccountMetaFlags::default(); flags.set_has_rent_epoch(optional_fields.rent_epoch.is_some()); + if optional_fields.lamports.is_some() { + flags.set_lamports_info(LAMPORTS_INFO_HAS_OPTIONAL_FIELD); + } else if lamports != 0 { + debug_assert!(lamports <= LAMPORTS_INFO_MAX_BALANCE); + flags.set_lamports_info((lamports + LAMPORTS_INFO_RESERVED_VALUES) as u32); + } flags.set_executable(false); flags } + + pub fn lamports(&self) -> Option { + match self.lamports_info() { + LAMPORTS_INFO_IS_ZERO_BALANCE => Some(0), + LAMPORTS_INFO_HAS_OPTIONAL_FIELD => None, + packed_lamports => Some(packed_lamports as u64 - LAMPORTS_INFO_RESERVED_VALUES), + } + } + + pub fn is_zero_lamport(&self) -> bool { + self.lamports_info() == LAMPORTS_INFO_IS_ZERO_BALANCE + } + + pub fn has_optional_lamports_field(&self) -> bool { + self.lamports_info() == LAMPORTS_INFO_HAS_OPTIONAL_FIELD + } + + pub fn get_optional_lamports_field(lamports: u64) -> Option { + if lamports > LAMPORTS_INFO_MAX_BALANCE { + Some(lamports) + } else { + None + } + } } /// The in-memory struct for the optional fields for tiered account meta. /// /// Note that the storage representation of the optional fields might be /// different from its in-memory representation. -#[derive(Debug, PartialEq, Eq, Clone)] +#[derive(Debug, PartialEq, Eq, Clone, Default)] pub struct AccountMetaOptionalFields { /// the epoch at which its associated account will next owe rent pub rent_epoch: Option, + /// The balance of this account. + /// + /// It is Some only when lamports balance of the current account + /// cannot be stored inside the AccountMeta. + pub lamports: Option, } impl AccountMetaOptionalFields { /// The size of the optional fields in bytes (excluding the boolean flags). pub fn size(&self) -> usize { self.rent_epoch.map_or(0, |_| std::mem::size_of::()) + + self.lamports.map_or(0, |_| std::mem::size_of::()) } /// Given the specified AccountMetaFlags, returns the size of its @@ -114,6 +177,10 @@ impl AccountMetaOptionalFields { fields_size += std::mem::size_of::(); } + if flags.lamports_info() == LAMPORTS_INFO_HAS_OPTIONAL_FIELD { + fields_size += std::mem::size_of::(); + } + fields_size } @@ -122,18 +189,35 @@ impl AccountMetaOptionalFields { pub fn rent_epoch_offset(_flags: &AccountMetaFlags) -> usize { 0 } + + /// Given the specified AccountMetaFlags, returns the relative offset + /// of its lamports field to the offset of its optional fields entry. + pub fn lamports_offset(flags: &AccountMetaFlags) -> usize { + let mut offset = Self::rent_epoch_offset(flags); + if flags.has_rent_epoch() { + offset += std::mem::size_of::(); + } + + offset + } } #[cfg(test)] pub mod tests { use super::*; + impl AccountMetaFlags { + pub fn new_from_test(optional_fields: &AccountMetaOptionalFields) -> Self { + AccountMetaFlags::new_from(optional_fields, 0) + } + } + #[test] fn test_account_meta_flags_new() { let flags = AccountMetaFlags::new(); assert!(!flags.has_rent_epoch()); - assert_eq!(flags.reserved(), 0u32); + assert_eq!(flags.lamports_info(), 0u32); assert_eq!( std::mem::size_of::(), @@ -160,65 +244,85 @@ pub mod tests { assert!(flags.executable()); verify_flags_serialization(&flags); - // make sure the reserved bits are untouched. - assert_eq!(flags.reserved(), 0u32); + // make sure the lamports_info bits are untouched. + assert_eq!(flags.lamports_info(), 0u32); } fn update_and_verify_flags(opt_fields: &AccountMetaOptionalFields) { - let flags: AccountMetaFlags = AccountMetaFlags::new_from(opt_fields); + let flags: AccountMetaFlags = AccountMetaFlags::new_from_test(opt_fields); assert_eq!(flags.has_rent_epoch(), opt_fields.rent_epoch.is_some()); - assert_eq!(flags.reserved(), 0u32); + assert_eq!( + flags.lamports_info(), + opt_fields + .lamports + .map_or(0, |_| LAMPORTS_INFO_HAS_OPTIONAL_FIELD) + ); } #[test] fn test_optional_fields_update_flags() { let test_epoch = 5432312; + let test_lamports = 2314312321321; for rent_epoch in [None, Some(test_epoch)] { - update_and_verify_flags(&AccountMetaOptionalFields { rent_epoch }); + for lamports in [None, Some(test_lamports)] { + update_and_verify_flags(&AccountMetaOptionalFields { + rent_epoch, + lamports, + }); + } } } #[test] fn test_optional_fields_size() { let test_epoch = 5432312; + let test_lamports = 2314312321321; for rent_epoch in [None, Some(test_epoch)] { - let opt_fields = AccountMetaOptionalFields { rent_epoch }; - assert_eq!( - opt_fields.size(), - rent_epoch.map_or(0, |_| std::mem::size_of::()), - ); - assert_eq!( - opt_fields.size(), - AccountMetaOptionalFields::size_from_flags(&AccountMetaFlags::new_from( - &opt_fields - )) - ); + for lamports in [None, Some(test_lamports)] { + let opt_fields = AccountMetaOptionalFields { + rent_epoch, + lamports, + }; + assert_eq!( + opt_fields.size(), + rent_epoch.map_or(0, |_| std::mem::size_of::()) + + lamports.map_or(0, |_| std::mem::size_of::()), + ); + assert_eq!( + opt_fields.size(), + AccountMetaOptionalFields::size_from_flags(&AccountMetaFlags::new_from_test( + &opt_fields, + )) + ); + } } } #[test] fn test_optional_fields_offset() { let test_epoch = 5432312; + let test_lamports = 2314312321321; for rent_epoch in [None, Some(test_epoch)] { - let rent_epoch_offset = 0; - let derived_size = if rent_epoch.is_some() { - std::mem::size_of::() - } else { - 0 - }; - let opt_fields = AccountMetaOptionalFields { rent_epoch }; - let flags = AccountMetaFlags::new_from(&opt_fields); - assert_eq!( - AccountMetaOptionalFields::rent_epoch_offset(&flags), - rent_epoch_offset - ); - assert_eq!( - AccountMetaOptionalFields::size_from_flags(&flags), - derived_size - ); + for lamports in [None, Some(test_lamports)] { + let opt_fields = AccountMetaOptionalFields { + rent_epoch, + lamports, + }; + let flags = AccountMetaFlags::new_from_test(&opt_fields); + assert_eq!(AccountMetaOptionalFields::rent_epoch_offset(&flags), 0,); + assert_eq!( + AccountMetaOptionalFields::lamports_offset(&flags), + rent_epoch.map_or(0, |_| std::mem::size_of::()), + ); + assert_eq!( + AccountMetaOptionalFields::size_from_flags(&flags), + rent_epoch.map_or(0, |_| std::mem::size_of::()) + + lamports.map_or(0, |_| std::mem::size_of::()), + ); + } } } } diff --git a/accounts-db/src/tiered_storage/readable.rs b/accounts-db/src/tiered_storage/readable.rs index e3d169d4f6d99e..c1ed721c7c9332 100644 --- a/accounts-db/src/tiered_storage/readable.rs +++ b/accounts-db/src/tiered_storage/readable.rs @@ -3,6 +3,7 @@ use { account_storage::meta::StoredAccountMeta, accounts_file::MatchAccountOwnerError, tiered_storage::{ + error::TieredStorageError, footer::{AccountMetaFormat, TieredStorageFooter}, hot::HotStorageReader, index::IndexOffset, @@ -24,7 +25,8 @@ impl TieredStorageReader { pub fn new_from_path(path: impl AsRef) -> TieredStorageResult { let footer = TieredStorageFooter::new_from_path(&path)?; match footer.account_meta_format { - AccountMetaFormat::Hot => Ok(Self::Hot(HotStorageReader::new_from_path(path)?)), + AccountMetaFormat::HotPacked => Ok(Self::Hot(HotStorageReader::new_from_path(path)?)), + _ => Err(TieredStorageError::UnsupportedAccountMetaFormat), } } @@ -87,3 +89,32 @@ impl TieredStorageReader { } } } + +#[cfg(test)] +pub mod tests { + use { + super::*, + crate::tiered_storage::{file::TieredStorageFile, footer::AccountMetaFormat}, + tempfile::TempDir, + }; + + #[test] + #[should_panic(expected = "UnsupportedAccountMetaFormat")] + fn test_unsupported_meta_format() { + // Generate a new temp path that is guaranteed to NOT already have a file. + let temp_dir = TempDir::new().unwrap(); + let path = temp_dir.path().join("test_unsupported_meta_format"); + + let footer = TieredStorageFooter { + account_meta_format: AccountMetaFormat::Hot, // deprecated + ..TieredStorageFooter::default() + }; + + { + let file = TieredStorageFile::new_writable(&path).unwrap(); + footer.write_footer_block(&file).unwrap(); + } + + TieredStorageReader::new_from_path(&path).unwrap(); + } +} diff --git a/accounts-db/src/tiered_storage/test_utils.rs b/accounts-db/src/tiered_storage/test_utils.rs index f44f20f77cc5dd..8a390ebce0c6d7 100644 --- a/accounts-db/src/tiered_storage/test_utils.rs +++ b/accounts-db/src/tiered_storage/test_utils.rs @@ -4,7 +4,7 @@ use { crate::{ account_storage::meta::{StoredAccountMeta, StoredMeta}, accounts_hash::AccountHash, - tiered_storage::owners::OWNER_NO_OWNER, + tiered_storage::{meta::LAMPORTS_INFO_MAX_BALANCE, owners::OWNER_NO_OWNER}, }, solana_sdk::{ account::{Account, AccountSharedData, ReadableAccount}, @@ -24,7 +24,13 @@ pub(super) fn create_test_account(seed: u64) -> (StoredMeta, AccountSharedData) let data_byte = seed as u8; let owner_byte = u8::MAX - data_byte; let account = Account { - lamports: seed, + // make sure the created test accounts cover both lamports inside meta + // and lamports inside optional fields. + lamports: if seed % 2 == 0 { + seed + } else { + seed + LAMPORTS_INFO_MAX_BALANCE + }, data: std::iter::repeat(data_byte).take(seed as usize).collect(), // this will allow some test account sharing the same owner. owner: [owner_byte; 32].into(),