diff --git a/accounts-db/src/accounts_db.rs b/accounts-db/src/accounts_db.rs index 0e9787eda10..8ed2767f5ae 100644 --- a/accounts-db/src/accounts_db.rs +++ b/accounts-db/src/accounts_db.rs @@ -2588,7 +2588,7 @@ impl AccountsDb { return; } if let Some(storage) = self.storage.get_slot_storage_entry(slot) { - storage.accounts.scan_accounts(|account| { + storage.accounts.scan_accounts(|_offset, account| { let pk = account.pubkey(); match pubkey_refcount.entry(*pk) { dashmap::mapref::entry::Entry::Occupied(mut occupied_entry) => { @@ -4487,12 +4487,12 @@ impl AccountsDb { self.scan_cache_storage_fallback(slot, cache_map_func, |retval, storage| { match scan_account_storage_data { ScanAccountStorageData::NoData => { - storage.scan_accounts_without_data(|account_without_data| { + storage.scan_accounts_without_data(|_offset, account_without_data| { storage_scan_func(retval, &account_without_data, None); }); } ScanAccountStorageData::DataRefForStorage => { - storage.scan_accounts(|account| { + storage.scan_accounts(|_offset, account| { let account_without_data = StoredAccountInfoWithoutData::new_from(&account); storage_scan_func(retval, &account_without_data, Some(account.data)); }); @@ -6384,7 +6384,7 @@ impl AccountsDb { let mut lt_hash = storages .par_iter() .fold(LtHash::identity, |mut accum, storage| { - storage.accounts.scan_accounts(|account| { + storage.accounts.scan_accounts(|_offset, account| { let account_lt_hash = Self::lt_hash_account(&account, account.pubkey()); accum.mix_in(&account_lt_hash.0); }); @@ -8081,7 +8081,7 @@ impl AccountsDb { }; if secondary { // scan storage a second time to update the secondary index - storage.accounts.scan_accounts(|stored_account| { + storage.accounts.scan_accounts(|_offset, stored_account| { self.accounts_index.update_secondary_indexes( stored_account.pubkey(), &stored_account, diff --git a/accounts-db/src/accounts_db/scan_account_storage.rs b/accounts-db/src/accounts_db/scan_account_storage.rs index 50ce870d361..fea7f9858d9 100644 --- a/accounts-db/src/accounts_db/scan_account_storage.rs +++ b/accounts-db/src/accounts_db/scan_account_storage.rs @@ -349,7 +349,7 @@ impl AccountsDb { where S: AppendVecScan, { - storage.accounts.scan_accounts(|account| { + storage.accounts.scan_accounts(|_offset, account| { if scanner.filter(account.pubkey()) { scanner.found_account(&LoadedAccount::Stored(account)) } @@ -675,7 +675,7 @@ mod tests { let slot = storage.slot(); let copied_storage = accounts_db.create_and_insert_store(slot, 10000, "test"); let mut all_accounts = Vec::default(); - storage.accounts.scan_accounts(|acct| { + storage.accounts.scan_accounts(|_offset, acct| { all_accounts.push((*acct.pubkey(), acct.to_account_shared_data())); }); let accounts = all_accounts @@ -709,7 +709,7 @@ mod tests { let slot = storage.slot() + max_slot; let copied_storage = accounts_db.create_and_insert_store(slot, 10000, "test"); let mut all_accounts = Vec::default(); - storage.accounts.scan_accounts(|acct| { + storage.accounts.scan_accounts(|_offset, acct| { all_accounts.push((*acct.pubkey(), acct.to_account_shared_data())); }); let accounts = all_accounts diff --git a/accounts-db/src/accounts_db/tests.rs b/accounts-db/src/accounts_db/tests.rs index 37d47e24609..da6b9f2fc9d 100644 --- a/accounts-db/src/accounts_db/tests.rs +++ b/accounts-db/src/accounts_db/tests.rs @@ -6655,7 +6655,7 @@ fn get_all_accounts_from_storages<'a>( storages .flat_map(|storage| { let mut vec = Vec::default(); - storage.accounts.scan_accounts(|account| { + storage.accounts.scan_accounts(|_offset, account| { vec.push((*account.pubkey(), account.to_account_shared_data())); }); // make sure scan_pubkeys results match diff --git a/accounts-db/src/accounts_file.rs b/accounts-db/src/accounts_file.rs index a3e6c663219..11caf6cf97f 100644 --- a/accounts-db/src/accounts_file.rs +++ b/accounts-db/src/accounts_file.rs @@ -2,7 +2,7 @@ use crate::append_vec::StoredAccountMeta; use { crate::{ - account_info::AccountInfo, + account_info::{AccountInfo, Offset}, account_storage::stored_account_info::{StoredAccountInfo, StoredAccountInfoWithoutData}, accounts_db::AccountsFileId, accounts_update_notifier_interface::AccountForGeyser, @@ -277,10 +277,14 @@ impl AccountsFile { /// Iterate over all accounts and call `callback` with each account. /// + /// `callback` parameters: + /// * Offset: the offset within the file of this account + /// * StoredAccountInfoWithoutData: the account itself, without account data + /// /// Note that account data is not read/passed to the callback. pub fn scan_accounts_without_data( &self, - callback: impl for<'local> FnMut(StoredAccountInfoWithoutData<'local>), + callback: impl for<'local> FnMut(Offset, StoredAccountInfoWithoutData<'local>), ) { match self { Self::AppendVec(av) => av.scan_accounts_without_data(callback), @@ -294,9 +298,16 @@ impl AccountsFile { /// Iterate over all accounts and call `callback` with each account. /// + /// `callback` parameters: + /// * Offset: the offset within the file of this account + /// * StoredAccountInfo: the account itself, with account data + /// /// Prefer scan_accounts_without_data() when account data is not needed, /// as it can potentially read less and be faster. - pub fn scan_accounts(&self, callback: impl for<'local> FnMut(StoredAccountInfo<'local>)) { + pub fn scan_accounts( + &self, + callback: impl for<'local> FnMut(Offset, StoredAccountInfo<'local>), + ) { match self { Self::AppendVec(av) => av.scan_accounts(callback), Self::TieredStorage(ts) => { @@ -330,7 +341,7 @@ impl AccountsFile { &self, mut callback: impl for<'local> FnMut(AccountForGeyser<'local>), ) { - self.scan_accounts(|account| { + self.scan_accounts(|_offset, account| { let account_for_geyser = AccountForGeyser { pubkey: account.pubkey(), lamports: account.lamports(), diff --git a/accounts-db/src/ancient_append_vecs.rs b/accounts-db/src/ancient_append_vecs.rs index ce7eee1a630..276f6783141 100644 --- a/accounts-db/src/ancient_append_vecs.rs +++ b/accounts-db/src/ancient_append_vecs.rs @@ -2079,7 +2079,7 @@ pub mod tests { shrink_in_progress .new_storage() .accounts - .scan_accounts(|_| { + .scan_accounts(|_, _| { count += 1; }); assert_eq!(count, 1); @@ -2240,7 +2240,7 @@ pub mod tests { }) .unwrap(); let mut count = 0; - storage.accounts.scan_accounts(|_| { + storage.accounts.scan_accounts(|_, _| { count += 1; }); assert_eq!(count, 2); @@ -3179,14 +3179,11 @@ pub mod tests { ); // make sure the single new append vec contains all the same accounts let mut two = Vec::default(); - one.first() - .unwrap() - .1 - .new_storage() - .accounts - .scan_accounts(|meta| { + one.first().unwrap().1.new_storage().accounts.scan_accounts( + |_offset, meta| { two.push((*meta.pubkey(), meta.to_account_shared_data())); - }); + }, + ); compare_all_accounts(&initial_accounts, &two[..]); } diff --git a/accounts-db/src/append_vec.rs b/accounts-db/src/append_vec.rs index 94054e33320..57b74399443 100644 --- a/accounts-db/src/append_vec.rs +++ b/accounts-db/src/append_vec.rs @@ -16,6 +16,7 @@ pub use meta::{AccountMeta, StoredMeta}; use meta::{AccountMeta, StoredMeta}; use { crate::{ + account_info::Offset, account_storage::stored_account_info::{StoredAccountInfo, StoredAccountInfoWithoutData}, accounts_file::{ AccountsFileError, InternalsForArchive, MatchAccountOwnerError, Result, StorageAccess, @@ -981,12 +982,17 @@ impl AppendVec { /// Iterate over all accounts and call `callback` with each account. /// + /// `callback` parameters: + /// * Offset: the offset within the file of this account + /// * StoredAccountInfoWithoutData: the account itself, without account data + /// /// Note that account data is not read/passed to the callback. pub fn scan_accounts_without_data( &self, - mut callback: impl for<'local> FnMut(StoredAccountInfoWithoutData<'local>), + mut callback: impl for<'local> FnMut(Offset, StoredAccountInfoWithoutData<'local>), ) { self.scan_stored_accounts_no_data(|stored_account| { + let offset = stored_account.offset(); let account = StoredAccountInfoWithoutData { pubkey: stored_account.pubkey(), lamports: stored_account.lamports(), @@ -995,16 +1001,24 @@ impl AppendVec { executable: stored_account.executable(), rent_epoch: stored_account.rent_epoch(), }; - callback(account); + callback(offset, account); }) } /// Iterate over all accounts and call `callback` with each account. /// + /// `callback` parameters: + /// * Offset: the offset within the file of this account + /// * StoredAccountInfo: the account itself, with account data + /// /// Prefer scan_accounts_without_data() when account data is not needed, /// as it can potentially read less and be faster. - pub fn scan_accounts(&self, mut callback: impl for<'local> FnMut(StoredAccountInfo<'local>)) { + pub fn scan_accounts( + &self, + mut callback: impl for<'local> FnMut(Offset, StoredAccountInfo<'local>), + ) { self.scan_accounts_stored_meta(|stored_account_meta| { + let offset = stored_account_meta.offset(); let account = StoredAccountInfo { pubkey: stored_account_meta.pubkey(), lamports: stored_account_meta.lamports(), @@ -1013,7 +1027,7 @@ impl AppendVec { executable: stored_account_meta.executable(), rent_epoch: stored_account_meta.rent_epoch(), }; - callback(account); + callback(offset, account); }) } diff --git a/accounts-db/src/tiered_storage.rs b/accounts-db/src/tiered_storage.rs index 71d542c6d3c..355b385caa2 100644 --- a/accounts-db/src/tiered_storage.rs +++ b/accounts-db/src/tiered_storage.rs @@ -384,7 +384,7 @@ mod tests { let mut max_pubkey = MIN_PUBKEY; reader - .scan_accounts(|stored_account| { + .scan_accounts(|_offset, stored_account| { if let Some(account) = expected_accounts_map.get(stored_account.pubkey()) { verify_test_account_with_footer( &stored_account, diff --git a/accounts-db/src/tiered_storage/hot.rs b/accounts-db/src/tiered_storage/hot.rs index 73337b9ee53..ad3004402dd 100644 --- a/accounts-db/src/tiered_storage/hot.rs +++ b/accounts-db/src/tiered_storage/hot.rs @@ -2,7 +2,7 @@ use { crate::{ - account_info::AccountInfo, + account_info::{AccountInfo, Offset}, account_storage::stored_account_info::StoredAccountInfo, accounts_file::{MatchAccountOwnerError, StoredAccountsInfo}, append_vec::{IndexInfo, IndexInfoInner}, @@ -639,10 +639,12 @@ impl HotStorageReader { /// Iterate over all accounts and call `callback` with each account. pub fn scan_accounts( &self, - mut callback: impl for<'local> FnMut(StoredAccountInfo<'local>), + mut callback: impl for<'local> FnMut(Offset, StoredAccountInfo<'local>), ) -> TieredStorageResult<()> { for i in 0..self.footer.account_entry_count { - self.get_stored_account_callback(IndexOffset(i), &mut callback)?; + self.get_stored_account_callback(IndexOffset(i), |account| { + callback(AccountInfo::reduced_offset_to_offset(i), account) + })?; } Ok(()) } @@ -1618,7 +1620,7 @@ mod tests { // verify everything let mut i = 0; hot_storage - .scan_accounts(|stored_account| { + .scan_accounts(|_offset, stored_account| { storable_accounts.account_default_if_zero_lamport(i, |account| { verify_test_account( &stored_account, diff --git a/accounts-db/src/tiered_storage/readable.rs b/accounts-db/src/tiered_storage/readable.rs index 3bd2c6d5b42..ffc47da8e04 100644 --- a/accounts-db/src/tiered_storage/readable.rs +++ b/accounts-db/src/tiered_storage/readable.rs @@ -1,5 +1,6 @@ use { crate::{ + account_info::Offset, account_storage::stored_account_info::{StoredAccountInfo, StoredAccountInfoWithoutData}, accounts_file::MatchAccountOwnerError, append_vec::IndexInfo, @@ -139,13 +140,17 @@ impl TieredStorageReader { /// Iterate over all accounts and call `callback` with each account. /// + /// `callback` parameters: + /// * Offset: the offset within the file of this account + /// * StoredAccountInfoWithoutData: the account itself, without account data + /// /// Note that account data is not read/passed to the callback. pub fn scan_accounts_without_data( &self, - mut callback: impl for<'local> FnMut(StoredAccountInfoWithoutData<'local>), + mut callback: impl for<'local> FnMut(Offset, StoredAccountInfoWithoutData<'local>), ) -> TieredStorageResult<()> { // Note, this should be reimplemented to not read account data - self.scan_accounts(|stored_account| { + self.scan_accounts(|offset, stored_account| { let account = StoredAccountInfoWithoutData { pubkey: stored_account.pubkey(), lamports: stored_account.lamports(), @@ -154,17 +159,21 @@ impl TieredStorageReader { executable: stored_account.executable(), rent_epoch: stored_account.rent_epoch(), }; - callback(account); + callback(offset, account); }) } /// Iterate over all accounts and call `callback` with each account. /// + /// `callback` parameters: + /// * Offset: the offset within the file of this account + /// * StoredAccountInfo: the account itself, with account data + /// /// Prefer scan_accounts_without_data() when account data is not needed, /// as it can potentially read less and be faster. pub fn scan_accounts( &self, - callback: impl for<'local> FnMut(StoredAccountInfo<'local>), + callback: impl for<'local> FnMut(Offset, StoredAccountInfo<'local>), ) -> TieredStorageResult<()> { match self { Self::Hot(hot) => hot.scan_accounts(callback), diff --git a/runtime/src/bank/accounts_lt_hash.rs b/runtime/src/bank/accounts_lt_hash.rs index ad87254cc00..75cd39578a4 100644 --- a/runtime/src/bank/accounts_lt_hash.rs +++ b/runtime/src/bank/accounts_lt_hash.rs @@ -922,7 +922,7 @@ mod tests { // get all the lt hashes for each version of all accounts let mut stored_accounts_map = HashMap::<_, Vec<_>>::new(); for storage in &storages { - storage.accounts.scan_accounts(|account| { + storage.accounts.scan_accounts(|_offset, account| { let pubkey = account.pubkey(); let account_lt_hash = AccountsDb::lt_hash_account(&account, pubkey); stored_accounts_map