v2.3: Adds Offset to callback on scan_accounts() family (backport of #6744)

accounts-db/src/accounts_db.rs

@@ -2588,7 +2588,7 @@ impl AccountsDb {
                 return;
             }
             if let Some(storage) = self.storage.get_slot_storage_entry(slot) {
-                storage.accounts.scan_accounts(|account| {
+                storage.accounts.scan_accounts(|_offset, account| {
                     let pk = account.pubkey();
                     match pubkey_refcount.entry(*pk) {
                         dashmap::mapref::entry::Entry::Occupied(mut occupied_entry) => {
@@ -4487,12 +4487,12 @@ impl AccountsDb {
         self.scan_cache_storage_fallback(slot, cache_map_func, |retval, storage| {
             match scan_account_storage_data {
                 ScanAccountStorageData::NoData => {
-                    storage.scan_accounts_without_data(|account_without_data| {
+                    storage.scan_accounts_without_data(|_offset, account_without_data| {
                         storage_scan_func(retval, &account_without_data, None);
                     });
                 }
                 ScanAccountStorageData::DataRefForStorage => {
-                    storage.scan_accounts(|account| {
+                    storage.scan_accounts(|_offset, account| {
                         let account_without_data = StoredAccountInfoWithoutData::new_from(&account);
                         storage_scan_func(retval, &account_without_data, Some(account.data));
                     });
@@ -6384,7 +6384,7 @@ impl AccountsDb {
         let mut lt_hash = storages
             .par_iter()
             .fold(LtHash::identity, |mut accum, storage| {
-                storage.accounts.scan_accounts(|account| {
+                storage.accounts.scan_accounts(|_offset, account| {
                     let account_lt_hash = Self::lt_hash_account(&account, account.pubkey());
                     accum.mix_in(&account_lt_hash.0);
                 });
@@ -8081,7 +8081,7 @@ impl AccountsDb {
         };
         if secondary {
             // scan storage a second time to update the secondary index
-            storage.accounts.scan_accounts(|stored_account| {
+            storage.accounts.scan_accounts(|_offset, stored_account| {
                 self.accounts_index.update_secondary_indexes(
                     stored_account.pubkey(),
                     &stored_account,

accounts-db/src/accounts_db/scan_account_storage.rs

@@ -349,7 +349,7 @@ impl AccountsDb {
     where
         S: AppendVecScan,
     {
-        storage.accounts.scan_accounts(|account| {
+        storage.accounts.scan_accounts(|_offset, account| {
             if scanner.filter(account.pubkey()) {
                 scanner.found_account(&LoadedAccount::Stored(account))
             }
@@ -675,7 +675,7 @@ mod tests {
                     let slot = storage.slot();
                     let copied_storage = accounts_db.create_and_insert_store(slot, 10000, "test");
                     let mut all_accounts = Vec::default();
-                    storage.accounts.scan_accounts(|acct| {
+                    storage.accounts.scan_accounts(|_offset, acct| {
                         all_accounts.push((*acct.pubkey(), acct.to_account_shared_data()));
                     });
                     let accounts = all_accounts
@@ -709,7 +709,7 @@ mod tests {
                 let slot = storage.slot() + max_slot;
                 let copied_storage = accounts_db.create_and_insert_store(slot, 10000, "test");
                 let mut all_accounts = Vec::default();
-                storage.accounts.scan_accounts(|acct| {
+                storage.accounts.scan_accounts(|_offset, acct| {
                     all_accounts.push((*acct.pubkey(), acct.to_account_shared_data()));
                 });
                 let accounts = all_accounts

accounts-db/src/accounts_db/tests.rs

@@ -6655,7 +6655,7 @@ fn get_all_accounts_from_storages<'a>(
     storages
         .flat_map(|storage| {
             let mut vec = Vec::default();
-            storage.accounts.scan_accounts(|account| {
+            storage.accounts.scan_accounts(|_offset, account| {
                 vec.push((*account.pubkey(), account.to_account_shared_data()));
             });
             // make sure scan_pubkeys results match

accounts-db/src/accounts_file.rs

@@ -2,7 +2,7 @@
 use crate::append_vec::StoredAccountMeta;
 use {
     crate::{
-        account_info::AccountInfo,
+        account_info::{AccountInfo, Offset},
         account_storage::stored_account_info::{StoredAccountInfo, StoredAccountInfoWithoutData},
         accounts_db::AccountsFileId,
         accounts_update_notifier_interface::AccountForGeyser,
@@ -277,10 +277,14 @@ impl AccountsFile {
 
     /// Iterate over all accounts and call `callback` with each account.
     ///
+    /// `callback` parameters:
+    /// * Offset: the offset within the file of this account
+    /// * StoredAccountInfoWithoutData: the account itself, without account data
+    ///
     /// Note that account data is not read/passed to the callback.
     pub fn scan_accounts_without_data(
         &self,
-        callback: impl for<'local> FnMut(StoredAccountInfoWithoutData<'local>),
+        callback: impl for<'local> FnMut(Offset, StoredAccountInfoWithoutData<'local>),
     ) {
         match self {
             Self::AppendVec(av) => av.scan_accounts_without_data(callback),
@@ -294,9 +298,16 @@ impl AccountsFile {
 
     /// Iterate over all accounts and call `callback` with each account.
     ///
+    /// `callback` parameters:
+    /// * Offset: the offset within the file of this account
+    /// * StoredAccountInfo: the account itself, with account data
+    ///
     /// Prefer scan_accounts_without_data() when account data is not needed,
     /// as it can potentially read less and be faster.
-    pub fn scan_accounts(&self, callback: impl for<'local> FnMut(StoredAccountInfo<'local>)) {
+    pub fn scan_accounts(
+        &self,
+        callback: impl for<'local> FnMut(Offset, StoredAccountInfo<'local>),
+    ) {
         match self {
             Self::AppendVec(av) => av.scan_accounts(callback),
             Self::TieredStorage(ts) => {
@@ -330,7 +341,7 @@ impl AccountsFile {
         &self,
         mut callback: impl for<'local> FnMut(AccountForGeyser<'local>),
     ) {
-        self.scan_accounts(|account| {
+        self.scan_accounts(|_offset, account| {
             let account_for_geyser = AccountForGeyser {
                 pubkey: account.pubkey(),
                 lamports: account.lamports(),

accounts-db/src/ancient_append_vecs.rs

@@ -2079,7 +2079,7 @@ pub mod tests {
             shrink_in_progress
                 .new_storage()
                 .accounts
-                .scan_accounts(|_| {
+                .scan_accounts(|_, _| {
                     count += 1;
                 });
             assert_eq!(count, 1);
@@ -2240,7 +2240,7 @@ pub mod tests {
                 })
                 .unwrap();
             let mut count = 0;
-            storage.accounts.scan_accounts(|_| {
+            storage.accounts.scan_accounts(|_, _| {
                 count += 1;
             });
             assert_eq!(count, 2);
@@ -3179,14 +3179,11 @@ pub mod tests {
                             );
                             // make sure the single new append vec contains all the same accounts
                             let mut two = Vec::default();
-                            one.first()
-                                .unwrap()
-                                .1
-                                .new_storage()
-                                .accounts
-                                .scan_accounts(|meta| {
+                            one.first().unwrap().1.new_storage().accounts.scan_accounts(
+                                |_offset, meta| {
                                     two.push((*meta.pubkey(), meta.to_account_shared_data()));
-                                });
+                                },
+                            );
 
                             compare_all_accounts(&initial_accounts, &two[..]);
                         }

accounts-db/src/append_vec.rs

@@ -16,6 +16,7 @@ pub use meta::{AccountMeta, StoredMeta};
 use meta::{AccountMeta, StoredMeta};
 use {
     crate::{
+        account_info::Offset,
         account_storage::stored_account_info::{StoredAccountInfo, StoredAccountInfoWithoutData},
         accounts_file::{
             AccountsFileError, InternalsForArchive, MatchAccountOwnerError, Result, StorageAccess,
@@ -981,12 +982,17 @@ impl AppendVec {
 
     /// Iterate over all accounts and call `callback` with each account.
     ///
+    /// `callback` parameters:
+    /// * Offset: the offset within the file of this account
+    /// * StoredAccountInfoWithoutData: the account itself, without account data
+    ///
     /// Note that account data is not read/passed to the callback.
     pub fn scan_accounts_without_data(
         &self,
-        mut callback: impl for<'local> FnMut(StoredAccountInfoWithoutData<'local>),
+        mut callback: impl for<'local> FnMut(Offset, StoredAccountInfoWithoutData<'local>),
     ) {
         self.scan_stored_accounts_no_data(|stored_account| {
+            let offset = stored_account.offset();
             let account = StoredAccountInfoWithoutData {
                 pubkey: stored_account.pubkey(),
                 lamports: stored_account.lamports(),
@@ -995,16 +1001,24 @@ impl AppendVec {
                 executable: stored_account.executable(),
                 rent_epoch: stored_account.rent_epoch(),
             };
-            callback(account);
+            callback(offset, account);
         })
     }
 
     /// Iterate over all accounts and call `callback` with each account.
     ///
+    /// `callback` parameters:
+    /// * Offset: the offset within the file of this account
+    /// * StoredAccountInfo: the account itself, with account data
+    ///
     /// Prefer scan_accounts_without_data() when account data is not needed,
     /// as it can potentially read less and be faster.
-    pub fn scan_accounts(&self, mut callback: impl for<'local> FnMut(StoredAccountInfo<'local>)) {
+    pub fn scan_accounts(
+        &self,
+        mut callback: impl for<'local> FnMut(Offset, StoredAccountInfo<'local>),
+    ) {
         self.scan_accounts_stored_meta(|stored_account_meta| {
+            let offset = stored_account_meta.offset();
             let account = StoredAccountInfo {
                 pubkey: stored_account_meta.pubkey(),
                 lamports: stored_account_meta.lamports(),
@@ -1013,7 +1027,7 @@ impl AppendVec {
                 executable: stored_account_meta.executable(),
                 rent_epoch: stored_account_meta.rent_epoch(),
             };
-            callback(account);
+            callback(offset, account);
         })
     }
 

accounts-db/src/tiered_storage.rs

@@ -384,7 +384,7 @@ mod tests {
         let mut max_pubkey = MIN_PUBKEY;
 
         reader
-            .scan_accounts(|stored_account| {
+            .scan_accounts(|_offset, stored_account| {
                 if let Some(account) = expected_accounts_map.get(stored_account.pubkey()) {
                     verify_test_account_with_footer(
                         &stored_account,

accounts-db/src/tiered_storage/hot.rs

@@ -2,7 +2,7 @@
 
 use {
     crate::{
-        account_info::AccountInfo,
+        account_info::{AccountInfo, Offset},
         account_storage::stored_account_info::StoredAccountInfo,
         accounts_file::{MatchAccountOwnerError, StoredAccountsInfo},
         append_vec::{IndexInfo, IndexInfoInner},
@@ -639,10 +639,12 @@ impl HotStorageReader {
     /// Iterate over all accounts and call `callback` with each account.
     pub fn scan_accounts(
         &self,
-        mut callback: impl for<'local> FnMut(StoredAccountInfo<'local>),
+        mut callback: impl for<'local> FnMut(Offset, StoredAccountInfo<'local>),
     ) -> TieredStorageResult<()> {
         for i in 0..self.footer.account_entry_count {
-            self.get_stored_account_callback(IndexOffset(i), &mut callback)?;
+            self.get_stored_account_callback(IndexOffset(i), |account| {
+                callback(AccountInfo::reduced_offset_to_offset(i), account)
+            })?;
         }
         Ok(())
     }
@@ -1618,7 +1620,7 @@ mod tests {
         // verify everything
         let mut i = 0;
         hot_storage
-            .scan_accounts(|stored_account| {
+            .scan_accounts(|_offset, stored_account| {
                 storable_accounts.account_default_if_zero_lamport(i, |account| {
                     verify_test_account(
                         &stored_account,

accounts-db/src/tiered_storage/readable.rs

@@ -1,5 +1,6 @@
 use {
     crate::{
+        account_info::Offset,
         account_storage::stored_account_info::{StoredAccountInfo, StoredAccountInfoWithoutData},
         accounts_file::MatchAccountOwnerError,
         append_vec::IndexInfo,
@@ -139,13 +140,17 @@ impl TieredStorageReader {
 
     /// Iterate over all accounts and call `callback` with each account.
     ///
+    /// `callback` parameters:
+    /// * Offset: the offset within the file of this account
+    /// * StoredAccountInfoWithoutData: the account itself, without account data
+    ///
     /// Note that account data is not read/passed to the callback.
     pub fn scan_accounts_without_data(
         &self,
-        mut callback: impl for<'local> FnMut(StoredAccountInfoWithoutData<'local>),
+        mut callback: impl for<'local> FnMut(Offset, StoredAccountInfoWithoutData<'local>),
     ) -> TieredStorageResult<()> {
         // Note, this should be reimplemented to not read account data
-        self.scan_accounts(|stored_account| {
+        self.scan_accounts(|offset, stored_account| {
             let account = StoredAccountInfoWithoutData {
                 pubkey: stored_account.pubkey(),
                 lamports: stored_account.lamports(),
@@ -154,17 +159,21 @@ impl TieredStorageReader {
                 executable: stored_account.executable(),
                 rent_epoch: stored_account.rent_epoch(),
             };
-            callback(account);
+            callback(offset, account);
         })
     }
 
     /// Iterate over all accounts and call `callback` with each account.
     ///
+    /// `callback` parameters:
+    /// * Offset: the offset within the file of this account
+    /// * StoredAccountInfo: the account itself, with account data
+    ///
     /// Prefer scan_accounts_without_data() when account data is not needed,
     /// as it can potentially read less and be faster.
     pub fn scan_accounts(
         &self,
-        callback: impl for<'local> FnMut(StoredAccountInfo<'local>),
+        callback: impl for<'local> FnMut(Offset, StoredAccountInfo<'local>),
     ) -> TieredStorageResult<()> {
         match self {
             Self::Hot(hot) => hot.scan_accounts(callback),

runtime/src/bank/accounts_lt_hash.rs

@@ -922,7 +922,7 @@ mod tests {
         // get all the lt hashes for each version of all accounts
         let mut stored_accounts_map = HashMap::<_, Vec<_>>::new();
         for storage in &storages {
-            storage.accounts.scan_accounts(|account| {
+            storage.accounts.scan_accounts(|_offset, account| {
                 let pubkey = account.pubkey();
                 let account_lt_hash = AccountsDb::lt_hash_account(&account, pubkey);
                 stored_accounts_map