Refactor Accounts Index and AccountsIndexIterator

accounts-db/src/accounts_db.rs

@@ -8527,12 +8527,9 @@ impl AccountsDb {
         let full_pubkey_range = Pubkey::from([0; 32])..=Pubkey::from([0xff; 32]);
 
         self.accounts_index.account_maps.iter().for_each(|map| {
-            for (pubkey, account_entry) in map.items(&full_pubkey_range) {
-                info!("  key: {} ref_count: {}", pubkey, account_entry.ref_count(),);
-                info!(
-                    "      slots: {:?}",
-                    *account_entry.slot_list.read().unwrap()
-                );
+            for (pubkey, slot_list) in map.items(&full_pubkey_range) {
+                info!("  key: {}", pubkey);
+                info!("      slots: {:?}", slot_list);
             }
         });
     }

accounts-db/src/accounts_index.rs

@@ -730,21 +730,27 @@ impl<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>> AccountsIndex<T, U> {
         for pubkey_list in self.iter(range.as_ref(), returns_items) {
             iterator_timer.stop();
             iterator_elapsed += iterator_timer.as_us();
-            for (pubkey, list) in pubkey_list {
+            for pubkey in pubkey_list {
                 num_keys_iterated += 1;
-                let mut read_lock_timer = Measure::start("read_lock");
-                let list_r = &list.slot_list.read().unwrap();
-                read_lock_timer.stop();
-                read_lock_elapsed += read_lock_timer.as_us();
-                let mut latest_slot_timer = Measure::start("latest_slot");
-                if let Some(index) = self.latest_slot(Some(ancestors), list_r, max_root) {
-                    latest_slot_timer.stop();
-                    latest_slot_elapsed += latest_slot_timer.as_us();
-                    let mut load_account_timer = Measure::start("load_account");
-                    func(&pubkey, (&list_r[index].1, list_r[index].0));
-                    load_account_timer.stop();
-                    load_account_elapsed += load_account_timer.as_us();
-                }
+                self.get_and_then(&pubkey, |entry| {
+                    if let Some(list) = entry {
+                        let mut read_lock_timer = Measure::start("read_lock");
+                        let list_r = &list.slot_list.read().unwrap();
+                        read_lock_timer.stop();
+                        read_lock_elapsed += read_lock_timer.as_us();
+                        let mut latest_slot_timer = Measure::start("latest_slot");
+                        if let Some(index) = self.latest_slot(Some(ancestors), list_r, max_root) {
+                            latest_slot_timer.stop();
+                            latest_slot_elapsed += latest_slot_timer.as_us();
+                            let mut load_account_timer = Measure::start("load_account");
+                            func(&pubkey, (&list_r[index].1, list_r[index].0));
+                            load_account_timer.stop();
+                            load_account_elapsed += load_account_timer.as_us();
+                        }
+                    }
+                    let add_to_in_mem_cache = false;
+                    (add_to_in_mem_cache, ())
+                });
                 if config.is_aborted() {
                     return;
                 }

accounts-db/src/accounts_index/in_mem_accounts_index.rs

@@ -250,7 +250,7 @@ impl<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>> InMemAccountsIndex<T,
         }
     }
 
-    pub fn items<R>(&self, range: &R) -> Vec<(Pubkey, Arc<AccountMapEntry<T>>)>
+    pub fn items<R>(&self, range: &R) -> Vec<(Pubkey, SlotList<T>)>
     where
         R: RangeBounds<Pubkey> + std::fmt::Debug,
     {
@@ -288,15 +288,15 @@ impl<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>> InMemAccountsIndex<T,
             .unwrap()
             .iter()
             .filter(|&(k, _v)| range.contains(k))
-            .map(|(k, v)| (*k, Arc::clone(v)))
+            .map(|(k, v)| (*k, v.slot_list.read().unwrap().clone()))
             .collect();
         self.hold_range_in_memory(range, false);
         Self::update_stat(&self.stats().items, 1);
         Self::update_time_stat(&self.stats().items_us, m);
         result
     }
 
-    // only called in debug code paths
+    /// return all keys in this bin
     pub fn keys(&self) -> Vec<Pubkey> {
         Self::update_stat(&self.stats().keys, 1);
         // easiest implementation is to load everything from disk into cache and return the keys

accounts-db/src/accounts_index/iter.rs

@@ -1,8 +1,5 @@
 use {
-    super::{
-        account_map_entry::AccountMapEntry, in_mem_accounts_index::InMemAccountsIndex,
-        AccountsIndex, DiskIndexValue, IndexValue,
-    },
+    super::{in_mem_accounts_index::InMemAccountsIndex, AccountsIndex, DiskIndexValue, IndexValue},
     solana_pubkey::Pubkey,
     std::{
         ops::{Bound, RangeBounds},
@@ -18,7 +15,7 @@ pub struct AccountsIndexIterator<'a, T: IndexValue, U: DiskIndexValue + From<T>
     end_bound: Bound<&'a Pubkey>,
     start_bin: usize,
     end_bin_inclusive: usize,
-    items: Vec<(Pubkey, Arc<AccountMapEntry<T>>)>,
+    items: Vec<Pubkey>,
     returns_items: AccountsIndexIteratorReturnsItems,
 }
 
@@ -61,18 +58,23 @@ impl<'a, T: IndexValue, U: DiskIndexValue + From<T> + Into<T>> AccountsIndexIter
 impl<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>> Iterator
     for AccountsIndexIterator<'_, T, U>
 {
-    type Item = Vec<(Pubkey, Arc<AccountMapEntry<T>>)>;
+    type Item = Vec<Pubkey>;
     fn next(&mut self) -> Option<Self::Item> {
+        let range = (self.start_bound, self.end_bound);
         while self.items.len() < ITER_BATCH_SIZE {
             if self.start_bin > self.end_bin_inclusive {
                 break;
             }
 
             let bin = self.start_bin;
             let map = &self.account_maps[bin];
-            let mut items = map.items(&(self.start_bound, self.end_bound));
+            let mut items = map
+                .keys()
+                .into_iter()
+                .filter(|k| range.contains(&k))
+                .collect::<Vec<_>>();
             if self.returns_items == AccountsIndexIteratorReturnsItems::Sorted {
-                items.sort_unstable_by(|a, b| a.0.cmp(&b.0));
+                items.sort_unstable();
             }
             self.items.append(&mut items);
             self.start_bin += 1;
@@ -141,7 +143,7 @@ mod tests {
             let x = iter.next().unwrap();
             assert_eq!(x.len(), 2 * ITER_BATCH_SIZE);
             assert_eq!(
-                x.is_sorted_by(|a, b| a.0 < b.0),
+                x.is_sorted(),
                 returns_items == AccountsIndexIteratorReturnsItems::Sorted
             );
             assert_eq!(iter.items.len(), 0); // should be empty.

accounts-db/src/append_vec.rs

@@ -169,7 +169,6 @@ enum AppendVecFileBacking {
     /// A file-backed block of memory that is used to store the data for each appended item.
     Mmap(MmapMut),
     /// This was opened as a read only file
-    #[cfg_attr(not(unix), allow(dead_code))]
     File(File),
 }
 
@@ -368,13 +367,7 @@ impl AppendVec {
 
     /// when we can use file i/o as opposed to mmap, this is the trigger to tell us
     /// that no more appending will occur and we can close the initial mmap.
-    #[cfg_attr(not(unix), allow(dead_code))]
     pub(crate) fn reopen_as_readonly(&self) -> Option<Self> {
-        #[cfg(not(unix))]
-        // must open as mmmap on non-unix
-        return None;
-
-        #[cfg(unix)]
         match &self.backing {
             AppendVecFileBacking::File(_file) => {
                 // already a file, so already read-only
@@ -477,7 +470,6 @@ impl AppendVec {
     }
 
     /// Creates an appendvec from file without performing sanitize checks or counting the number of accounts
-    #[cfg_attr(not(unix), allow(unused_variables))]
     pub fn new_from_file_unchecked(
         path: impl Into<PathBuf>,
         current_len: usize,
@@ -493,8 +485,6 @@ impl AppendVec {
             .create(false)
             .open(&path)?;
 
-        #[cfg(unix)]
-        // we must use mmap on non-linux
         if storage_access == StorageAccess::File {
             APPEND_VEC_STATS.files_open.fetch_add(1, Ordering::Relaxed);