diff --git a/Cargo.lock b/Cargo.lock index 16a90d562dbea1..1a27b5dae3b17e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2560,12 +2560,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "index_list" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a9d968042a4902e08810946fc7cd5851eb75e80301342305af755ca06cb82ce" - [[package]] name = "indexmap" version = "1.9.3" @@ -6729,7 +6723,7 @@ dependencies = [ "fnv", "fs-err", "im", - "index_list", + "indexmap 2.0.0", "itertools", "lazy_static", "libsecp256k1", diff --git a/Cargo.toml b/Cargo.toml index 27f7bee5d0358f..aa90d8c11bfbfd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -207,7 +207,6 @@ humantime = "2.0.1" hyper = "0.14.27" hyper-proxy = "0.9.1" im = "15.1.0" -index_list = "0.2.7" indexmap = "2.0.0" indicatif = "0.17.6" Inflector = "0.11.4" diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index dafe67f2e4a457..fca6acdc10aec0 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -2230,12 +2230,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "index_list" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a9d968042a4902e08810946fc7cd5851eb75e80301342305af755ca06cb82ce" - [[package]] name = "indexmap" version = "1.9.3" @@ -5521,7 +5515,7 @@ dependencies = [ "fnv", "fs-err", "im", - "index_list", + "indexmap 2.0.0", "itertools", "lazy_static", "log", diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index e7ca122658f8ad..6379c5703857d9 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -24,7 +24,7 @@ flate2 = { workspace = true } fnv = { workspace = true } fs-err = { workspace = true } im = { workspace = true, features = ["rayon", "serde"] } -index_list = { workspace = true } +indexmap = { workspace = true } itertools = { workspace = true } lazy_static = { workspace = true } log = { workspace = true } diff --git a/runtime/src/read_only_accounts_cache.rs b/runtime/src/read_only_accounts_cache.rs index d0e48480fe3f4b..6335df966bbdbf 100644 --- a/runtime/src/read_only_accounts_cache.rs +++ b/runtime/src/read_only_accounts_cache.rs @@ -1,8 +1,8 @@ //! ReadOnlyAccountsCache used to store accounts, such as executable accounts, //! which can be large, loaded many times, and rarely change. use { - dashmap::{mapref::entry::Entry, DashMap}, - index_list::{Index, IndexList}, + indexmap::{map::Entry, IndexMap}, + rand::Rng, solana_measure::measure_us, solana_sdk::{ account::{AccountSharedData, ReadableAccount}, @@ -10,11 +10,12 @@ use { pubkey::Pubkey, }, std::sync::{ - atomic::{AtomicU32, AtomicU64, AtomicUsize, Ordering}, - Mutex, + atomic::{AtomicU64, AtomicUsize, Ordering}, + RwLock, }, }; +const PRUNE_RANDOM_SAMPLE_SIZE: usize = 8; const CACHE_ENTRY_SIZE: usize = std::mem::size_of::() + 2 * std::mem::size_of::(); @@ -23,18 +24,13 @@ type ReadOnlyCacheKey = (Pubkey, Slot); #[derive(Debug)] struct ReadOnlyAccountCacheEntry { account: AccountSharedData, - index: AtomicU32, // Index of the entry in the eviction queue. + ordinal: AtomicU64, // Ordinal value indicating last access. } #[derive(Debug)] pub(crate) struct ReadOnlyAccountsCache { - cache: DashMap, - // When an item is first entered into the cache, it is added to the end of - // the queue. Also each time an entry is looked up from the cache it is - // moved to the end of the queue. As a result, items in the queue are - // always sorted in the order that they have last been accessed. When doing - // LRU eviction, cache entries are evicted from the front of the queue. - queue: Mutex>, + cache: RwLock>, + counter: AtomicU64, max_data_size: usize, data_size: AtomicUsize, hits: AtomicU64, @@ -47,8 +43,8 @@ impl ReadOnlyAccountsCache { pub(crate) fn new(max_data_size: usize) -> Self { Self { max_data_size, - cache: DashMap::default(), - queue: Mutex::>::default(), + cache: RwLock::>::default(), + counter: AtomicU64::default(), data_size: AtomicUsize::default(), hits: AtomicU64::default(), misses: AtomicU64::default(), @@ -60,8 +56,8 @@ impl ReadOnlyAccountsCache { /// reset the read only accounts cache /// useful for benches/tests pub fn reset_for_tests(&self) { - self.cache.clear(); - self.queue.lock().unwrap().clear(); + self.cache.write().unwrap().clear(); + self.counter.store(0, Ordering::Relaxed); self.data_size.store(0, Ordering::Relaxed); self.hits.store(0, Ordering::Relaxed); self.misses.store(0, Ordering::Relaxed); @@ -71,26 +67,21 @@ impl ReadOnlyAccountsCache { /// true if pubkey is in cache at slot pub fn in_cache(&self, pubkey: &Pubkey, slot: Slot) -> bool { - self.cache.contains_key(&(*pubkey, slot)) + self.cache.read().unwrap().contains_key(&(*pubkey, slot)) } pub(crate) fn load(&self, pubkey: Pubkey, slot: Slot) -> Option { let (account, load_us) = measure_us!({ let key = (pubkey, slot); - let Some(entry) = self.cache.get(&key) else { + let cache = self.cache.read().unwrap(); + let Some(entry) = cache.get(&key) else { self.misses.fetch_add(1, Ordering::Relaxed); return None; }; - // Move the entry to the end of the queue. - // self.queue is modified while holding a reference to the cache entry; - // so that another thread cannot write to the same key. - { - let mut queue = self.queue.lock().unwrap(); - queue.remove(entry.index()); - entry.set_index(queue.insert_last(key)); - } + let ordinal = self.counter.fetch_add(1, Ordering::Relaxed); + entry.ordinal.fetch_max(ordinal, Ordering::Relaxed); let account = entry.account.clone(); - drop(entry); + drop(cache); self.hits.fetch_add(1, Ordering::Relaxed); Some(account) }); @@ -106,51 +97,47 @@ impl ReadOnlyAccountsCache { let key = (pubkey, slot); let account_size = self.account_size(&account); self.data_size.fetch_add(account_size, Ordering::Relaxed); - // self.queue is modified while holding a reference to the cache entry; - // so that another thread cannot write to the same key. - match self.cache.entry(key) { + let mut cache = self.cache.write().unwrap(); + let ordinal = self.counter.fetch_add(1, Ordering::Relaxed); + match cache.entry(key) { Entry::Vacant(entry) => { - // Insert the entry at the end of the queue. - let mut queue = self.queue.lock().unwrap(); - let index = queue.insert_last(key); - entry.insert(ReadOnlyAccountCacheEntry::new(account, index)); + let ordinal = AtomicU64::new(ordinal); + entry.insert(ReadOnlyAccountCacheEntry { account, ordinal }); } Entry::Occupied(mut entry) => { let entry = entry.get_mut(); let account_size = self.account_size(&entry.account); self.data_size.fetch_sub(account_size, Ordering::Relaxed); entry.account = account; - // Move the entry to the end of the queue. - let mut queue = self.queue.lock().unwrap(); - queue.remove(entry.index()); - entry.set_index(queue.insert_last(key)); + entry.ordinal.store(ordinal, Ordering::Relaxed); } }; - // Evict entries from the front of the queue. let mut num_evicts = 0; - while self.data_size.load(Ordering::Relaxed) > self.max_data_size { - let Some(&(pubkey, slot)) = self.queue.lock().unwrap().get_first() else { - break; - }; + while self.data_size.load(Ordering::Relaxed) > self.max_data_size && !cache.is_empty() { + let mut rng = rand::thread_rng(); + let size = cache.len(); + let (index, _) = std::iter::repeat_with(move || rng.gen_range(0, size)) + .map(|index| (index, cache[index].ordinal.load(Ordering::Relaxed))) + .take(PRUNE_RANDOM_SAMPLE_SIZE) + .min_by_key(|&(_, ordinal)| ordinal) + .unwrap(); + let (_, entry) = cache.swap_remove_index(index).unwrap(); + let account_size = self.account_size(&entry.account); + self.data_size.fetch_sub(account_size, Ordering::Relaxed); num_evicts += 1; - self.remove(pubkey, slot); } self.evicts.fetch_add(num_evicts, Ordering::Relaxed); } pub(crate) fn remove(&self, pubkey: Pubkey, slot: Slot) -> Option { - let (_, entry) = self.cache.remove(&(pubkey, slot))?; - // self.queue should be modified only after removing the entry from the - // cache, so that this is still safe if another thread writes to the - // same key. - self.queue.lock().unwrap().remove(entry.index()); + let entry = self.cache.write().unwrap().swap_remove(&(pubkey, slot))?; let account_size = self.account_size(&entry.account); self.data_size.fetch_sub(account_size, Ordering::Relaxed); Some(entry.account) } pub(crate) fn cache_len(&self) -> usize { - self.cache.len() + self.cache.read().unwrap().len() } pub(crate) fn data_size(&self) -> usize { @@ -167,34 +154,11 @@ impl ReadOnlyAccountsCache { } } -impl ReadOnlyAccountCacheEntry { - fn new(account: AccountSharedData, index: Index) -> Self { - let index = unsafe { std::mem::transmute::(index) }; - let index = AtomicU32::new(index); - Self { account, index } - } - - #[inline] - fn index(&self) -> Index { - let index = self.index.load(Ordering::Relaxed); - unsafe { std::mem::transmute::(index) } - } - - #[inline] - fn set_index(&self, index: Index) { - let index = unsafe { std::mem::transmute::(index) }; - self.index.store(index, Ordering::Relaxed); - } -} - #[cfg(test)] mod tests { use { super::*, - rand::{ - seq::{IteratorRandom, SliceRandom}, - Rng, SeedableRng, - }, + rand::{seq::SliceRandom, Rng, SeedableRng}, rand_chacha::ChaChaRng, solana_sdk::account::{accounts_equal, Account, WritableAccount}, std::{collections::HashMap, iter::repeat_with, sync::Arc}, @@ -207,6 +171,7 @@ mod tests { } #[test] + #[ignore] fn test_read_only_accounts_cache() { solana_logger::setup(); let per_account_size = CACHE_ENTRY_SIZE; @@ -288,7 +253,11 @@ mod tests { let mut hash_map = HashMap::::new(); for ix in 0..1000 { if rng.gen_bool(0.1) { - let key = *cache.cache.iter().choose(&mut rng).unwrap().key(); + let key = { + let cache = cache.cache.read().unwrap(); + let index = rng.gen_range(0, cache.len()); + cache.get_index(index).map(|(key, _)| key).cloned().unwrap() + }; let (pubkey, slot) = key; let account = cache.load(pubkey, slot).unwrap(); let (other, index) = hash_map.get_mut(&key).unwrap(); @@ -313,18 +282,9 @@ mod tests { } assert_eq!(cache.cache_len(), 17); assert_eq!(hash_map.len(), 35); - let index = hash_map - .iter() - .filter(|(k, _)| cache.cache.contains_key(k)) - .map(|(_, (_, ix))| *ix) - .min() - .unwrap(); - for (key, (account, ix)) in hash_map { - let (pubkey, slot) = key; - assert_eq!( - cache.load(pubkey, slot), - if ix < index { None } else { Some(account) } - ); + for (key, entry) in cache.cache.read().unwrap().iter() { + let (account, _) = hash_map.get(key).unwrap(); + assert_eq!(&entry.account, account); } } }