diff --git a/Cargo.lock b/Cargo.lock index 7e0d167478637f..f4d10eef08d98f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -332,6 +332,12 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "allocator-api2" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -2565,6 +2571,10 @@ name = "hashbrown" version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +dependencies = [ + "ahash 0.8.10", + "allocator-api2", +] [[package]] name = "headers" @@ -3118,6 +3128,15 @@ dependencies = [ "winapi-build", ] +[[package]] +name = "lazy-lru" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81b33bc1276f3df38e938ed17bbb3d5c5eef758aa1a9997ec8388799ba3eef1" +dependencies = [ + "hashbrown 0.14.3", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -6338,6 +6357,7 @@ dependencies = [ "fs_extra", "futures 0.3.30", "itertools", + "lazy-lru", "lazy_static", "libc", "log", diff --git a/Cargo.toml b/Cargo.toml index 581ee2efeec769..6ab3c2de406fb5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -238,6 +238,7 @@ jsonrpc-derive = "18.0.0" jsonrpc-http-server = "18.0.0" jsonrpc-ipc-server = "18.0.0" jsonrpc-pubsub = "18.0.0" +lazy-lru = "0.1.2" lazy_static = "1.4.0" libc = "0.2.153" libloading = "0.7.4" diff --git a/core/src/repair/repair_response.rs b/core/src/repair/repair_response.rs index d0a02fca2b0176..0a82935e89892c 100644 --- a/core/src/repair/repair_response.rs +++ b/core/src/repair/repair_response.rs @@ -55,7 +55,7 @@ mod test { super::*, solana_ledger::{ shred::{Shred, ShredFlags}, - sigverify_shreds::verify_shred_cpu, + sigverify_shreds::{verify_shred_cpu, LruCache}, }, solana_sdk::{ packet::PacketFlags, @@ -64,11 +64,13 @@ mod test { std::{ collections::HashMap, net::{IpAddr, Ipv4Addr}, + sync::RwLock, }, }; fn run_test_sigverify_shred_cpu_repair(slot: Slot) { solana_logger::setup(); + let cache = RwLock::new(LruCache::new(/*capacity:*/ 128)); let mut shred = Shred::new_from_data( slot, 0xc0de, @@ -93,14 +95,14 @@ mod test { packet.meta_mut().flags |= PacketFlags::REPAIR; let leader_slots = HashMap::from([(slot, keypair.pubkey())]); - assert!(verify_shred_cpu(&packet, &leader_slots)); + assert!(verify_shred_cpu(&packet, &leader_slots, &cache)); let wrong_keypair = Keypair::new(); let leader_slots = HashMap::from([(slot, wrong_keypair.pubkey())]); - assert!(!verify_shred_cpu(&packet, &leader_slots)); + assert!(!verify_shred_cpu(&packet, &leader_slots, &cache)); let leader_slots = HashMap::new(); - assert!(!verify_shred_cpu(&packet, &leader_slots)); + assert!(!verify_shred_cpu(&packet, &leader_slots, &cache)); } #[test] diff --git a/ledger/Cargo.toml b/ledger/Cargo.toml index 7665428981ed82..5322176dd540be 100644 --- a/ledger/Cargo.toml +++ b/ledger/Cargo.toml @@ -21,6 +21,7 @@ dashmap = { workspace = true, features = ["rayon", "raw-api"] } fs_extra = { workspace = true } futures = { workspace = true } itertools = { workspace = true } +lazy-lru = { workspace = true } lazy_static = { workspace = true } libc = { workspace = true } log = { workspace = true } diff --git a/ledger/src/sigverify_shreds.rs b/ledger/src/sigverify_shreds.rs index f6d060d686757d..836da089103efa 100644 --- a/ledger/src/sigverify_shreds.rs +++ b/ledger/src/sigverify_shreds.rs @@ -1,6 +1,6 @@ #![allow(clippy::implicit_hasher)] use { - crate::shred::{self, SIZE_OF_MERKLE_ROOT}, + crate::shred::{self, SignedData, SIZE_OF_MERKLE_ROOT}, itertools::{izip, Itertools}, rayon::{prelude::*, ThreadPool}, sha2::{Digest, Sha512}, @@ -18,13 +18,25 @@ use { pubkey::Pubkey, signature::{Keypair, Signature, Signer}, }, - std::{collections::HashMap, iter::repeat, mem::size_of, ops::Range, sync::Arc}, + std::{ + collections::HashMap, + iter::repeat, + mem::size_of, + ops::Range, + sync::{Arc, RwLock}, + }, }; const SIGN_SHRED_GPU_MIN: usize = 256; +pub type LruCache = lazy_lru::LruCache<(Signature, Pubkey, /*merkle root:*/ Hash), ()>; + #[must_use] -pub fn verify_shred_cpu(packet: &Packet, slot_leaders: &HashMap) -> bool { +pub fn verify_shred_cpu( + packet: &Packet, + slot_leaders: &HashMap, + cache: &RwLock, +) -> bool { if packet.meta().discard() { return false; } @@ -45,13 +57,27 @@ pub fn verify_shred_cpu(packet: &Packet, slot_leaders: &HashMap) - let Some(data) = shred::layout::get_signed_data(shred) else { return false; }; - signature.verify(pubkey.as_ref(), data.as_ref()) + match data { + SignedData::Chunk(chunk) => signature.verify(pubkey.as_ref(), chunk), + SignedData::MerkleRoot(root) => { + let key = (signature, *pubkey, root); + if cache.read().unwrap().get(&key).is_some() { + true + } else if key.0.verify(key.1.as_ref(), key.2.as_ref()) { + cache.write().unwrap().put(key, ()); + true + } else { + false + } + } + } } fn verify_shreds_cpu( thread_pool: &ThreadPool, batches: &[PacketBatch], slot_leaders: &HashMap, + cache: &RwLock, ) -> Vec> { let packet_count = count_packets_in_batches(batches); debug!("CPU SHRED ECDSA for {}", packet_count); @@ -61,7 +87,7 @@ fn verify_shreds_cpu( .map(|batch| { batch .par_iter() - .map(|packet| u8::from(verify_shred_cpu(packet, slot_leaders))) + .map(|packet| u8::from(verify_shred_cpu(packet, slot_leaders, cache))) .collect() }) .collect() @@ -240,9 +266,10 @@ pub fn verify_shreds_gpu( batches: &[PacketBatch], slot_leaders: &HashMap, recycler_cache: &RecyclerCache, + cache: &RwLock, ) -> Vec> { let Some(api) = perf_libs::api() else { - return verify_shreds_cpu(thread_pool, batches, slot_leaders); + return verify_shreds_cpu(thread_pool, batches, slot_leaders, cache); }; let (pubkeys, pubkey_offsets) = slot_key_data_for_gpu(thread_pool, batches, slot_leaders, recycler_cache); @@ -479,6 +506,7 @@ mod tests { fn run_test_sigverify_shred_cpu(slot: Slot) { solana_logger::setup(); let mut packet = Packet::default(); + let cache = RwLock::new(LruCache::new(/*capacity:*/ 128)); let mut shred = Shred::new_from_data( slot, 0xc0de, @@ -497,14 +525,14 @@ mod tests { packet.meta_mut().size = shred.payload().len(); let leader_slots = HashMap::from([(slot, keypair.pubkey())]); - assert!(verify_shred_cpu(&packet, &leader_slots)); + assert!(verify_shred_cpu(&packet, &leader_slots, &cache)); let wrong_keypair = Keypair::new(); let leader_slots = HashMap::from([(slot, wrong_keypair.pubkey())]); - assert!(!verify_shred_cpu(&packet, &leader_slots)); + assert!(!verify_shred_cpu(&packet, &leader_slots, &cache)); let leader_slots = HashMap::new(); - assert!(!verify_shred_cpu(&packet, &leader_slots)); + assert!(!verify_shred_cpu(&packet, &leader_slots, &cache)); } #[test] @@ -515,6 +543,7 @@ mod tests { fn run_test_sigverify_shreds_cpu(thread_pool: &ThreadPool, slot: Slot) { solana_logger::setup(); let mut batches = [PacketBatch::default()]; + let cache = RwLock::new(LruCache::new(/*capacity:*/ 128)); let mut shred = Shred::new_from_data( slot, 0xc0de, @@ -532,21 +561,21 @@ mod tests { batches[0][0].meta_mut().size = shred.payload().len(); let leader_slots = HashMap::from([(slot, keypair.pubkey())]); - let rv = verify_shreds_cpu(thread_pool, &batches, &leader_slots); + let rv = verify_shreds_cpu(thread_pool, &batches, &leader_slots, &cache); assert_eq!(rv, vec![vec![1]]); let wrong_keypair = Keypair::new(); let leader_slots = HashMap::from([(slot, wrong_keypair.pubkey())]); - let rv = verify_shreds_cpu(thread_pool, &batches, &leader_slots); + let rv = verify_shreds_cpu(thread_pool, &batches, &leader_slots, &cache); assert_eq!(rv, vec![vec![0]]); let leader_slots = HashMap::new(); - let rv = verify_shreds_cpu(thread_pool, &batches, &leader_slots); + let rv = verify_shreds_cpu(thread_pool, &batches, &leader_slots, &cache); assert_eq!(rv, vec![vec![0]]); let leader_slots = HashMap::from([(slot, keypair.pubkey())]); batches[0][0].meta_mut().size = 0; - let rv = verify_shreds_cpu(thread_pool, &batches, &leader_slots); + let rv = verify_shreds_cpu(thread_pool, &batches, &leader_slots, &cache); assert_eq!(rv, vec![vec![0]]); } @@ -559,6 +588,7 @@ mod tests { fn run_test_sigverify_shreds_gpu(thread_pool: &ThreadPool, slot: Slot) { solana_logger::setup(); let recycler_cache = RecyclerCache::default(); + let cache = RwLock::new(LruCache::new(/*capacity:*/ 128)); let mut batches = [PacketBatch::default()]; let mut shred = Shred::new_from_data( @@ -579,7 +609,13 @@ mod tests { let leader_slots = HashMap::from([(std::u64::MAX, Pubkey::default()), (slot, keypair.pubkey())]); - let rv = verify_shreds_gpu(thread_pool, &batches, &leader_slots, &recycler_cache); + let rv = verify_shreds_gpu( + thread_pool, + &batches, + &leader_slots, + &recycler_cache, + &cache, + ); assert_eq!(rv, vec![vec![1]]); let wrong_keypair = Keypair::new(); @@ -587,17 +623,35 @@ mod tests { (std::u64::MAX, Pubkey::default()), (slot, wrong_keypair.pubkey()), ]); - let rv = verify_shreds_gpu(thread_pool, &batches, &leader_slots, &recycler_cache); + let rv = verify_shreds_gpu( + thread_pool, + &batches, + &leader_slots, + &recycler_cache, + &cache, + ); assert_eq!(rv, vec![vec![0]]); let leader_slots = HashMap::from([(std::u64::MAX, Pubkey::default())]); - let rv = verify_shreds_gpu(thread_pool, &batches, &leader_slots, &recycler_cache); + let rv = verify_shreds_gpu( + thread_pool, + &batches, + &leader_slots, + &recycler_cache, + &cache, + ); assert_eq!(rv, vec![vec![0]]); batches[0][0].meta_mut().size = 0; let leader_slots = HashMap::from([(std::u64::MAX, Pubkey::default()), (slot, keypair.pubkey())]); - let rv = verify_shreds_gpu(thread_pool, &batches, &leader_slots, &recycler_cache); + let rv = verify_shreds_gpu( + thread_pool, + &batches, + &leader_slots, + &recycler_cache, + &cache, + ); assert_eq!(rv, vec![vec![0]]); } @@ -610,6 +664,7 @@ mod tests { fn run_test_sigverify_shreds_sign_gpu(thread_pool: &ThreadPool, slot: Slot) { solana_logger::setup(); let recycler_cache = RecyclerCache::default(); + let cache = RwLock::new(LruCache::new(/*capacity:*/ 128)); let num_packets = 32; let num_batches = 100; @@ -635,7 +690,7 @@ mod tests { let pinned_keypair = Some(Arc::new(pinned_keypair)); let pubkeys = HashMap::from([(std::u64::MAX, Pubkey::default()), (slot, keypair.pubkey())]); //unsigned - let rv = verify_shreds_gpu(thread_pool, &batches, &pubkeys, &recycler_cache); + let rv = verify_shreds_gpu(thread_pool, &batches, &pubkeys, &recycler_cache, &cache); assert_eq!(rv, vec![vec![0; num_packets]; num_batches]); //signed sign_shreds_gpu( @@ -645,10 +700,10 @@ mod tests { &mut batches, &recycler_cache, ); - let rv = verify_shreds_cpu(thread_pool, &batches, &pubkeys); + let rv = verify_shreds_cpu(thread_pool, &batches, &pubkeys, &cache); assert_eq!(rv, vec![vec![1; num_packets]; num_batches]); - let rv = verify_shreds_gpu(thread_pool, &batches, &pubkeys, &recycler_cache); + let rv = verify_shreds_gpu(thread_pool, &batches, &pubkeys, &recycler_cache, &cache); assert_eq!(rv, vec![vec![1; num_packets]; num_batches]); } @@ -662,6 +717,7 @@ mod tests { solana_logger::setup(); let mut batches = [PacketBatch::default()]; + let cache = RwLock::new(LruCache::new(/*capacity:*/ 128)); let keypair = Keypair::new(); let shred = Shred::new_from_data( slot, @@ -679,11 +735,11 @@ mod tests { let pubkeys = HashMap::from([(slot, keypair.pubkey()), (std::u64::MAX, Pubkey::default())]); //unsigned - let rv = verify_shreds_cpu(thread_pool, &batches, &pubkeys); + let rv = verify_shreds_cpu(thread_pool, &batches, &pubkeys, &cache); assert_eq!(rv, vec![vec![0]]); //signed sign_shreds_cpu(thread_pool, &keypair, &mut batches); - let rv = verify_shreds_cpu(thread_pool, &batches, &pubkeys); + let rv = verify_shreds_cpu(thread_pool, &batches, &pubkeys, &cache); assert_eq!(rv, vec![vec![1]]); } @@ -799,6 +855,7 @@ mod tests { #[test] fn test_verify_shreds_fuzz() { let mut rng = rand::thread_rng(); + let cache = RwLock::new(LruCache::new(/*capacity:*/ 128)); let thread_pool = ThreadPoolBuilder::new().num_threads(3).build().unwrap(); let recycler_cache = RecyclerCache::default(); let keypairs = repeat_with(|| rng.gen_range(169_367_809..169_906_789)) @@ -813,7 +870,7 @@ mod tests { .collect(); let mut packets = make_packets(&mut rng, &shreds); assert_eq!( - verify_shreds_gpu(&thread_pool, &packets, &pubkeys, &recycler_cache), + verify_shreds_gpu(&thread_pool, &packets, &pubkeys, &recycler_cache, &cache), packets .iter() .map(|batch| vec![1u8; batch.len()]) @@ -836,7 +893,7 @@ mod tests { }) .collect(); assert_eq!( - verify_shreds_gpu(&thread_pool, &packets, &pubkeys, &recycler_cache), + verify_shreds_gpu(&thread_pool, &packets, &pubkeys, &recycler_cache, &cache), out ); } @@ -844,6 +901,7 @@ mod tests { #[test] fn test_sign_shreds_gpu() { let mut rng = rand::thread_rng(); + let cache = RwLock::new(LruCache::new(/*capacity:*/ 128)); let thread_pool = ThreadPoolBuilder::new().num_threads(3).build().unwrap(); let recycler_cache = RecyclerCache::default(); let shreds = { @@ -866,7 +924,7 @@ mod tests { let mut packets = make_packets(&mut rng, &shreds); // Assert that initially all signatrues are invalid. assert_eq!( - verify_shreds_gpu(&thread_pool, &packets, &pubkeys, &recycler_cache), + verify_shreds_gpu(&thread_pool, &packets, &pubkeys, &recycler_cache, &cache), packets .iter() .map(|batch| vec![0u8; batch.len()]) @@ -883,7 +941,7 @@ mod tests { &recycler_cache, ); assert_eq!( - verify_shreds_gpu(&thread_pool, &packets, &pubkeys, &recycler_cache), + verify_shreds_gpu(&thread_pool, &packets, &pubkeys, &recycler_cache, &cache), packets .iter() .map(|batch| vec![1u8; batch.len()]) diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index 417e8a388ecea6..7fff595121a59c 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -195,6 +195,12 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "allocator-api2" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -1334,7 +1340,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if 1.0.0", - "hashbrown 0.14.1", + "hashbrown 0.14.3", "lock_api", "once_cell", "parking_lot_core 0.9.8", @@ -2056,9 +2062,13 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.1" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +dependencies = [ + "ahash 0.8.10", + "allocator-api2", +] [[package]] name = "headers" @@ -2367,7 +2377,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4" dependencies = [ "equivalent", - "hashbrown 0.14.1", + "hashbrown 0.14.3", "rayon", ] @@ -2582,6 +2592,15 @@ dependencies = [ "winapi-build", ] +[[package]] +name = "lazy-lru" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81b33bc1276f3df38e938ed17bbb3d5c5eef758aa1a9997ec8388799ba3eef1" +dependencies = [ + "hashbrown 0.14.3", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -5207,6 +5226,7 @@ dependencies = [ "fs_extra", "futures 0.3.30", "itertools", + "lazy-lru", "lazy_static", "libc", "log", diff --git a/turbine/src/sigverify_shreds.rs b/turbine/src/sigverify_shreds.rs index 76088a8115a837..2349e289905b46 100644 --- a/turbine/src/sigverify_shreds.rs +++ b/turbine/src/sigverify_shreds.rs @@ -3,7 +3,9 @@ use { rayon::{prelude::*, ThreadPool, ThreadPoolBuilder}, solana_gossip::cluster_info::ClusterInfo, solana_ledger::{ - leader_schedule_cache::LeaderScheduleCache, shred, sigverify_shreds::verify_shreds_gpu, + leader_schedule_cache::LeaderScheduleCache, + shred, + sigverify_shreds::{verify_shreds_gpu, LruCache}, }, solana_perf::{self, deduper::Deduper, packet::PacketBatch, recycler_cache::RecyclerCache}, solana_rayon_threadlimit::get_thread_count, @@ -17,6 +19,9 @@ use { }, }; +// 34MB where each cache entry is 136 bytes. +const SIGVERIFY_LRU_CACHE_CAPACITY: usize = 1 << 18; + const DEDUPER_FALSE_POSITIVE_RATE: f64 = 0.001; const DEDUPER_NUM_BITS: u64 = 637_534_199; // 76MB const DEDUPER_RESET_CYCLE: Duration = Duration::from_secs(5 * 60); @@ -38,6 +43,7 @@ pub fn spawn_shred_sigverify( ) -> JoinHandle<()> { let recycler_cache = RecyclerCache::warmed(); let mut stats = ShredSigVerifyStats::new(Instant::now()); + let cache = RwLock::new(LruCache::new(SIGVERIFY_LRU_CACHE_CAPACITY)); let thread_pool = ThreadPoolBuilder::new() .num_threads(get_thread_count()) .thread_name(|i| format!("solSvrfyShred{i:02}")) @@ -62,6 +68,7 @@ pub fn spawn_shred_sigverify( &shred_fetch_receiver, &retransmit_sender, &verified_sender, + &cache, &mut stats, ) { Ok(()) => (), @@ -89,6 +96,7 @@ fn run_shred_sigverify( shred_fetch_receiver: &Receiver, retransmit_sender: &Sender>>, verified_sender: &Sender>, + cache: &RwLock, stats: &mut ShredSigVerifyStats, ) -> Result<(), Error> { const RECV_TIMEOUT: Duration = Duration::from_secs(1); @@ -121,6 +129,7 @@ fn run_shred_sigverify( leader_schedule_cache, recycler_cache, &mut packets, + cache, ); stats.num_discards_post += count_discards(&packets); // Exclude repair packets from retransmit. @@ -145,6 +154,7 @@ fn verify_packets( leader_schedule_cache: &LeaderScheduleCache, recycler_cache: &RecyclerCache, packets: &mut [PacketBatch], + cache: &RwLock, ) { let working_bank = bank_forks.read().unwrap().working_bank(); let leader_slots: HashMap = @@ -153,7 +163,7 @@ fn verify_packets( .filter_map(|(slot, pubkey)| Some((slot, pubkey?))) .chain(std::iter::once((Slot::MAX, Pubkey::default()))) .collect(); - let out = verify_shreds_gpu(thread_pool, packets, &leader_slots, recycler_cache); + let out = verify_shreds_gpu(thread_pool, packets, &leader_slots, recycler_cache, cache); solana_perf::sigverify::mark_disabled(packets, &out); } @@ -319,6 +329,7 @@ mod tests { batches[0][1].buffer_mut()[..shred.payload().len()].copy_from_slice(shred.payload()); batches[0][1].meta_mut().size = shred.payload().len(); + let cache = RwLock::new(LruCache::new(/*capacity:*/ 128)); let thread_pool = ThreadPoolBuilder::new().num_threads(3).build().unwrap(); verify_packets( &thread_pool, @@ -327,6 +338,7 @@ mod tests { &leader_schedule_cache, &RecyclerCache::warmed(), &mut batches, + &cache, ); assert!(!batches[0][0].meta().discard()); assert!(batches[0][1].meta().discard());