From a29b9d1ba8b293f87f7b005aed7ad6032b105e2d Mon Sep 17 00:00:00 2001 From: behzad nouri Date: Tue, 12 Mar 2024 12:07:22 -0500 Subject: [PATCH] caches Merkle shreds signature verifications Merkle shreds sign the Merkle root of the erasure batch, so all shreds within the same erasure batch have the same signature. The commit improves shreds signature verification by adding an LRU cache. --- Cargo.lock | 20 ++++++ Cargo.toml | 1 + core/src/repair/repair_response.rs | 10 +-- ledger/Cargo.toml | 1 + ledger/src/sigverify_shreds.rs | 110 ++++++++++++++++++++++------- programs/sbf/Cargo.lock | 28 ++++++-- turbine/src/sigverify_shreds.rs | 16 ++++- 7 files changed, 150 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7e0d167478637f..f4d10eef08d98f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -332,6 +332,12 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "allocator-api2" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -2565,6 +2571,10 @@ name = "hashbrown" version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +dependencies = [ + "ahash 0.8.10", + "allocator-api2", +] [[package]] name = "headers" @@ -3118,6 +3128,15 @@ dependencies = [ "winapi-build", ] +[[package]] +name = "lazy-lru" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81b33bc1276f3df38e938ed17bbb3d5c5eef758aa1a9997ec8388799ba3eef1" +dependencies = [ + "hashbrown 0.14.3", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -6338,6 +6357,7 @@ dependencies = [ "fs_extra", "futures 0.3.30", "itertools", + "lazy-lru", "lazy_static", "libc", "log", diff --git a/Cargo.toml b/Cargo.toml index 581ee2efeec769..6ab3c2de406fb5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -238,6 +238,7 @@ jsonrpc-derive = "18.0.0" jsonrpc-http-server = "18.0.0" jsonrpc-ipc-server = "18.0.0" jsonrpc-pubsub = "18.0.0" +lazy-lru = "0.1.2" lazy_static = "1.4.0" libc = "0.2.153" libloading = "0.7.4" diff --git a/core/src/repair/repair_response.rs b/core/src/repair/repair_response.rs index d0a02fca2b0176..0a82935e89892c 100644 --- a/core/src/repair/repair_response.rs +++ b/core/src/repair/repair_response.rs @@ -55,7 +55,7 @@ mod test { super::*, solana_ledger::{ shred::{Shred, ShredFlags}, - sigverify_shreds::verify_shred_cpu, + sigverify_shreds::{verify_shred_cpu, LruCache}, }, solana_sdk::{ packet::PacketFlags, @@ -64,11 +64,13 @@ mod test { std::{ collections::HashMap, net::{IpAddr, Ipv4Addr}, + sync::RwLock, }, }; fn run_test_sigverify_shred_cpu_repair(slot: Slot) { solana_logger::setup(); + let cache = RwLock::new(LruCache::new(/*capacity:*/ 128)); let mut shred = Shred::new_from_data( slot, 0xc0de, @@ -93,14 +95,14 @@ mod test { packet.meta_mut().flags |= PacketFlags::REPAIR; let leader_slots = HashMap::from([(slot, keypair.pubkey())]); - assert!(verify_shred_cpu(&packet, &leader_slots)); + assert!(verify_shred_cpu(&packet, &leader_slots, &cache)); let wrong_keypair = Keypair::new(); let leader_slots = HashMap::from([(slot, wrong_keypair.pubkey())]); - assert!(!verify_shred_cpu(&packet, &leader_slots)); + assert!(!verify_shred_cpu(&packet, &leader_slots, &cache)); let leader_slots = HashMap::new(); - assert!(!verify_shred_cpu(&packet, &leader_slots)); + assert!(!verify_shred_cpu(&packet, &leader_slots, &cache)); } #[test] diff --git a/ledger/Cargo.toml b/ledger/Cargo.toml index 7665428981ed82..5322176dd540be 100644 --- a/ledger/Cargo.toml +++ b/ledger/Cargo.toml @@ -21,6 +21,7 @@ dashmap = { workspace = true, features = ["rayon", "raw-api"] } fs_extra = { workspace = true } futures = { workspace = true } itertools = { workspace = true } +lazy-lru = { workspace = true } lazy_static = { workspace = true } libc = { workspace = true } log = { workspace = true } diff --git a/ledger/src/sigverify_shreds.rs b/ledger/src/sigverify_shreds.rs index f6d060d686757d..836da089103efa 100644 --- a/ledger/src/sigverify_shreds.rs +++ b/ledger/src/sigverify_shreds.rs @@ -1,6 +1,6 @@ #![allow(clippy::implicit_hasher)] use { - crate::shred::{self, SIZE_OF_MERKLE_ROOT}, + crate::shred::{self, SignedData, SIZE_OF_MERKLE_ROOT}, itertools::{izip, Itertools}, rayon::{prelude::*, ThreadPool}, sha2::{Digest, Sha512}, @@ -18,13 +18,25 @@ use { pubkey::Pubkey, signature::{Keypair, Signature, Signer}, }, - std::{collections::HashMap, iter::repeat, mem::size_of, ops::Range, sync::Arc}, + std::{ + collections::HashMap, + iter::repeat, + mem::size_of, + ops::Range, + sync::{Arc, RwLock}, + }, }; const SIGN_SHRED_GPU_MIN: usize = 256; +pub type LruCache = lazy_lru::LruCache<(Signature, Pubkey, /*merkle root:*/ Hash), ()>; + #[must_use] -pub fn verify_shred_cpu(packet: &Packet, slot_leaders: &HashMap) -> bool { +pub fn verify_shred_cpu( + packet: &Packet, + slot_leaders: &HashMap, + cache: &RwLock, +) -> bool { if packet.meta().discard() { return false; } @@ -45,13 +57,27 @@ pub fn verify_shred_cpu(packet: &Packet, slot_leaders: &HashMap) - let Some(data) = shred::layout::get_signed_data(shred) else { return false; }; - signature.verify(pubkey.as_ref(), data.as_ref()) + match data { + SignedData::Chunk(chunk) => signature.verify(pubkey.as_ref(), chunk), + SignedData::MerkleRoot(root) => { + let key = (signature, *pubkey, root); + if cache.read().unwrap().get(&key).is_some() { + true + } else if key.0.verify(key.1.as_ref(), key.2.as_ref()) { + cache.write().unwrap().put(key, ()); + true + } else { + false + } + } + } } fn verify_shreds_cpu( thread_pool: &ThreadPool, batches: &[PacketBatch], slot_leaders: &HashMap, + cache: &RwLock, ) -> Vec> { let packet_count = count_packets_in_batches(batches); debug!("CPU SHRED ECDSA for {}", packet_count); @@ -61,7 +87,7 @@ fn verify_shreds_cpu( .map(|batch| { batch .par_iter() - .map(|packet| u8::from(verify_shred_cpu(packet, slot_leaders))) + .map(|packet| u8::from(verify_shred_cpu(packet, slot_leaders, cache))) .collect() }) .collect() @@ -240,9 +266,10 @@ pub fn verify_shreds_gpu( batches: &[PacketBatch], slot_leaders: &HashMap, recycler_cache: &RecyclerCache, + cache: &RwLock, ) -> Vec> { let Some(api) = perf_libs::api() else { - return verify_shreds_cpu(thread_pool, batches, slot_leaders); + return verify_shreds_cpu(thread_pool, batches, slot_leaders, cache); }; let (pubkeys, pubkey_offsets) = slot_key_data_for_gpu(thread_pool, batches, slot_leaders, recycler_cache); @@ -479,6 +506,7 @@ mod tests { fn run_test_sigverify_shred_cpu(slot: Slot) { solana_logger::setup(); let mut packet = Packet::default(); + let cache = RwLock::new(LruCache::new(/*capacity:*/ 128)); let mut shred = Shred::new_from_data( slot, 0xc0de, @@ -497,14 +525,14 @@ mod tests { packet.meta_mut().size = shred.payload().len(); let leader_slots = HashMap::from([(slot, keypair.pubkey())]); - assert!(verify_shred_cpu(&packet, &leader_slots)); + assert!(verify_shred_cpu(&packet, &leader_slots, &cache)); let wrong_keypair = Keypair::new(); let leader_slots = HashMap::from([(slot, wrong_keypair.pubkey())]); - assert!(!verify_shred_cpu(&packet, &leader_slots)); + assert!(!verify_shred_cpu(&packet, &leader_slots, &cache)); let leader_slots = HashMap::new(); - assert!(!verify_shred_cpu(&packet, &leader_slots)); + assert!(!verify_shred_cpu(&packet, &leader_slots, &cache)); } #[test] @@ -515,6 +543,7 @@ mod tests { fn run_test_sigverify_shreds_cpu(thread_pool: &ThreadPool, slot: Slot) { solana_logger::setup(); let mut batches = [PacketBatch::default()]; + let cache = RwLock::new(LruCache::new(/*capacity:*/ 128)); let mut shred = Shred::new_from_data( slot, 0xc0de, @@ -532,21 +561,21 @@ mod tests { batches[0][0].meta_mut().size = shred.payload().len(); let leader_slots = HashMap::from([(slot, keypair.pubkey())]); - let rv = verify_shreds_cpu(thread_pool, &batches, &leader_slots); + let rv = verify_shreds_cpu(thread_pool, &batches, &leader_slots, &cache); assert_eq!(rv, vec![vec![1]]); let wrong_keypair = Keypair::new(); let leader_slots = HashMap::from([(slot, wrong_keypair.pubkey())]); - let rv = verify_shreds_cpu(thread_pool, &batches, &leader_slots); + let rv = verify_shreds_cpu(thread_pool, &batches, &leader_slots, &cache); assert_eq!(rv, vec![vec![0]]); let leader_slots = HashMap::new(); - let rv = verify_shreds_cpu(thread_pool, &batches, &leader_slots); + let rv = verify_shreds_cpu(thread_pool, &batches, &leader_slots, &cache); assert_eq!(rv, vec![vec![0]]); let leader_slots = HashMap::from([(slot, keypair.pubkey())]); batches[0][0].meta_mut().size = 0; - let rv = verify_shreds_cpu(thread_pool, &batches, &leader_slots); + let rv = verify_shreds_cpu(thread_pool, &batches, &leader_slots, &cache); assert_eq!(rv, vec![vec![0]]); } @@ -559,6 +588,7 @@ mod tests { fn run_test_sigverify_shreds_gpu(thread_pool: &ThreadPool, slot: Slot) { solana_logger::setup(); let recycler_cache = RecyclerCache::default(); + let cache = RwLock::new(LruCache::new(/*capacity:*/ 128)); let mut batches = [PacketBatch::default()]; let mut shred = Shred::new_from_data( @@ -579,7 +609,13 @@ mod tests { let leader_slots = HashMap::from([(std::u64::MAX, Pubkey::default()), (slot, keypair.pubkey())]); - let rv = verify_shreds_gpu(thread_pool, &batches, &leader_slots, &recycler_cache); + let rv = verify_shreds_gpu( + thread_pool, + &batches, + &leader_slots, + &recycler_cache, + &cache, + ); assert_eq!(rv, vec![vec![1]]); let wrong_keypair = Keypair::new(); @@ -587,17 +623,35 @@ mod tests { (std::u64::MAX, Pubkey::default()), (slot, wrong_keypair.pubkey()), ]); - let rv = verify_shreds_gpu(thread_pool, &batches, &leader_slots, &recycler_cache); + let rv = verify_shreds_gpu( + thread_pool, + &batches, + &leader_slots, + &recycler_cache, + &cache, + ); assert_eq!(rv, vec![vec![0]]); let leader_slots = HashMap::from([(std::u64::MAX, Pubkey::default())]); - let rv = verify_shreds_gpu(thread_pool, &batches, &leader_slots, &recycler_cache); + let rv = verify_shreds_gpu( + thread_pool, + &batches, + &leader_slots, + &recycler_cache, + &cache, + ); assert_eq!(rv, vec![vec![0]]); batches[0][0].meta_mut().size = 0; let leader_slots = HashMap::from([(std::u64::MAX, Pubkey::default()), (slot, keypair.pubkey())]); - let rv = verify_shreds_gpu(thread_pool, &batches, &leader_slots, &recycler_cache); + let rv = verify_shreds_gpu( + thread_pool, + &batches, + &leader_slots, + &recycler_cache, + &cache, + ); assert_eq!(rv, vec![vec![0]]); } @@ -610,6 +664,7 @@ mod tests { fn run_test_sigverify_shreds_sign_gpu(thread_pool: &ThreadPool, slot: Slot) { solana_logger::setup(); let recycler_cache = RecyclerCache::default(); + let cache = RwLock::new(LruCache::new(/*capacity:*/ 128)); let num_packets = 32; let num_batches = 100; @@ -635,7 +690,7 @@ mod tests { let pinned_keypair = Some(Arc::new(pinned_keypair)); let pubkeys = HashMap::from([(std::u64::MAX, Pubkey::default()), (slot, keypair.pubkey())]); //unsigned - let rv = verify_shreds_gpu(thread_pool, &batches, &pubkeys, &recycler_cache); + let rv = verify_shreds_gpu(thread_pool, &batches, &pubkeys, &recycler_cache, &cache); assert_eq!(rv, vec![vec![0; num_packets]; num_batches]); //signed sign_shreds_gpu( @@ -645,10 +700,10 @@ mod tests { &mut batches, &recycler_cache, ); - let rv = verify_shreds_cpu(thread_pool, &batches, &pubkeys); + let rv = verify_shreds_cpu(thread_pool, &batches, &pubkeys, &cache); assert_eq!(rv, vec![vec![1; num_packets]; num_batches]); - let rv = verify_shreds_gpu(thread_pool, &batches, &pubkeys, &recycler_cache); + let rv = verify_shreds_gpu(thread_pool, &batches, &pubkeys, &recycler_cache, &cache); assert_eq!(rv, vec![vec![1; num_packets]; num_batches]); } @@ -662,6 +717,7 @@ mod tests { solana_logger::setup(); let mut batches = [PacketBatch::default()]; + let cache = RwLock::new(LruCache::new(/*capacity:*/ 128)); let keypair = Keypair::new(); let shred = Shred::new_from_data( slot, @@ -679,11 +735,11 @@ mod tests { let pubkeys = HashMap::from([(slot, keypair.pubkey()), (std::u64::MAX, Pubkey::default())]); //unsigned - let rv = verify_shreds_cpu(thread_pool, &batches, &pubkeys); + let rv = verify_shreds_cpu(thread_pool, &batches, &pubkeys, &cache); assert_eq!(rv, vec![vec![0]]); //signed sign_shreds_cpu(thread_pool, &keypair, &mut batches); - let rv = verify_shreds_cpu(thread_pool, &batches, &pubkeys); + let rv = verify_shreds_cpu(thread_pool, &batches, &pubkeys, &cache); assert_eq!(rv, vec![vec![1]]); } @@ -799,6 +855,7 @@ mod tests { #[test] fn test_verify_shreds_fuzz() { let mut rng = rand::thread_rng(); + let cache = RwLock::new(LruCache::new(/*capacity:*/ 128)); let thread_pool = ThreadPoolBuilder::new().num_threads(3).build().unwrap(); let recycler_cache = RecyclerCache::default(); let keypairs = repeat_with(|| rng.gen_range(169_367_809..169_906_789)) @@ -813,7 +870,7 @@ mod tests { .collect(); let mut packets = make_packets(&mut rng, &shreds); assert_eq!( - verify_shreds_gpu(&thread_pool, &packets, &pubkeys, &recycler_cache), + verify_shreds_gpu(&thread_pool, &packets, &pubkeys, &recycler_cache, &cache), packets .iter() .map(|batch| vec![1u8; batch.len()]) @@ -836,7 +893,7 @@ mod tests { }) .collect(); assert_eq!( - verify_shreds_gpu(&thread_pool, &packets, &pubkeys, &recycler_cache), + verify_shreds_gpu(&thread_pool, &packets, &pubkeys, &recycler_cache, &cache), out ); } @@ -844,6 +901,7 @@ mod tests { #[test] fn test_sign_shreds_gpu() { let mut rng = rand::thread_rng(); + let cache = RwLock::new(LruCache::new(/*capacity:*/ 128)); let thread_pool = ThreadPoolBuilder::new().num_threads(3).build().unwrap(); let recycler_cache = RecyclerCache::default(); let shreds = { @@ -866,7 +924,7 @@ mod tests { let mut packets = make_packets(&mut rng, &shreds); // Assert that initially all signatrues are invalid. assert_eq!( - verify_shreds_gpu(&thread_pool, &packets, &pubkeys, &recycler_cache), + verify_shreds_gpu(&thread_pool, &packets, &pubkeys, &recycler_cache, &cache), packets .iter() .map(|batch| vec![0u8; batch.len()]) @@ -883,7 +941,7 @@ mod tests { &recycler_cache, ); assert_eq!( - verify_shreds_gpu(&thread_pool, &packets, &pubkeys, &recycler_cache), + verify_shreds_gpu(&thread_pool, &packets, &pubkeys, &recycler_cache, &cache), packets .iter() .map(|batch| vec![1u8; batch.len()]) diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index 417e8a388ecea6..7fff595121a59c 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -195,6 +195,12 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "allocator-api2" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -1334,7 +1340,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if 1.0.0", - "hashbrown 0.14.1", + "hashbrown 0.14.3", "lock_api", "once_cell", "parking_lot_core 0.9.8", @@ -2056,9 +2062,13 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.1" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +dependencies = [ + "ahash 0.8.10", + "allocator-api2", +] [[package]] name = "headers" @@ -2367,7 +2377,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4" dependencies = [ "equivalent", - "hashbrown 0.14.1", + "hashbrown 0.14.3", "rayon", ] @@ -2582,6 +2592,15 @@ dependencies = [ "winapi-build", ] +[[package]] +name = "lazy-lru" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81b33bc1276f3df38e938ed17bbb3d5c5eef758aa1a9997ec8388799ba3eef1" +dependencies = [ + "hashbrown 0.14.3", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -5207,6 +5226,7 @@ dependencies = [ "fs_extra", "futures 0.3.30", "itertools", + "lazy-lru", "lazy_static", "libc", "log", diff --git a/turbine/src/sigverify_shreds.rs b/turbine/src/sigverify_shreds.rs index 76088a8115a837..2349e289905b46 100644 --- a/turbine/src/sigverify_shreds.rs +++ b/turbine/src/sigverify_shreds.rs @@ -3,7 +3,9 @@ use { rayon::{prelude::*, ThreadPool, ThreadPoolBuilder}, solana_gossip::cluster_info::ClusterInfo, solana_ledger::{ - leader_schedule_cache::LeaderScheduleCache, shred, sigverify_shreds::verify_shreds_gpu, + leader_schedule_cache::LeaderScheduleCache, + shred, + sigverify_shreds::{verify_shreds_gpu, LruCache}, }, solana_perf::{self, deduper::Deduper, packet::PacketBatch, recycler_cache::RecyclerCache}, solana_rayon_threadlimit::get_thread_count, @@ -17,6 +19,9 @@ use { }, }; +// 34MB where each cache entry is 136 bytes. +const SIGVERIFY_LRU_CACHE_CAPACITY: usize = 1 << 18; + const DEDUPER_FALSE_POSITIVE_RATE: f64 = 0.001; const DEDUPER_NUM_BITS: u64 = 637_534_199; // 76MB const DEDUPER_RESET_CYCLE: Duration = Duration::from_secs(5 * 60); @@ -38,6 +43,7 @@ pub fn spawn_shred_sigverify( ) -> JoinHandle<()> { let recycler_cache = RecyclerCache::warmed(); let mut stats = ShredSigVerifyStats::new(Instant::now()); + let cache = RwLock::new(LruCache::new(SIGVERIFY_LRU_CACHE_CAPACITY)); let thread_pool = ThreadPoolBuilder::new() .num_threads(get_thread_count()) .thread_name(|i| format!("solSvrfyShred{i:02}")) @@ -62,6 +68,7 @@ pub fn spawn_shred_sigverify( &shred_fetch_receiver, &retransmit_sender, &verified_sender, + &cache, &mut stats, ) { Ok(()) => (), @@ -89,6 +96,7 @@ fn run_shred_sigverify( shred_fetch_receiver: &Receiver, retransmit_sender: &Sender>>, verified_sender: &Sender>, + cache: &RwLock, stats: &mut ShredSigVerifyStats, ) -> Result<(), Error> { const RECV_TIMEOUT: Duration = Duration::from_secs(1); @@ -121,6 +129,7 @@ fn run_shred_sigverify( leader_schedule_cache, recycler_cache, &mut packets, + cache, ); stats.num_discards_post += count_discards(&packets); // Exclude repair packets from retransmit. @@ -145,6 +154,7 @@ fn verify_packets( leader_schedule_cache: &LeaderScheduleCache, recycler_cache: &RecyclerCache, packets: &mut [PacketBatch], + cache: &RwLock, ) { let working_bank = bank_forks.read().unwrap().working_bank(); let leader_slots: HashMap = @@ -153,7 +163,7 @@ fn verify_packets( .filter_map(|(slot, pubkey)| Some((slot, pubkey?))) .chain(std::iter::once((Slot::MAX, Pubkey::default()))) .collect(); - let out = verify_shreds_gpu(thread_pool, packets, &leader_slots, recycler_cache); + let out = verify_shreds_gpu(thread_pool, packets, &leader_slots, recycler_cache, cache); solana_perf::sigverify::mark_disabled(packets, &out); } @@ -319,6 +329,7 @@ mod tests { batches[0][1].buffer_mut()[..shred.payload().len()].copy_from_slice(shred.payload()); batches[0][1].meta_mut().size = shred.payload().len(); + let cache = RwLock::new(LruCache::new(/*capacity:*/ 128)); let thread_pool = ThreadPoolBuilder::new().num_threads(3).build().unwrap(); verify_packets( &thread_pool, @@ -327,6 +338,7 @@ mod tests { &leader_schedule_cache, &RecyclerCache::warmed(), &mut batches, + &cache, ); assert!(!batches[0][0].meta().discard()); assert!(batches[0][1].meta().discard());