From d5592c06d731bc141d3bf6817cfb4b45fcecec0f Mon Sep 17 00:00:00 2001 From: Justin Starry Date: Sat, 11 Dec 2021 09:44:15 -0500 Subject: [PATCH 1/2] Rename Packets to PacketBatch (#21794) (cherry picked from commit 254ef3e7b666f0405d7192c2696122b120ef5f32) # Conflicts: # core/src/ancestor_hashes_service.rs # core/src/banking_stage.rs # core/src/cluster_info_vote_listener.rs # core/src/fetch_stage.rs # core/src/serve_repair.rs # core/src/shred_fetch_stage.rs # core/src/sigverify_stage.rs # core/src/verified_vote_packets.rs # core/src/window_service.rs # gossip/src/cluster_info.rs # ledger/src/entry.rs # streamer/src/streamer.rs --- banking-bench/src/main.rs | 6 +- bench-streamer/src/main.rs | 22 +- core/benches/banking_stage.rs | 12 +- core/benches/sigverify_stage.rs | 8 +- core/src/ancestor_hashes_service.rs | 1548 ++++++++++++++++++++++++ core/src/banking_stage.rs | 410 ++++--- core/src/cluster_info_vote_listener.rs | 108 +- core/src/fetch_stage.rs | 43 +- core/src/retransmit_stage.rs | 12 +- core/src/serve_repair.rs | 177 ++- core/src/shred_fetch_stage.rs | 23 +- core/src/sigverify.rs | 10 +- core/src/sigverify_shreds.rs | 38 +- core/src/sigverify_stage.rs | 76 +- core/src/verified_vote_packets.rs | 432 +++++++ core/src/window_service.rs | 10 +- gossip/src/cluster_info.rs | 114 +- ledger/benches/sigverify_shreds.rs | 28 +- ledger/src/entry.rs | 209 ++++ ledger/src/sigverify_shreds.rs | 149 +-- perf/benches/recycler.rs | 4 +- perf/benches/sigverify.rs | 6 +- perf/src/packet.rs | 79 +- perf/src/recycler.rs | 4 +- perf/src/sigverify.rs | 106 +- streamer/src/packet.rs | 52 +- streamer/src/streamer.rs | 99 +- 27 files changed, 3197 insertions(+), 588 deletions(-) create mode 100644 core/src/ancestor_hashes_service.rs diff --git a/banking-bench/src/main.rs b/banking-bench/src/main.rs index 0f4e4e45031..6f39931bbcb 100644 --- a/banking-bench/src/main.rs +++ b/banking-bench/src/main.rs @@ -13,7 +13,7 @@ use { get_tmp_ledger_path, }, solana_measure::measure::Measure, - solana_perf::packet::to_packets_chunked, + solana_perf::packet::to_packet_batches, solana_poh::poh_recorder::{create_test_recorder, PohRecorder, WorkingBankEntry}, solana_runtime::{ accounts_background_service::AbsRequestSender, bank::Bank, bank_forks::BankForks, @@ -211,7 +211,7 @@ fn main() { bank.clear_signatures(); } - let mut verified: Vec<_> = to_packets_chunked(&transactions, packets_per_chunk); + let mut verified: Vec<_> = to_packet_batches(&transactions, packets_per_chunk); let ledger_path = get_tmp_ledger_path!(); { let blockstore = Arc::new( @@ -364,7 +364,7 @@ fn main() { let sig: Vec = (0..64).map(|_| thread_rng().gen::()).collect(); tx.signatures[0] = Signature::new(&sig[0..64]); } - verified = to_packets_chunked(&transactions.clone(), packets_per_chunk); + verified = to_packet_batches(&transactions.clone(), packets_per_chunk); } start += chunk_len; diff --git a/bench-streamer/src/main.rs b/bench-streamer/src/main.rs index bade7a94309..46eeeb76138 100644 --- a/bench-streamer/src/main.rs +++ b/bench-streamer/src/main.rs @@ -2,8 +2,8 @@ use { clap::{crate_description, crate_name, App, Arg}, solana_streamer::{ - packet::{Packet, Packets, PacketsRecycler, PACKET_DATA_SIZE}, - streamer::{receiver, PacketReceiver}, + packet::{Packet, PacketBatch, PacketBatchRecycler, PACKET_DATA_SIZE}, + streamer::{receiver, PacketBatchReceiver}, }, std::{ cmp::max, @@ -20,19 +20,19 @@ use { fn producer(addr: &SocketAddr, exit: Arc) -> JoinHandle<()> { let send = UdpSocket::bind("0.0.0.0:0").unwrap(); - let mut msgs = Packets::default(); - msgs.packets.resize(10, Packet::default()); - for w in msgs.packets.iter_mut() { + let mut packet_batch = PacketBatch::default(); + packet_batch.packets.resize(10, Packet::default()); + for w in packet_batch.packets.iter_mut() { w.meta.size = PACKET_DATA_SIZE; w.meta.set_addr(addr); } - let msgs = Arc::new(msgs); + let packet_batch = Arc::new(packet_batch); spawn(move || loop { if exit.load(Ordering::Relaxed) { return; } let mut num = 0; - for p in &msgs.packets { + for p in &packet_batch.packets { let a = p.meta.addr(); assert!(p.meta.size <= PACKET_DATA_SIZE); send.send_to(&p.data[..p.meta.size], &a).unwrap(); @@ -42,14 +42,14 @@ fn producer(addr: &SocketAddr, exit: Arc) -> JoinHandle<()> { }) } -fn sink(exit: Arc, rvs: Arc, r: PacketReceiver) -> JoinHandle<()> { +fn sink(exit: Arc, rvs: Arc, r: PacketBatchReceiver) -> JoinHandle<()> { spawn(move || loop { if exit.load(Ordering::Relaxed) { return; } let timer = Duration::new(1, 0); - if let Ok(msgs) = r.recv_timeout(timer) { - rvs.fetch_add(msgs.packets.len(), Ordering::Relaxed); + if let Ok(packet_batch) = r.recv_timeout(timer) { + rvs.fetch_add(packet_batch.packets.len(), Ordering::Relaxed); } }) } @@ -81,7 +81,7 @@ fn main() -> Result<()> { let mut read_channels = Vec::new(); let mut read_threads = Vec::new(); - let recycler = PacketsRecycler::default(); + let recycler = PacketBatchRecycler::default(); for _ in 0..num_sockets { let read = solana_net_utils::bind_to(ip_addr, port, false).unwrap(); read.set_read_timeout(Some(Duration::new(1, 0))).unwrap(); diff --git a/core/benches/banking_stage.rs b/core/benches/banking_stage.rs index 5057a78dd28..044eb4820b5 100644 --- a/core/benches/banking_stage.rs +++ b/core/benches/banking_stage.rs @@ -17,7 +17,7 @@ use { genesis_utils::{create_genesis_config, GenesisConfigInfo}, get_tmp_ledger_path, }, - solana_perf::{packet::to_packets_chunked, test_tx::test_tx}, + solana_perf::{packet::to_packet_batches, test_tx::test_tx}, solana_poh::poh_recorder::{create_test_recorder, WorkingBankEntry}, solana_runtime::{bank::Bank, cost_model::CostModel}, solana_sdk::{ @@ -74,11 +74,11 @@ fn bench_consume_buffered(bencher: &mut Bencher) { let tx = test_tx(); let len = 4096; let chunk_size = 1024; - let batches = to_packets_chunked(&vec![tx; len], chunk_size); - let mut packets = VecDeque::new(); + let batches = to_packet_batches(&vec![tx; len], chunk_size); + let mut packet_batches = VecDeque::new(); for batch in batches { let batch_len = batch.packets.len(); - packets.push_back((batch, vec![0usize; batch_len], false)); + packet_batches.push_back((batch, vec![0usize; batch_len], false)); } let (s, _r) = unbounded(); // This tests the performance of buffering packets. @@ -88,7 +88,7 @@ fn bench_consume_buffered(bencher: &mut Bencher) { &my_pubkey, std::u128::MAX, &poh_recorder, - &mut packets, + &mut packet_batches, None, &s, None::>, @@ -203,7 +203,7 @@ fn bench_banking(bencher: &mut Bencher, tx_type: TransactionType) { assert!(r.is_ok(), "sanity parallel execution"); } bank.clear_signatures(); - let verified: Vec<_> = to_packets_chunked(&transactions, PACKETS_PER_BATCH); + let verified: Vec<_> = to_packet_batches(&transactions, PACKETS_PER_BATCH); let ledger_path = get_tmp_ledger_path!(); { let blockstore = Arc::new( diff --git a/core/benches/sigverify_stage.rs b/core/benches/sigverify_stage.rs index e48ab9301c1..894c474ce8b 100644 --- a/core/benches/sigverify_stage.rs +++ b/core/benches/sigverify_stage.rs @@ -8,7 +8,7 @@ use { log::*, rand::{thread_rng, Rng}, solana_core::{sigverify::TransactionSigVerifier, sigverify_stage::SigVerifyStage}, - solana_perf::{packet::to_packets_chunked, test_tx::test_tx}, + solana_perf::{packet::to_packet_batches, test_tx::test_tx}, solana_sdk::{ hash::Hash, signature::{Keypair, Signer}, @@ -28,7 +28,7 @@ fn bench_packet_discard(bencher: &mut Bencher) { let len = 30 * 1000; let chunk_size = 1024; let tx = test_tx(); - let mut batches = to_packets_chunked(&vec![tx; len], chunk_size); + let mut batches = to_packet_batches(&vec![tx; len], chunk_size); let mut total = 0; @@ -74,7 +74,7 @@ fn bench_sigverify_stage(bencher: &mut Bencher) { let chunk_size = 1024; let mut batches = if use_same_tx { let tx = test_tx(); - to_packets_chunked(&vec![tx; len], chunk_size) + to_packet_batches(&vec![tx; len], chunk_size) } else { let from_keypair = Keypair::new(); let to_keypair = Keypair::new(); @@ -89,7 +89,7 @@ fn bench_sigverify_stage(bencher: &mut Bencher) { ) }) .collect(); - to_packets_chunked(&txs, chunk_size) + to_packet_batches(&txs, chunk_size) }; trace!( diff --git a/core/src/ancestor_hashes_service.rs b/core/src/ancestor_hashes_service.rs new file mode 100644 index 00000000000..4ccdb33338e --- /dev/null +++ b/core/src/ancestor_hashes_service.rs @@ -0,0 +1,1548 @@ +use { + crate::{ + cluster_slots::ClusterSlots, + duplicate_repair_status::{DeadSlotAncestorRequestStatus, DuplicateAncestorDecision}, + outstanding_requests::OutstandingRequests, + repair_response::{self}, + repair_service::{DuplicateSlotsResetSender, RepairInfo, RepairStatsGroup}, + replay_stage::DUPLICATE_THRESHOLD, + result::{Error, Result}, + serve_repair::{AncestorHashesRepairType, ServeRepair}, + }, + crossbeam_channel::{unbounded, Receiver, Sender}, + dashmap::{mapref::entry::Entry::Occupied, DashMap}, + solana_ledger::{blockstore::Blockstore, shred::SIZE_OF_NONCE}, + solana_measure::measure::Measure, + solana_perf::{ + packet::{limited_deserialize, Packet, PacketBatch}, + recycler::Recycler, + }, + solana_runtime::bank::Bank, + solana_sdk::{ + clock::{Slot, SLOT_MS}, + pubkey::Pubkey, + timing::timestamp, + }, + solana_streamer::streamer::{self, PacketBatchReceiver}, + std::{ + collections::HashSet, + net::UdpSocket, + sync::{ + atomic::{AtomicBool, Ordering}, + mpsc::channel, + Arc, RwLock, + }, + thread::{self, sleep, Builder, JoinHandle}, + time::{Duration, Instant}, + }, +}; + +#[derive(Debug, PartialEq)] +pub enum AncestorHashesReplayUpdate { + Dead(Slot), + DeadDuplicateConfirmed(Slot), +} + +impl AncestorHashesReplayUpdate { + fn slot(&self) -> Slot { + match self { + AncestorHashesReplayUpdate::Dead(slot) => *slot, + AncestorHashesReplayUpdate::DeadDuplicateConfirmed(slot) => *slot, + } + } +} + +pub const MAX_ANCESTOR_HASHES_SLOT_REQUESTS_PER_SECOND: usize = 2; + +pub type AncestorHashesReplayUpdateSender = Sender; +pub type AncestorHashesReplayUpdateReceiver = Receiver; + +type RetryableSlotsSender = Sender; +type RetryableSlotsReceiver = Receiver; +type OutstandingAncestorHashesRepairs = OutstandingRequests; + +#[derive(Default)] +pub struct AncestorHashesResponsesStats { + pub total_packets: usize, + pub dropped_packets: usize, + pub invalid_packets: usize, + pub processed: usize, +} + +impl AncestorHashesResponsesStats { + fn report(&mut self) { + inc_new_counter_info!( + "ancestor_hashes_responses-total_packets", + self.total_packets + ); + inc_new_counter_info!("ancestor_hashes_responses-processed", self.processed); + inc_new_counter_info!( + "ancestor_hashes_responses-dropped_packets", + self.dropped_packets + ); + inc_new_counter_info!( + "ancestor_hashes_responses-invalid_packets", + self.invalid_packets + ); + *self = AncestorHashesResponsesStats::default(); + } +} + +pub struct AncestorRepairRequestsStats { + pub ancestor_requests: RepairStatsGroup, + last_report: Instant, +} + +impl Default for AncestorRepairRequestsStats { + fn default() -> Self { + AncestorRepairRequestsStats { + ancestor_requests: RepairStatsGroup::default(), + last_report: Instant::now(), + } + } +} + +impl AncestorRepairRequestsStats { + fn report(&mut self) { + let slot_to_count: Vec<_> = self + .ancestor_requests + .slot_pubkeys + .iter() + .map(|(slot, slot_repairs)| { + ( + slot, + slot_repairs + .pubkey_repairs() + .iter() + .map(|(_key, count)| count) + .sum::(), + ) + }) + .collect(); + + let repair_total = self.ancestor_requests.count; + if self.last_report.elapsed().as_secs() > 2 && repair_total > 0 { + info!("ancestor_repair_requests_stats: {:?}", slot_to_count); + datapoint_info!( + "ancestor-repair", + ("ancestor-repair-count", self.ancestor_requests.count, i64) + ); + + *self = AncestorRepairRequestsStats::default(); + } + } +} + +pub struct AncestorHashesService { + thread_hdls: Vec>, +} + +impl AncestorHashesService { + pub fn new( + exit: Arc, + blockstore: Arc, + ancestor_hashes_request_socket: Arc, + repair_info: RepairInfo, + ancestor_hashes_replay_update_receiver: AncestorHashesReplayUpdateReceiver, + ) -> Self { + let outstanding_requests: Arc> = + Arc::new(RwLock::new(OutstandingAncestorHashesRepairs::default())); + let (response_sender, response_receiver) = channel(); + let t_receiver = streamer::receiver( + ancestor_hashes_request_socket.clone(), + &exit, + response_sender, + Recycler::default(), + "ancestor_hashes_response_receiver", + 1, + false, + ); + + let ancestor_hashes_request_statuses: Arc> = + Arc::new(DashMap::new()); + let (retryable_slots_sender, retryable_slots_receiver) = unbounded(); + + // Listen for responses to our ancestor requests + let t_ancestor_hashes_responses = Self::run_responses_listener( + ancestor_hashes_request_statuses.clone(), + response_receiver, + blockstore, + outstanding_requests.clone(), + exit.clone(), + repair_info.duplicate_slots_reset_sender.clone(), + retryable_slots_sender, + ); + + // Generate ancestor requests for dead slots that are repairable + let t_ancestor_requests = Self::run_manage_ancestor_requests( + ancestor_hashes_request_statuses, + ancestor_hashes_request_socket, + repair_info, + outstanding_requests, + exit, + ancestor_hashes_replay_update_receiver, + retryable_slots_receiver, + ); + let thread_hdls = vec![t_receiver, t_ancestor_hashes_responses, t_ancestor_requests]; + Self { thread_hdls } + } + + pub fn join(self) -> thread::Result<()> { + for thread_hdl in self.thread_hdls { + thread_hdl.join()?; + } + Ok(()) + } + + /// Listen for responses to our ancestors hashes repair requests + fn run_responses_listener( + ancestor_hashes_request_statuses: Arc>, + response_receiver: PacketBatchReceiver, + blockstore: Arc, + outstanding_requests: Arc>, + exit: Arc, + duplicate_slots_reset_sender: DuplicateSlotsResetSender, + retryable_slots_sender: RetryableSlotsSender, + ) -> JoinHandle<()> { + Builder::new() + .name("solana-ancestor-hashes-responses-service".to_string()) + .spawn(move || { + let mut last_stats_report = Instant::now(); + let mut stats = AncestorHashesResponsesStats::default(); + let mut max_packets = 1024; + loop { + let result = Self::process_new_packets_from_channel( + &ancestor_hashes_request_statuses, + &response_receiver, + &blockstore, + &outstanding_requests, + &mut stats, + &mut max_packets, + &duplicate_slots_reset_sender, + &retryable_slots_sender, + ); + match result { + Err(Error::RecvTimeout(_)) | Ok(_) => {} + Err(err) => info!("ancestors hashes reponses listener error: {:?}", err), + }; + if exit.load(Ordering::Relaxed) { + return; + } + if last_stats_report.elapsed().as_secs() > 2 { + stats.report(); + last_stats_report = Instant::now(); + } + } + }) + .unwrap() + } + + /// Process messages from the network + fn process_new_packets_from_channel( + ancestor_hashes_request_statuses: &DashMap, + response_receiver: &PacketBatchReceiver, + blockstore: &Blockstore, + outstanding_requests: &RwLock, + stats: &mut AncestorHashesResponsesStats, + max_packets: &mut usize, + duplicate_slots_reset_sender: &DuplicateSlotsResetSender, + retryable_slots_sender: &RetryableSlotsSender, + ) -> Result<()> { + let timeout = Duration::new(1, 0); + let mut packet_batches = vec![response_receiver.recv_timeout(timeout)?]; + let mut total_packets = packet_batches[0].packets.len(); + + let mut dropped_packets = 0; + while let Ok(batch) = response_receiver.try_recv() { + total_packets += batch.packets.len(); + if total_packets < *max_packets { + // Drop the rest in the channel in case of DOS + packet_batches.push(batch); + } else { + dropped_packets += batch.packets.len(); + } + } + + stats.dropped_packets += dropped_packets; + stats.total_packets += total_packets; + + let mut time = Measure::start("ancestor_hashes::handle_packets"); + for packet_batch in packet_batches { + Self::process_packet_batch( + ancestor_hashes_request_statuses, + packet_batch, + stats, + outstanding_requests, + blockstore, + duplicate_slots_reset_sender, + retryable_slots_sender, + ); + } + time.stop(); + if total_packets >= *max_packets { + if time.as_ms() > 1000 { + *max_packets = (*max_packets * 9) / 10; + } else { + *max_packets = (*max_packets * 10) / 9; + } + } + Ok(()) + } + + fn process_packet_batch( + ancestor_hashes_request_statuses: &DashMap, + packet_batch: PacketBatch, + stats: &mut AncestorHashesResponsesStats, + outstanding_requests: &RwLock, + blockstore: &Blockstore, + duplicate_slots_reset_sender: &DuplicateSlotsResetSender, + retryable_slots_sender: &RetryableSlotsSender, + ) { + packet_batch.packets.iter().for_each(|packet| { + let decision = Self::verify_and_process_ancestor_response( + packet, + ancestor_hashes_request_statuses, + stats, + outstanding_requests, + blockstore, + ); + if let Some((slot, decision)) = decision { + Self::handle_ancestor_request_decision( + slot, + decision, + duplicate_slots_reset_sender, + retryable_slots_sender, + ); + } + }); + } + + /// Returns `Some((request_slot, decision))`, where `decision` is an actionable + /// result after processing sufficient responses for the subject of the query, + /// `request_slot` + fn verify_and_process_ancestor_response( + packet: &Packet, + ancestor_hashes_request_statuses: &DashMap, + stats: &mut AncestorHashesResponsesStats, + outstanding_requests: &RwLock, + blockstore: &Blockstore, + ) -> Option<(Slot, DuplicateAncestorDecision)> { + let from_addr = packet.meta.addr(); + limited_deserialize(&packet.data[..packet.meta.size - SIZE_OF_NONCE]) + .ok() + .and_then(|ancestor_hashes_response| { + // Verify the response + let request_slot = repair_response::nonce(&packet.data[..packet.meta.size]) + .and_then(|nonce| { + outstanding_requests.write().unwrap().register_response( + nonce, + &ancestor_hashes_response, + timestamp(), + // If the response is valid, return the slot the request + // was for + |ancestor_hashes_request| ancestor_hashes_request.0, + ) + }); + + if request_slot.is_none() { + stats.invalid_packets += 1; + return None; + } + + // If was a valid response, there must be a valid `request_slot` + let request_slot = request_slot.unwrap(); + stats.processed += 1; + + if let Occupied(mut ancestor_hashes_status_ref) = + ancestor_hashes_request_statuses.entry(request_slot) + { + let decision = ancestor_hashes_status_ref.get_mut().add_response( + &from_addr, + ancestor_hashes_response.into_slot_hashes(), + blockstore, + ); + if decision.is_some() { + // Once a request is completed, remove it from the map so that new + // requests for the same slot can be made again if necessary. It's + // important to hold the `write` lock here via + // `ancestor_hashes_status_ref` so that we don't race with deletion + + // insertion from the `t_ancestor_requests` thread, which may + // 1) Remove expired statuses from `ancestor_hashes_request_statuses` + // 2) Insert another new one via `manage_ancestor_requests()`. + // In which case we wouldn't want to delete the newly inserted entry here. + ancestor_hashes_status_ref.remove(); + } + decision.map(|decision| (request_slot, decision)) + } else { + None + } + }) + } + + fn handle_ancestor_request_decision( + slot: Slot, + decision: DuplicateAncestorDecision, + duplicate_slots_reset_sender: &DuplicateSlotsResetSender, + retryable_slots_sender: &RetryableSlotsSender, + ) { + if decision.is_retryable() { + let _ = retryable_slots_sender.send(slot); + } + let potential_slots_to_dump = { + // TODO: In the case of DuplicateAncestorDecision::ContinueSearch + // This means all the ancestors were mismatched, which + // means the earliest mismatched ancestor has yet to be found. + // + // In the best case scenario, this means after ReplayStage dumps + // the earliest known ancestor `A` here, and then repairs `A`, + // because we may still have the incorrect version of some ancestor + // of `A`, we will mark `A` as dead and then continue the search + // protocol through another round of ancestor repairs. + // + // However this process is a bit slow, so in an ideal world, the + // protocol could be extended to keep searching by making + // another ancestor repair request from the earliest returned + // ancestor from this search. + decision + .repair_status() + .map(|status| status.correct_ancestors_to_repair.clone()) + }; + + // Now signal ReplayStage about the new updated slots. It's important to do this + // AFTER we've removed the ancestor_hashes_status_ref in case replay + // then sends us another dead slot signal based on the updates we are + // about to send. + if let Some(potential_slots_to_dump) = potential_slots_to_dump { + // Signal ReplayStage to dump the fork that is descended from + // `earliest_mismatched_slot_to_dump`. + if !potential_slots_to_dump.is_empty() { + let _ = duplicate_slots_reset_sender.send(potential_slots_to_dump); + } + } + } + + fn process_replay_updates( + ancestor_hashes_replay_update_receiver: &AncestorHashesReplayUpdateReceiver, + ancestor_hashes_request_statuses: &DashMap, + dead_slot_pool: &mut HashSet, + repairable_dead_slot_pool: &mut HashSet, + root_slot: Slot, + ) { + for update in ancestor_hashes_replay_update_receiver.try_iter() { + let slot = update.slot(); + if slot <= root_slot || ancestor_hashes_request_statuses.contains_key(&slot) { + return; + } + match update { + AncestorHashesReplayUpdate::Dead(dead_slot) => { + if repairable_dead_slot_pool.contains(&dead_slot) { + return; + } else { + dead_slot_pool.insert(dead_slot); + } + } + AncestorHashesReplayUpdate::DeadDuplicateConfirmed(dead_slot) => { + dead_slot_pool.remove(&dead_slot); + repairable_dead_slot_pool.insert(dead_slot); + } + } + } + } + + fn run_manage_ancestor_requests( + ancestor_hashes_request_statuses: Arc>, + ancestor_hashes_request_socket: Arc, + repair_info: RepairInfo, + outstanding_requests: Arc>, + exit: Arc, + ancestor_hashes_replay_update_receiver: AncestorHashesReplayUpdateReceiver, + retryable_slots_receiver: RetryableSlotsReceiver, + ) -> JoinHandle<()> { + let serve_repair = ServeRepair::new(repair_info.cluster_info.clone()); + let mut repair_stats = AncestorRepairRequestsStats::default(); + + let mut dead_slot_pool = HashSet::new(); + let mut repairable_dead_slot_pool = HashSet::new(); + + // Sliding window that limits the number of slots repaired via AncestorRepair + // to MAX_ANCESTOR_HASHES_SLOT_REQUESTS_PER_SECOND/second + let mut request_throttle = vec![]; + Builder::new() + .name("solana-manage-ancestor-requests".to_string()) + .spawn(move || loop { + if exit.load(Ordering::Relaxed) { + return; + } + + Self::manage_ancestor_requests( + &ancestor_hashes_request_statuses, + &ancestor_hashes_request_socket, + &repair_info, + &outstanding_requests, + &ancestor_hashes_replay_update_receiver, + &retryable_slots_receiver, + &serve_repair, + &mut repair_stats, + &mut dead_slot_pool, + &mut repairable_dead_slot_pool, + &mut request_throttle, + ); + + sleep(Duration::from_millis(SLOT_MS)); + }) + .unwrap() + } + + #[allow(clippy::too_many_arguments)] + fn manage_ancestor_requests( + ancestor_hashes_request_statuses: &DashMap, + ancestor_hashes_request_socket: &UdpSocket, + repair_info: &RepairInfo, + outstanding_requests: &RwLock, + ancestor_hashes_replay_update_receiver: &AncestorHashesReplayUpdateReceiver, + retryable_slots_receiver: &RetryableSlotsReceiver, + serve_repair: &ServeRepair, + repair_stats: &mut AncestorRepairRequestsStats, + dead_slot_pool: &mut HashSet, + repairable_dead_slot_pool: &mut HashSet, + request_throttle: &mut Vec, + ) { + let root_bank = repair_info.bank_forks.read().unwrap().root_bank(); + for slot in retryable_slots_receiver.try_iter() { + datapoint_info!("ancestor-repair-retry", ("slot", slot, i64)); + repairable_dead_slot_pool.insert(slot); + } + + Self::process_replay_updates( + ancestor_hashes_replay_update_receiver, + ancestor_hashes_request_statuses, + dead_slot_pool, + repairable_dead_slot_pool, + root_bank.slot(), + ); + + Self::find_epoch_slots_frozen_dead_slots( + &repair_info.cluster_slots, + dead_slot_pool, + repairable_dead_slot_pool, + &root_bank, + ); + + dead_slot_pool.retain(|slot| *slot > root_bank.slot()); + + repairable_dead_slot_pool.retain(|slot| *slot > root_bank.slot()); + + ancestor_hashes_request_statuses.retain(|slot, status| { + if *slot <= root_bank.slot() { + false + } else if status.is_expired() { + // Add the slot back to the repairable pool to retry + repairable_dead_slot_pool.insert(*slot); + false + } else { + true + } + }); + + // Keep around the last second of requests in the throttler. + request_throttle.retain(|request_time| *request_time > (timestamp() - 1000)); + + let number_of_allowed_requests = + MAX_ANCESTOR_HASHES_SLOT_REQUESTS_PER_SECOND.saturating_sub(request_throttle.len()); + + // Find dead slots for which it's worthwhile to ask the network for their + // ancestors + for _ in 0..number_of_allowed_requests { + let slot = repairable_dead_slot_pool.iter().next().cloned(); + if let Some(slot) = slot { + warn!( + "Cluster froze slot: {}, but we marked it as dead. + Initiating protocol to sample cluster for dead slot ancestors.", + slot + ); + + if Self::initiate_ancestor_hashes_requests_for_duplicate_slot( + ancestor_hashes_request_statuses, + ancestor_hashes_request_socket, + &repair_info.cluster_slots, + serve_repair, + &repair_info.repair_validators, + slot, + repair_stats, + outstanding_requests, + ) { + request_throttle.push(timestamp()); + repairable_dead_slot_pool.take(&slot).unwrap(); + } + } else { + break; + } + } + + repair_stats.report(); + } + + /// Find if any dead slots in `dead_slot_pool` have been frozen by sufficient + /// number of nodes in the cluster to justify adding to the `repairable_dead_slot_pool`. + fn find_epoch_slots_frozen_dead_slots( + cluster_slots: &ClusterSlots, + dead_slot_pool: &mut HashSet, + repairable_dead_slot_pool: &mut HashSet, + root_bank: &Bank, + ) { + dead_slot_pool.retain(|dead_slot| { + let epoch = root_bank.get_epoch_and_slot_index(*dead_slot).0; + if let Some(epoch_stakes) = root_bank.epoch_stakes(epoch) { + let status = cluster_slots.lookup(*dead_slot); + if let Some(completed_dead_slot_pubkeys) = status { + let total_stake = epoch_stakes.total_stake(); + let node_id_to_vote_accounts = epoch_stakes.node_id_to_vote_accounts(); + let total_completed_slot_stake: u64 = completed_dead_slot_pubkeys + .read() + .unwrap() + .iter() + .map(|(node_key, _)| { + node_id_to_vote_accounts + .get(node_key) + .map(|v| v.total_stake) + .unwrap_or(0) + }) + .sum(); + // If sufficient number of validators froze this slot, then there's a chance + // this dead slot was duplicate confirmed and will make it into in the main fork. + // This means it's worth asking the cluster to get the correct version. + if total_completed_slot_stake as f64 / total_stake as f64 > DUPLICATE_THRESHOLD + { + repairable_dead_slot_pool.insert(*dead_slot); + false + } else { + true + } + } else { + true + } + } else { + warn!( + "Dead slot {} is too far ahead of root bank {}", + dead_slot, + root_bank.slot() + ); + false + } + }) + } + + /// Returns true if a request was successfully made and the status + /// added to `ancestor_hashes_request_statuses` + fn initiate_ancestor_hashes_requests_for_duplicate_slot( + ancestor_hashes_request_statuses: &DashMap, + ancestor_hashes_request_socket: &UdpSocket, + cluster_slots: &ClusterSlots, + serve_repair: &ServeRepair, + repair_validators: &Option>, + duplicate_slot: Slot, + repair_stats: &mut AncestorRepairRequestsStats, + outstanding_requests: &RwLock, + ) -> bool { + let sampled_validators = serve_repair.repair_request_ancestor_hashes_sample_peers( + duplicate_slot, + cluster_slots, + repair_validators, + ); + + if let Ok(sampled_validators) = sampled_validators { + for (pubkey, socket_addr) in sampled_validators.iter() { + repair_stats + .ancestor_requests + .update(pubkey, duplicate_slot, 0); + let nonce = outstanding_requests + .write() + .unwrap() + .add_request(AncestorHashesRepairType(duplicate_slot), timestamp()); + let request_bytes = + serve_repair.ancestor_repair_request_bytes(duplicate_slot, nonce); + if let Ok(request_bytes) = request_bytes { + let _ = ancestor_hashes_request_socket.send_to(&request_bytes, socket_addr); + } + } + + let ancestor_request_status = DeadSlotAncestorRequestStatus::new( + sampled_validators + .into_iter() + .map(|(_pk, socket_addr)| socket_addr), + duplicate_slot, + ); + assert!(!ancestor_hashes_request_statuses.contains_key(&duplicate_slot)); + ancestor_hashes_request_statuses.insert(duplicate_slot, ancestor_request_status); + true + } else { + false + } + } +} + +#[cfg(test)] +mod test { + use { + super::*, + crate::{ + cluster_slot_state_verifier::DuplicateSlotsToRepair, + repair_service::DuplicateSlotsResetReceiver, + replay_stage::{ + tests::{replay_blockstore_components, ReplayBlockstoreComponents}, + ReplayStage, + }, + serve_repair::MAX_ANCESTOR_RESPONSES, + vote_simulator::VoteSimulator, + }, + solana_gossip::{ + cluster_info::{ClusterInfo, Node}, + contact_info::ContactInfo, + }, + solana_ledger::{blockstore::make_many_slot_entries, get_tmp_ledger_path}, + solana_runtime::{accounts_background_service::AbsRequestSender, bank_forks::BankForks}, + solana_sdk::{hash::Hash, signature::Keypair}, + solana_streamer::socket::SocketAddrSpace, + std::{collections::HashMap, sync::mpsc::channel}, + trees::tr, + }; + + #[test] + pub fn test_ancestor_hashes_service_process_replay_updates() { + let (ancestor_hashes_replay_update_sender, ancestor_hashes_replay_update_receiver) = + unbounded(); + let ancestor_hashes_request_statuses = DashMap::new(); + let mut dead_slot_pool = HashSet::new(); + let mut repairable_dead_slot_pool = HashSet::new(); + let slot = 10; + let mut root_slot = 0; + + // 1) Getting a dead signal should only add the slot to the dead pool + ancestor_hashes_replay_update_sender + .send(AncestorHashesReplayUpdate::Dead(slot)) + .unwrap(); + AncestorHashesService::process_replay_updates( + &ancestor_hashes_replay_update_receiver, + &ancestor_hashes_request_statuses, + &mut dead_slot_pool, + &mut repairable_dead_slot_pool, + root_slot, + ); + assert!(dead_slot_pool.contains(&slot)); + assert!(repairable_dead_slot_pool.is_empty()); + + // 2) Getting a duplicate confirmed dead slot should move the slot + // from the dead pool to the repairable pool + ancestor_hashes_replay_update_sender + .send(AncestorHashesReplayUpdate::DeadDuplicateConfirmed(slot)) + .unwrap(); + AncestorHashesService::process_replay_updates( + &ancestor_hashes_replay_update_receiver, + &ancestor_hashes_request_statuses, + &mut dead_slot_pool, + &mut repairable_dead_slot_pool, + root_slot, + ); + assert!(dead_slot_pool.is_empty()); + assert!(repairable_dead_slot_pool.contains(&slot)); + + // 3) Getting another dead signal should not add it back to the dead pool + ancestor_hashes_replay_update_sender + .send(AncestorHashesReplayUpdate::Dead(slot)) + .unwrap(); + AncestorHashesService::process_replay_updates( + &ancestor_hashes_replay_update_receiver, + &ancestor_hashes_request_statuses, + &mut dead_slot_pool, + &mut repairable_dead_slot_pool, + root_slot, + ); + assert!(dead_slot_pool.is_empty()); + assert!(repairable_dead_slot_pool.contains(&slot)); + + // 4) If an outstanding request for a slot already exists, should + // ignore any signals from replay stage + ancestor_hashes_request_statuses.insert(slot, DeadSlotAncestorRequestStatus::default()); + dead_slot_pool.clear(); + repairable_dead_slot_pool.clear(); + ancestor_hashes_replay_update_sender + .send(AncestorHashesReplayUpdate::Dead(slot)) + .unwrap(); + ancestor_hashes_replay_update_sender + .send(AncestorHashesReplayUpdate::DeadDuplicateConfirmed(slot)) + .unwrap(); + AncestorHashesService::process_replay_updates( + &ancestor_hashes_replay_update_receiver, + &ancestor_hashes_request_statuses, + &mut dead_slot_pool, + &mut repairable_dead_slot_pool, + root_slot, + ); + assert!(dead_slot_pool.is_empty()); + assert!(repairable_dead_slot_pool.is_empty()); + + // 5) If we get any signals for slots <= root_slot, they should be ignored + root_slot = 15; + ancestor_hashes_request_statuses.clear(); + dead_slot_pool.clear(); + repairable_dead_slot_pool.clear(); + ancestor_hashes_replay_update_sender + .send(AncestorHashesReplayUpdate::Dead(root_slot - 1)) + .unwrap(); + ancestor_hashes_replay_update_sender + .send(AncestorHashesReplayUpdate::DeadDuplicateConfirmed( + root_slot - 2, + )) + .unwrap(); + AncestorHashesService::process_replay_updates( + &ancestor_hashes_replay_update_receiver, + &ancestor_hashes_request_statuses, + &mut dead_slot_pool, + &mut repairable_dead_slot_pool, + root_slot, + ); + assert!(dead_slot_pool.is_empty()); + assert!(repairable_dead_slot_pool.is_empty()); + } + + #[test] + fn test_ancestor_hashes_service_find_epoch_slots_frozen_dead_slots() { + let vote_simulator = VoteSimulator::new(3); + let cluster_slots = ClusterSlots::default(); + let mut dead_slot_pool = HashSet::new(); + let mut repairable_dead_slot_pool = HashSet::new(); + let root_bank = vote_simulator.bank_forks.read().unwrap().root_bank(); + let dead_slot = 10; + dead_slot_pool.insert(dead_slot); + + // ClusterSlots doesn't have an entry for this slot yet, shouldn't move the slot + // from the dead slot pool. + AncestorHashesService::find_epoch_slots_frozen_dead_slots( + &cluster_slots, + &mut dead_slot_pool, + &mut repairable_dead_slot_pool, + &root_bank, + ); + assert_eq!(dead_slot_pool.len(), 1); + assert!(dead_slot_pool.contains(&dead_slot)); + assert!(repairable_dead_slot_pool.is_empty()); + + let max_epoch = root_bank.epoch_stakes_map().keys().max().unwrap(); + let slot_outside_known_epochs = root_bank + .epoch_schedule() + .get_last_slot_in_epoch(*max_epoch) + + 1; + dead_slot_pool.insert(slot_outside_known_epochs); + + // Should remove `slot_outside_known_epochs` + AncestorHashesService::find_epoch_slots_frozen_dead_slots( + &cluster_slots, + &mut dead_slot_pool, + &mut repairable_dead_slot_pool, + &root_bank, + ); + assert_eq!(dead_slot_pool.len(), 1); + assert!(dead_slot_pool.contains(&dead_slot)); + assert!(repairable_dead_slot_pool.is_empty()); + + // Slot hasn't reached the threshold + for (i, key) in (0..2).zip(vote_simulator.node_pubkeys.iter()) { + cluster_slots.insert_node_id(dead_slot, *key); + AncestorHashesService::find_epoch_slots_frozen_dead_slots( + &cluster_slots, + &mut dead_slot_pool, + &mut repairable_dead_slot_pool, + &root_bank, + ); + if i == 0 { + assert_eq!(dead_slot_pool.len(), 1); + assert!(dead_slot_pool.contains(&dead_slot)); + assert!(repairable_dead_slot_pool.is_empty()); + } else { + assert!(dead_slot_pool.is_empty()); + assert_eq!(repairable_dead_slot_pool.len(), 1); + assert!(repairable_dead_slot_pool.contains(&dead_slot)); + } + } + } + + struct ResponderThreads { + t_request_receiver: JoinHandle<()>, + t_listen: JoinHandle<()>, + exit: Arc, + responder_info: ContactInfo, + response_receiver: PacketBatchReceiver, + correct_bank_hashes: HashMap, + } + + impl ResponderThreads { + fn shutdown(self) { + self.exit.store(true, Ordering::Relaxed); + self.t_request_receiver.join().unwrap(); + self.t_listen.join().unwrap(); + } + + fn new(slot_to_query: Slot) -> Self { + assert!(slot_to_query >= MAX_ANCESTOR_RESPONSES as Slot); + let responder_node = Node::new_localhost(); + let cluster_info = ClusterInfo::new( + responder_node.info.clone(), + Arc::new(Keypair::new()), + SocketAddrSpace::Unspecified, + ); + let responder_serve_repair = + Arc::new(RwLock::new(ServeRepair::new(Arc::new(cluster_info)))); + + // Set up thread to give us responses + let ledger_path = get_tmp_ledger_path!(); + let exit = Arc::new(AtomicBool::new(false)); + let (requests_sender, requests_receiver) = channel(); + let (response_sender, response_receiver) = channel(); + + // Set up blockstore for responses + let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap()); + // Create slots [slot, slot + MAX_ANCESTOR_RESPONSES) with 5 shreds apiece + let (shreds, _) = + make_many_slot_entries(slot_to_query, MAX_ANCESTOR_RESPONSES as u64, 5); + blockstore + .insert_shreds(shreds, None, false) + .expect("Expect successful ledger write"); + let mut correct_bank_hashes = HashMap::new(); + for duplicate_confirmed_slot in + slot_to_query - MAX_ANCESTOR_RESPONSES as Slot + 1..=slot_to_query + { + let hash = Hash::new_unique(); + correct_bank_hashes.insert(duplicate_confirmed_slot, hash); + blockstore.insert_bank_hash(duplicate_confirmed_slot, hash, true); + } + + // Set up response threads + let t_request_receiver = streamer::receiver( + Arc::new(responder_node.sockets.serve_repair), + &exit, + requests_sender, + Recycler::default(), + "serve_repair_receiver", + 1, + false, + ); + let t_listen = ServeRepair::listen( + responder_serve_repair, + Some(blockstore), + requests_receiver, + response_sender, + &exit, + ); + + Self { + t_request_receiver, + t_listen, + exit, + responder_info: responder_node.info, + response_receiver, + correct_bank_hashes, + } + } + } + + struct ManageAncestorHashesState { + ancestor_hashes_request_statuses: Arc>, + ancestor_hashes_request_socket: Arc, + requester_serve_repair: ServeRepair, + repair_info: RepairInfo, + outstanding_requests: Arc>, + dead_slot_pool: HashSet, + repairable_dead_slot_pool: HashSet, + request_throttle: Vec, + repair_stats: AncestorRepairRequestsStats, + _duplicate_slots_reset_receiver: DuplicateSlotsResetReceiver, + retryable_slots_sender: RetryableSlotsSender, + retryable_slots_receiver: RetryableSlotsReceiver, + ancestor_hashes_replay_update_sender: AncestorHashesReplayUpdateSender, + ancestor_hashes_replay_update_receiver: AncestorHashesReplayUpdateReceiver, + } + + impl ManageAncestorHashesState { + fn new(bank_forks: Arc>) -> Self { + let ancestor_hashes_request_statuses = Arc::new(DashMap::new()); + let ancestor_hashes_request_socket = Arc::new(UdpSocket::bind("0.0.0.0:0").unwrap()); + let epoch_schedule = *bank_forks.read().unwrap().root_bank().epoch_schedule(); + let requester_cluster_info = Arc::new(ClusterInfo::new( + Node::new_localhost().info, + Arc::new(Keypair::new()), + SocketAddrSpace::Unspecified, + )); + let requester_serve_repair = ServeRepair::new(requester_cluster_info.clone()); + let (duplicate_slots_reset_sender, _duplicate_slots_reset_receiver) = unbounded(); + let repair_info = RepairInfo { + bank_forks, + cluster_info: requester_cluster_info, + cluster_slots: Arc::new(ClusterSlots::default()), + epoch_schedule, + duplicate_slots_reset_sender, + repair_validators: None, + }; + + let (ancestor_hashes_replay_update_sender, ancestor_hashes_replay_update_receiver) = + unbounded(); + let (retryable_slots_sender, retryable_slots_receiver) = unbounded(); + Self { + ancestor_hashes_request_statuses, + ancestor_hashes_request_socket, + requester_serve_repair, + repair_info, + outstanding_requests: Arc::new(RwLock::new( + OutstandingAncestorHashesRepairs::default(), + )), + dead_slot_pool: HashSet::new(), + repairable_dead_slot_pool: HashSet::new(), + request_throttle: vec![], + repair_stats: AncestorRepairRequestsStats::default(), + _duplicate_slots_reset_receiver, + ancestor_hashes_replay_update_sender, + ancestor_hashes_replay_update_receiver, + retryable_slots_sender, + retryable_slots_receiver, + } + } + } + + fn setup_dead_slot( + dead_slot: Slot, + correct_bank_hashes: &HashMap, + ) -> ReplayBlockstoreComponents { + assert!(dead_slot >= MAX_ANCESTOR_RESPONSES as Slot); + let mut forks = tr(0); + + // Create a bank_forks that includes everything but the dead slot + for slot in 1..dead_slot { + forks.push_front(tr(slot)); + } + let mut replay_blockstore_components = replay_blockstore_components(Some(forks), 1, None); + let ReplayBlockstoreComponents { + ref blockstore, + ref mut vote_simulator, + .. + } = replay_blockstore_components; + + // Create dead slot in bank_forks + let is_frozen = false; + vote_simulator.fill_bank_forks( + tr(dead_slot - 1) / tr(dead_slot), + &HashMap::new(), + is_frozen, + ); + + /*{ + let w_bank_forks = bank_forks.write().unwrap(); + assert!(w_bank_forks.get(dead_slot).is_none()); + let parent = w_bank_forks.get(dead_slot - 1).unwrap().clone(); + let dead_bank = Bank::new_from_parent(&parent, &Pubkey::default(), dead_slot); + bank_forks.insert(dead_bank); + + }*/ + + // Create slots [slot, slot + num_ancestors) with 5 shreds apiece + let (shreds, _) = make_many_slot_entries(dead_slot, dead_slot, 5); + blockstore + .insert_shreds(shreds, None, false) + .expect("Expect successful ledger write"); + for duplicate_confirmed_slot in 0..dead_slot { + let bank_hash = correct_bank_hashes + .get(&duplicate_confirmed_slot) + .cloned() + .unwrap_or_else(Hash::new_unique); + blockstore.insert_bank_hash(duplicate_confirmed_slot, bank_hash, true); + } + blockstore.set_dead_slot(dead_slot).unwrap(); + replay_blockstore_components + } + + #[test] + fn test_ancestor_hashes_service_initiate_ancestor_hashes_requests_for_duplicate_slot() { + let dead_slot = MAX_ANCESTOR_RESPONSES as Slot; + let responder_threads = ResponderThreads::new(dead_slot); + + let ResponderThreads { + ref responder_info, + ref response_receiver, + ref correct_bank_hashes, + .. + } = responder_threads; + + let ReplayBlockstoreComponents { + blockstore: requester_blockstore, + vote_simulator, + .. + } = setup_dead_slot(dead_slot, correct_bank_hashes); + + let ManageAncestorHashesState { + ancestor_hashes_request_statuses, + ancestor_hashes_request_socket, + repair_info, + outstanding_requests, + requester_serve_repair, + mut repair_stats, + .. + } = ManageAncestorHashesState::new(vote_simulator.bank_forks); + + let RepairInfo { + cluster_info: requester_cluster_info, + cluster_slots, + repair_validators, + .. + } = repair_info; + + AncestorHashesService::initiate_ancestor_hashes_requests_for_duplicate_slot( + &ancestor_hashes_request_statuses, + &ancestor_hashes_request_socket, + &cluster_slots, + &requester_serve_repair, + &repair_validators, + dead_slot, + &mut repair_stats, + &outstanding_requests, + ); + assert!(ancestor_hashes_request_statuses.is_empty()); + + // Add the responder to the eligible list for requests + let responder_id = responder_info.id; + cluster_slots.insert_node_id(dead_slot, responder_id); + requester_cluster_info.insert_info(responder_info.clone()); + // Now the request should actually be made + AncestorHashesService::initiate_ancestor_hashes_requests_for_duplicate_slot( + &ancestor_hashes_request_statuses, + &ancestor_hashes_request_socket, + &cluster_slots, + &requester_serve_repair, + &repair_validators, + dead_slot, + &mut repair_stats, + &outstanding_requests, + ); + + assert_eq!(ancestor_hashes_request_statuses.len(), 1); + assert!(ancestor_hashes_request_statuses.contains_key(&dead_slot)); + + // Should have received valid response + let mut response_packet = response_receiver + .recv_timeout(Duration::from_millis(10_000)) + .unwrap(); + let packet = &mut response_packet.packets[0]; + packet.meta.set_addr(&responder_info.serve_repair); + let decision = AncestorHashesService::verify_and_process_ancestor_response( + packet, + &ancestor_hashes_request_statuses, + &mut AncestorHashesResponsesStats::default(), + &outstanding_requests, + &requester_blockstore, + ) + .unwrap(); + + assert_matches!( + decision, + ( + _dead_slot, + DuplicateAncestorDecision::EarliestAncestorNotFrozen(_) + ) + ); + assert_eq!( + decision + .1 + .repair_status() + .unwrap() + .correct_ancestors_to_repair, + vec![(dead_slot, *correct_bank_hashes.get(&dead_slot).unwrap())] + ); + + // Should have removed the ancestor status on successful + // completion + assert!(ancestor_hashes_request_statuses.is_empty()); + responder_threads.shutdown(); + } + + #[test] + fn test_ancestor_hashes_service_manage_ancestor_requests() { + let vote_simulator = VoteSimulator::new(3); + let ManageAncestorHashesState { + ancestor_hashes_request_statuses, + ancestor_hashes_request_socket, + requester_serve_repair, + repair_info, + outstanding_requests, + mut dead_slot_pool, + mut repairable_dead_slot_pool, + mut request_throttle, + ancestor_hashes_replay_update_sender, + ancestor_hashes_replay_update_receiver, + retryable_slots_receiver, + .. + } = ManageAncestorHashesState::new(vote_simulator.bank_forks); + let responder_node = Node::new_localhost(); + let RepairInfo { + ref bank_forks, + ref cluster_info, + .. + } = repair_info; + cluster_info.insert_info(responder_node.info); + bank_forks.read().unwrap().root_bank().epoch_schedule(); + // 1) No signals from ReplayStage, no requests should be made + AncestorHashesService::manage_ancestor_requests( + &ancestor_hashes_request_statuses, + &ancestor_hashes_request_socket, + &repair_info, + &outstanding_requests, + &ancestor_hashes_replay_update_receiver, + &retryable_slots_receiver, + &requester_serve_repair, + &mut AncestorRepairRequestsStats::default(), + &mut dead_slot_pool, + &mut repairable_dead_slot_pool, + &mut request_throttle, + ); + + assert!(dead_slot_pool.is_empty()); + assert!(repairable_dead_slot_pool.is_empty()); + assert!(ancestor_hashes_request_statuses.is_empty()); + + // 2) Simulate signals from ReplayStage, should make a request + // for `dead_duplicate_confirmed_slot` + let dead_slot = 10; + let dead_duplicate_confirmed_slot = 14; + ancestor_hashes_replay_update_sender + .send(AncestorHashesReplayUpdate::Dead(dead_slot)) + .unwrap(); + ancestor_hashes_replay_update_sender + .send(AncestorHashesReplayUpdate::DeadDuplicateConfirmed( + dead_duplicate_confirmed_slot, + )) + .unwrap(); + ancestor_hashes_replay_update_sender + .send(AncestorHashesReplayUpdate::DeadDuplicateConfirmed( + dead_duplicate_confirmed_slot, + )) + .unwrap(); + AncestorHashesService::manage_ancestor_requests( + &ancestor_hashes_request_statuses, + &ancestor_hashes_request_socket, + &repair_info, + &outstanding_requests, + &ancestor_hashes_replay_update_receiver, + &retryable_slots_receiver, + &requester_serve_repair, + &mut AncestorRepairRequestsStats::default(), + &mut dead_slot_pool, + &mut repairable_dead_slot_pool, + &mut request_throttle, + ); + + assert_eq!(dead_slot_pool.len(), 1); + assert!(dead_slot_pool.contains(&dead_slot)); + assert!(repairable_dead_slot_pool.is_empty()); + assert_eq!(ancestor_hashes_request_statuses.len(), 1); + assert!(ancestor_hashes_request_statuses.contains_key(&dead_duplicate_confirmed_slot)); + + // 3) Simulate an outstanding request timing out + ancestor_hashes_request_statuses + .get_mut(&dead_duplicate_confirmed_slot) + .unwrap() + .value_mut() + .make_expired(); + + // If the request timed out, we should remove the slot from `ancestor_hashes_request_statuses`, + // and add it to `repairable_dead_slot_pool`. Because the request_throttle is at its limit, + // we should not immediately retry the timed request. + request_throttle.resize(MAX_ANCESTOR_HASHES_SLOT_REQUESTS_PER_SECOND, std::u64::MAX); + AncestorHashesService::manage_ancestor_requests( + &ancestor_hashes_request_statuses, + &ancestor_hashes_request_socket, + &repair_info, + &outstanding_requests, + &ancestor_hashes_replay_update_receiver, + &retryable_slots_receiver, + &requester_serve_repair, + &mut AncestorRepairRequestsStats::default(), + &mut dead_slot_pool, + &mut repairable_dead_slot_pool, + &mut request_throttle, + ); + + assert_eq!(dead_slot_pool.len(), 1); + assert!(dead_slot_pool.contains(&dead_slot)); + assert_eq!(repairable_dead_slot_pool.len(), 1); + assert!(repairable_dead_slot_pool.contains(&dead_duplicate_confirmed_slot)); + assert!(ancestor_hashes_request_statuses.is_empty()); + + // 4) If the throttle only has expired timestamps from more than a second ago, + // then on the next iteration, we should clear the entries in the throttle + // and retry a request for the timed out request + request_throttle.clear(); + request_throttle.resize( + MAX_ANCESTOR_HASHES_SLOT_REQUESTS_PER_SECOND, + timestamp() - 1001, + ); + AncestorHashesService::manage_ancestor_requests( + &ancestor_hashes_request_statuses, + &ancestor_hashes_request_socket, + &repair_info, + &outstanding_requests, + &ancestor_hashes_replay_update_receiver, + &retryable_slots_receiver, + &requester_serve_repair, + &mut AncestorRepairRequestsStats::default(), + &mut dead_slot_pool, + &mut repairable_dead_slot_pool, + &mut request_throttle, + ); + assert_eq!(dead_slot_pool.len(), 1); + assert!(dead_slot_pool.contains(&dead_slot)); + assert!(repairable_dead_slot_pool.is_empty()); + assert_eq!(ancestor_hashes_request_statuses.len(), 1); + assert!(ancestor_hashes_request_statuses.contains_key(&dead_duplicate_confirmed_slot)); + // Request throttle includes one item for the request we just made + assert_eq!( + request_throttle.len(), + ancestor_hashes_request_statuses.len() + ); + + // 5) If we've reached the throttle limit, no requests should be made, + // but should still read off the channel for replay updates + request_throttle.clear(); + request_throttle.resize(MAX_ANCESTOR_HASHES_SLOT_REQUESTS_PER_SECOND, std::u64::MAX); + let dead_duplicate_confirmed_slot_2 = 15; + ancestor_hashes_replay_update_sender + .send(AncestorHashesReplayUpdate::DeadDuplicateConfirmed( + dead_duplicate_confirmed_slot_2, + )) + .unwrap(); + AncestorHashesService::manage_ancestor_requests( + &ancestor_hashes_request_statuses, + &ancestor_hashes_request_socket, + &repair_info, + &outstanding_requests, + &ancestor_hashes_replay_update_receiver, + &retryable_slots_receiver, + &requester_serve_repair, + &mut AncestorRepairRequestsStats::default(), + &mut dead_slot_pool, + &mut repairable_dead_slot_pool, + &mut request_throttle, + ); + + assert_eq!(dead_slot_pool.len(), 1); + assert!(dead_slot_pool.contains(&dead_slot)); + assert_eq!(repairable_dead_slot_pool.len(), 1); + assert!(repairable_dead_slot_pool.contains(&dead_duplicate_confirmed_slot_2)); + assert_eq!(ancestor_hashes_request_statuses.len(), 1); + assert!(ancestor_hashes_request_statuses.contains_key(&dead_duplicate_confirmed_slot)); + + // 6) If root moves past slot, should remove it from all state + let bank_forks = &repair_info.bank_forks; + let root_bank = bank_forks.read().unwrap().root_bank(); + let new_root_slot = dead_duplicate_confirmed_slot_2 + 1; + let new_root_bank = Bank::new_from_parent(&root_bank, &Pubkey::default(), new_root_slot); + new_root_bank.freeze(); + { + let mut w_bank_forks = bank_forks.write().unwrap(); + w_bank_forks.insert(new_root_bank); + w_bank_forks.set_root(new_root_slot, &AbsRequestSender::default(), None); + } + assert!(!dead_slot_pool.is_empty()); + assert!(!repairable_dead_slot_pool.is_empty()); + assert!(!ancestor_hashes_request_statuses.is_empty()); + request_throttle.clear(); + AncestorHashesService::manage_ancestor_requests( + &ancestor_hashes_request_statuses, + &ancestor_hashes_request_socket, + &repair_info, + &outstanding_requests, + &ancestor_hashes_replay_update_receiver, + &retryable_slots_receiver, + &requester_serve_repair, + &mut AncestorRepairRequestsStats::default(), + &mut dead_slot_pool, + &mut repairable_dead_slot_pool, + &mut request_throttle, + ); + assert!(dead_slot_pool.is_empty()); + assert!(repairable_dead_slot_pool.is_empty()); + assert!(ancestor_hashes_request_statuses.is_empty()); + } + + #[test] + fn test_ancestor_hashes_service_manage_ancestor_hashes_after_replay_dump() { + let dead_slot = MAX_ANCESTOR_RESPONSES as Slot; + let responder_threads = ResponderThreads::new(dead_slot); + + let ResponderThreads { + ref responder_info, + ref response_receiver, + ref correct_bank_hashes, + .. + } = responder_threads; + + let ReplayBlockstoreComponents { + blockstore: requester_blockstore, + vote_simulator, + .. + } = setup_dead_slot(dead_slot, correct_bank_hashes); + + let VoteSimulator { + bank_forks, + mut progress, + .. + } = vote_simulator; + + let ManageAncestorHashesState { + ancestor_hashes_request_statuses, + ancestor_hashes_request_socket, + requester_serve_repair, + repair_info, + outstanding_requests, + mut dead_slot_pool, + mut repairable_dead_slot_pool, + mut request_throttle, + ancestor_hashes_replay_update_sender, + ancestor_hashes_replay_update_receiver, + retryable_slots_receiver, + .. + } = ManageAncestorHashesState::new(bank_forks.clone()); + + let RepairInfo { + cluster_info: ref requester_cluster_info, + ref cluster_slots, + .. + } = repair_info; + + // Add the responder to the eligible list for requests + let responder_id = responder_info.id; + cluster_slots.insert_node_id(dead_slot, responder_id); + requester_cluster_info.insert_info(responder_info.clone()); + + // Simulate getting duplicate confirmed dead slot + ancestor_hashes_replay_update_sender + .send(AncestorHashesReplayUpdate::DeadDuplicateConfirmed( + dead_slot, + )) + .unwrap(); + + // Simulate Replay dumping this slot + let mut duplicate_slots_to_repair = DuplicateSlotsToRepair::default(); + duplicate_slots_to_repair.insert(dead_slot, Hash::new_unique()); + ReplayStage::dump_then_repair_correct_slots( + &mut duplicate_slots_to_repair, + &mut bank_forks.read().unwrap().ancestors(), + &mut bank_forks.read().unwrap().descendants().clone(), + &mut progress, + &bank_forks, + &requester_blockstore, + None, + ); + + // Simulate making a request + AncestorHashesService::manage_ancestor_requests( + &ancestor_hashes_request_statuses, + &ancestor_hashes_request_socket, + &repair_info, + &outstanding_requests, + &ancestor_hashes_replay_update_receiver, + &retryable_slots_receiver, + &requester_serve_repair, + &mut AncestorRepairRequestsStats::default(), + &mut dead_slot_pool, + &mut repairable_dead_slot_pool, + &mut request_throttle, + ); + + assert_eq!(ancestor_hashes_request_statuses.len(), 1); + assert!(ancestor_hashes_request_statuses.contains_key(&dead_slot)); + + // Should have received valid response + let mut response_packet = response_receiver + .recv_timeout(Duration::from_millis(10_000)) + .unwrap(); + let packet = &mut response_packet.packets[0]; + packet.meta.set_addr(&responder_info.serve_repair); + let decision = AncestorHashesService::verify_and_process_ancestor_response( + packet, + &ancestor_hashes_request_statuses, + &mut AncestorHashesResponsesStats::default(), + &outstanding_requests, + &requester_blockstore, + ) + .unwrap(); + + assert_matches!( + decision, + ( + _dead_slot, + DuplicateAncestorDecision::EarliestAncestorNotFrozen(_) + ) + ); + assert_eq!( + decision + .1 + .repair_status() + .unwrap() + .correct_ancestors_to_repair, + vec![(dead_slot, *correct_bank_hashes.get(&dead_slot).unwrap())] + ); + + // Should have removed the ancestor status on successful + // completion + assert!(ancestor_hashes_request_statuses.is_empty()); + responder_threads.shutdown(); + } + + #[test] + fn test_ancestor_hashes_service_retryable_duplicate_ancestor_decision() { + let vote_simulator = VoteSimulator::new(1); + let ManageAncestorHashesState { + ancestor_hashes_request_statuses, + ancestor_hashes_request_socket, + requester_serve_repair, + repair_info, + outstanding_requests, + mut dead_slot_pool, + mut repairable_dead_slot_pool, + mut request_throttle, + ancestor_hashes_replay_update_receiver, + retryable_slots_receiver, + retryable_slots_sender, + .. + } = ManageAncestorHashesState::new(vote_simulator.bank_forks); + + let decision = DuplicateAncestorDecision::SampleNotDuplicateConfirmed; + assert!(decision.is_retryable()); + + // Simulate network response processing thread reaching a retryable + // decision + let request_slot = 10; + AncestorHashesService::handle_ancestor_request_decision( + request_slot, + decision, + &repair_info.duplicate_slots_reset_sender, + &retryable_slots_sender, + ); + + // Simulate ancestor request thread getting the retry signal + assert!(dead_slot_pool.is_empty()); + assert!(repairable_dead_slot_pool.is_empty()); + AncestorHashesService::manage_ancestor_requests( + &ancestor_hashes_request_statuses, + &ancestor_hashes_request_socket, + &repair_info, + &outstanding_requests, + &ancestor_hashes_replay_update_receiver, + &retryable_slots_receiver, + &requester_serve_repair, + &mut AncestorRepairRequestsStats::default(), + &mut dead_slot_pool, + &mut repairable_dead_slot_pool, + &mut request_throttle, + ); + + assert!(dead_slot_pool.is_empty()); + assert!(repairable_dead_slot_pool.contains(&request_slot)); + } +} diff --git a/core/src/banking_stage.rs b/core/src/banking_stage.rs index 90717ac8c42..32e4381e651 100644 --- a/core/src/banking_stage.rs +++ b/core/src/banking_stage.rs @@ -14,7 +14,7 @@ use { solana_perf::{ cuda_runtime::PinnedVec, data_budget::DataBudget, - packet::{limited_deserialize, Packet, Packets, PACKETS_PER_BATCH}, + packet::{limited_deserialize, Packet, PacketBatch, PACKETS_PER_BATCH}, perf_libs, }, solana_poh::poh_recorder::{PohRecorder, PohRecorderError, TransactionRecorder}, @@ -65,10 +65,10 @@ use { }; /// (packets, valid_indexes, forwarded) -/// Set of packets with a list of which are valid and if this batch has been forwarded. -type PacketsAndOffsets = (Packets, Vec, bool); +/// Batch of packets with a list of which are valid and if this batch has been forwarded. +type PacketBatchAndOffsets = (PacketBatch, Vec, bool); -pub type UnprocessedPackets = VecDeque; +pub type UnprocessedPacketBatches = VecDeque; /// Transaction forwarding pub const FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET: u64 = 2; @@ -287,9 +287,9 @@ impl BankingStage { pub fn new( cluster_info: &Arc, poh_recorder: &Arc>, - verified_receiver: CrossbeamReceiver>, - tpu_verified_vote_receiver: CrossbeamReceiver>, - verified_vote_receiver: CrossbeamReceiver>, + verified_receiver: CrossbeamReceiver>, + tpu_verified_vote_receiver: CrossbeamReceiver>, + verified_vote_receiver: CrossbeamReceiver>, transaction_status_sender: Option, gossip_vote_sender: ReplayVoteSender, cost_model: Arc>, @@ -310,9 +310,9 @@ impl BankingStage { fn new_num_threads( cluster_info: &Arc, poh_recorder: &Arc>, - verified_receiver: CrossbeamReceiver>, - tpu_verified_vote_receiver: CrossbeamReceiver>, - verified_vote_receiver: CrossbeamReceiver>, + verified_receiver: CrossbeamReceiver>, + tpu_verified_vote_receiver: CrossbeamReceiver>, + verified_vote_receiver: CrossbeamReceiver>, num_threads: u32, transaction_status_sender: Option, gossip_vote_sender: ReplayVoteSender, @@ -379,12 +379,12 @@ impl BankingStage { } fn filter_valid_packets_for_forwarding<'a>( - all_packets: impl Iterator, + packet_batches: impl Iterator, ) -> Vec<&'a Packet> { - all_packets - .filter(|(_p, _indexes, forwarded)| !forwarded) - .flat_map(|(p, valid_indexes, _forwarded)| { - valid_indexes.iter().map(move |x| &p.packets[*x]) + packet_batches + .filter(|(_batch, _indexes, forwarded)| !forwarded) + .flat_map(|(batch, valid_indexes, _forwarded)| { + valid_indexes.iter().map(move |x| &batch.packets[*x]) }) .collect() } @@ -392,10 +392,10 @@ impl BankingStage { fn forward_buffered_packets( socket: &std::net::UdpSocket, tpu_forwards: &std::net::SocketAddr, - unprocessed_packets: &UnprocessedPackets, + buffered_packet_batches: &UnprocessedPacketBatches, data_budget: &DataBudget, ) -> std::io::Result<()> { - let packets = Self::filter_valid_packets_for_forwarding(unprocessed_packets.iter()); + let packets = Self::filter_valid_packets_for_forwarding(buffered_packet_batches.iter()); inc_new_counter_info!("banking_stage-forwarded_packets", packets.len()); const INTERVAL_MS: u64 = 100; const MAX_BYTES_PER_SECOND: usize = 10_000 * 1200; @@ -413,7 +413,7 @@ impl BankingStage { Ok(()) } - // Returns whether the given `Packets` has any more remaining unprocessed + // Returns whether the given `PacketBatch` has any more remaining unprocessed // transactions fn update_buffered_packets_with_new_unprocessed( original_unprocessed_indexes: &mut Vec, @@ -432,7 +432,7 @@ impl BankingStage { my_pubkey: &Pubkey, max_tx_ingestion_ns: u128, poh_recorder: &Arc>, - buffered_packets: &mut UnprocessedPackets, + buffered_packet_batches: &mut UnprocessedPacketBatches, transaction_status_sender: Option, gossip_vote_sender: &ReplayVoteSender, test_fn: Option, @@ -440,19 +440,21 @@ impl BankingStage { recorder: &TransactionRecorder, cost_model: &Arc>, ) { - let mut rebuffered_packets_len = 0; + let mut rebuffered_packet_count = 0; let mut new_tx_count = 0; - let buffered_len = buffered_packets.len(); + let buffered_packet_batches_len = buffered_packet_batches.len(); let mut proc_start = Measure::start("consume_buffered_process"); let mut reached_end_of_slot = None; - buffered_packets.retain_mut(|(msgs, ref mut original_unprocessed_indexes, _forwarded)| { + buffered_packet_batches.retain_mut(|buffered_packet_batch_and_offsets| { + let (packet_batch, ref mut original_unprocessed_indexes, _forwarded) = + buffered_packet_batch_and_offsets; if let Some((next_leader, bank)) = &reached_end_of_slot { // We've hit the end of this slot, no need to perform more processing, // just filter the remaining packets for the invalid (e.g. too old) ones let new_unprocessed_indexes = Self::filter_unprocessed_packets( bank, - msgs, + packet_batch, original_unprocessed_indexes, my_pubkey, *next_leader, @@ -471,7 +473,7 @@ impl BankingStage { &bank, &bank_creation_time, recorder, - msgs, + packet_batch, original_unprocessed_indexes.to_owned(), transaction_status_sender.clone(), gossip_vote_sender, @@ -490,7 +492,7 @@ impl BankingStage { new_tx_count += processed; // Out of the buffered packets just retried, collect any still unprocessed // transactions in this batch for forwarding - rebuffered_packets_len += new_unprocessed_indexes.len(); + rebuffered_packet_count += new_unprocessed_indexes.len(); let has_more_unprocessed_transactions = Self::update_buffered_packets_with_new_unprocessed( original_unprocessed_indexes, @@ -501,7 +503,7 @@ impl BankingStage { } has_more_unprocessed_transactions } else { - rebuffered_packets_len += original_unprocessed_indexes.len(); + rebuffered_packet_count += original_unprocessed_indexes.len(); // `original_unprocessed_indexes` must have remaining packets to process // if not yet processed. assert!(Self::packet_has_more_unprocessed_transactions( @@ -517,7 +519,7 @@ impl BankingStage { debug!( "@{:?} done processing buffered batches: {} time: {:?}ms tx count: {} tx/s: {}", timestamp(), - buffered_len, + buffered_packet_batches_len, proc_start.as_ms(), new_tx_count, (new_tx_count as f32) / (proc_start.as_s()) @@ -528,7 +530,7 @@ impl BankingStage { .fetch_add(proc_start.as_us(), Ordering::Relaxed); banking_stage_stats .rebuffered_packets_count - .fetch_add(rebuffered_packets_len, Ordering::Relaxed); + .fetch_add(rebuffered_packet_count, Ordering::Relaxed); banking_stage_stats .consumed_buffered_packets_count .fetch_add(new_tx_count, Ordering::Relaxed); @@ -573,7 +575,7 @@ impl BankingStage { socket: &std::net::UdpSocket, poh_recorder: &Arc>, cluster_info: &ClusterInfo, - buffered_packets: &mut UnprocessedPackets, + buffered_packet_batches: &mut UnprocessedPacketBatches, forward_option: &ForwardOption, transaction_status_sender: Option, gossip_vote_sender: &ReplayVoteSender, @@ -615,7 +617,7 @@ impl BankingStage { my_pubkey, max_tx_ingestion_ns, poh_recorder, - buffered_packets, + buffered_packet_batches, transaction_status_sender, gossip_vote_sender, None::>, @@ -628,7 +630,7 @@ impl BankingStage { Self::handle_forwarding( forward_option, cluster_info, - buffered_packets, + buffered_packet_batches, poh_recorder, socket, false, @@ -639,7 +641,7 @@ impl BankingStage { Self::handle_forwarding( forward_option, cluster_info, - buffered_packets, + buffered_packet_batches, poh_recorder, socket, true, @@ -654,7 +656,7 @@ impl BankingStage { fn handle_forwarding( forward_option: &ForwardOption, cluster_info: &ClusterInfo, - buffered_packets: &mut UnprocessedPackets, + buffered_packet_batches: &mut UnprocessedPacketBatches, poh_recorder: &Arc>, socket: &UdpSocket, hold: bool, @@ -663,7 +665,7 @@ impl BankingStage { let addr = match forward_option { ForwardOption::NotForward => { if !hold { - buffered_packets.clear(); + buffered_packet_batches.clear(); } return; } @@ -676,21 +678,26 @@ impl BankingStage { Some(addr) => addr, None => return, }; - let _ = Self::forward_buffered_packets(socket, &addr, buffered_packets, data_budget); + let _ = Self::forward_buffered_packets(socket, &addr, buffered_packet_batches, data_budget); if hold { - buffered_packets.retain(|(_, index, _)| !index.is_empty()); - for (_, _, forwarded) in buffered_packets.iter_mut() { + buffered_packet_batches.retain(|(_, index, _)| !index.is_empty()); + for (_, _, forwarded) in buffered_packet_batches.iter_mut() { *forwarded = true; } } else { - buffered_packets.clear(); + buffered_packet_batches.clear(); } } #[allow(clippy::too_many_arguments)] +<<<<<<< HEAD pub fn process_loop( my_pubkey: Pubkey, verified_receiver: &CrossbeamReceiver>, +======= + fn process_loop( + verified_receiver: &CrossbeamReceiver>, +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) poh_recorder: &Arc>, cluster_info: &ClusterInfo, recv_start: &mut Instant, @@ -705,16 +712,21 @@ impl BankingStage { ) { let recorder = poh_recorder.lock().unwrap().recorder(); let socket = UdpSocket::bind("0.0.0.0:0").unwrap(); - let mut buffered_packets = VecDeque::with_capacity(batch_limit); + let mut buffered_packet_batches = VecDeque::with_capacity(batch_limit); let banking_stage_stats = BankingStageStats::new(id); loop { +<<<<<<< HEAD while !buffered_packets.is_empty() { +======= + let my_pubkey = cluster_info.id(); + while !buffered_packet_batches.is_empty() { +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) let decision = Self::process_buffered_packets( &my_pubkey, &socket, poh_recorder, cluster_info, - &mut buffered_packets, + &mut buffered_packet_batches, &forward_option, transaction_status_sender.clone(), &gossip_vote_sender, @@ -732,7 +744,7 @@ impl BankingStage { } } - let recv_timeout = if !buffered_packets.is_empty() { + let recv_timeout = if !buffered_packet_batches.is_empty() { // If packets are buffered, let's wait for less time on recv from the channel. // This helps detect the next leader faster, and processing the buffered // packets quickly @@ -752,7 +764,7 @@ impl BankingStage { batch_limit, transaction_status_sender.clone(), &gossip_vote_sender, - &mut buffered_packets, + &mut buffered_packet_batches, &banking_stage_stats, duplicates, &recorder, @@ -1091,7 +1103,7 @@ impl BankingStage { // Also returned is packet indexes for transaction should be retried due to cost limits. #[allow(clippy::needless_collect)] fn transactions_from_packets( - msgs: &Packets, + packet_batch: &PacketBatch, transaction_indexes: &[usize], feature_set: &Arc, read_cost_tracker: &RwLockReadGuard, @@ -1105,7 +1117,7 @@ impl BankingStage { let verified_transactions_with_packet_indexes: Vec<_> = transaction_indexes .iter() .filter_map(|tx_index| { - let p = &msgs.packets[*tx_index]; + let p = &packet_batch.packets[*tx_index]; if votes_only && !p.meta.is_simple_vote_tx { return None; } @@ -1225,7 +1237,7 @@ impl BankingStage { bank: &Arc, bank_creation_time: &Instant, poh: &TransactionRecorder, - msgs: &Packets, + packet_batch: &PacketBatch, packet_indexes: Vec, transaction_status_sender: Option, gossip_vote_sender: &ReplayVoteSender, @@ -1233,6 +1245,7 @@ impl BankingStage { cost_model: &Arc>, ) -> (usize, usize, Vec) { let mut packet_conversion_time = Measure::start("packet_conversion"); +<<<<<<< HEAD let (transactions, transaction_to_packet_indexes, retryable_packet_indexes) = Self::transactions_from_packets( msgs, @@ -1244,6 +1257,14 @@ impl BankingStage { bank.vote_only_bank(), cost_model, ); +======= + let (transactions, transaction_to_packet_indexes) = Self::transactions_from_packets( + packet_batch, + &packet_indexes, + &bank.feature_set, + bank.vote_only_bank(), + ); +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) packet_conversion_time.stop(); inc_new_counter_info!("banking_stage-packet_conversion", 1); @@ -1326,7 +1347,7 @@ impl BankingStage { fn filter_unprocessed_packets( bank: &Arc, - msgs: &Packets, + packet_batch: &PacketBatch, transaction_indexes: &[usize], my_pubkey: &Pubkey, next_leader: Option, @@ -1344,6 +1365,7 @@ impl BankingStage { let mut unprocessed_packet_conversion_time = Measure::start("unprocessed_packet_conversion"); +<<<<<<< HEAD let (transactions, transaction_to_packet_indexes, retry_packet_indexes) = Self::transactions_from_packets( msgs, @@ -1355,6 +1377,14 @@ impl BankingStage { bank.vote_only_bank(), cost_model, ); +======= + let (transactions, transaction_to_packet_indexes) = Self::transactions_from_packets( + packet_batch, + transaction_indexes, + &bank.feature_set, + bank.vote_only_bank(), + ); +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) unprocessed_packet_conversion_time.stop(); let tx_count = transaction_to_packet_indexes.len(); @@ -1402,7 +1432,7 @@ impl BankingStage { /// Process the incoming packets pub fn process_packets( my_pubkey: &Pubkey, - verified_receiver: &CrossbeamReceiver>, + verified_receiver: &CrossbeamReceiver>, poh: &Arc>, recv_start: &mut Instant, recv_timeout: Duration, @@ -1410,40 +1440,48 @@ impl BankingStage { batch_limit: usize, transaction_status_sender: Option, gossip_vote_sender: &ReplayVoteSender, - buffered_packets: &mut UnprocessedPackets, + buffered_packet_batches: &mut UnprocessedPacketBatches, banking_stage_stats: &BankingStageStats, duplicates: &Arc, PacketHasher)>>, recorder: &TransactionRecorder, cost_model: &Arc>, ) -> Result<(), RecvTimeoutError> { let mut recv_time = Measure::start("process_packets_recv"); - let mms = verified_receiver.recv_timeout(recv_timeout)?; + let packet_batches = verified_receiver.recv_timeout(recv_timeout)?; recv_time.stop(); - let mms_len = mms.len(); - let count: usize = mms.iter().map(|x| x.packets.len()).sum(); + let packet_batches_len = packet_batches.len(); + let packet_count: usize = packet_batches.iter().map(|x| x.packets.len()).sum(); debug!( "@{:?} process start stalled for: {:?}ms txs: {} id: {}", timestamp(), duration_as_ms(&recv_start.elapsed()), - count, + packet_count, id, ); - inc_new_counter_debug!("banking_stage-transactions_received", count); + inc_new_counter_debug!("banking_stage-transactions_received", packet_count); let mut proc_start = Measure::start("process_packets_transactions_process"); let mut new_tx_count = 0; - let mut mms_iter = mms.into_iter(); + let mut packet_batch_iter = packet_batches.into_iter(); let mut dropped_packets_count = 0; let mut dropped_packet_batches_count = 0; let mut newly_buffered_packets_count = 0; +<<<<<<< HEAD while let Some(msgs) = mms_iter.next() { let packet_indexes = Self::generate_packet_indexes(&msgs.packets); let bank_start = poh.lock().unwrap().bank_start(); if PohRecorder::get_bank_still_processing_txs(&bank_start).is_none() { +======= + while let Some(packet_batch) = packet_batch_iter.next() { + let packet_indexes = Self::generate_packet_indexes(&packet_batch.packets); + let poh_recorder_bank = poh.lock().unwrap().get_poh_recorder_bank(); + let working_bank_start = poh_recorder_bank.working_bank_start(); + if PohRecorder::get_working_bank_if_not_expired(&working_bank_start).is_none() { +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) Self::push_unprocessed( - buffered_packets, - msgs, + buffered_packet_batches, + packet_batch, packet_indexes, &mut dropped_packet_batches_count, &mut dropped_packets_count, @@ -1461,7 +1499,7 @@ impl BankingStage { &bank, &bank_creation_time, recorder, - &msgs, + &packet_batch, packet_indexes, transaction_status_sender.clone(), gossip_vote_sender, @@ -1473,8 +1511,8 @@ impl BankingStage { // Collect any unprocessed transactions in this batch for forwarding Self::push_unprocessed( - buffered_packets, - msgs, + buffered_packet_batches, + packet_batch, unprocessed_indexes, &mut dropped_packet_batches_count, &mut dropped_packets_count, @@ -1490,11 +1528,16 @@ impl BankingStage { let next_leader = poh.lock().unwrap().next_slot_leader(); // Walk thru rest of the transactions and filter out the invalid (e.g. too old) ones #[allow(clippy::while_let_on_iterator)] - while let Some(msgs) = mms_iter.next() { - let packet_indexes = Self::generate_packet_indexes(&msgs.packets); + while let Some(packet_batch) = packet_batch_iter.next() { + let packet_indexes = Self::generate_packet_indexes(&packet_batch.packets); let unprocessed_indexes = Self::filter_unprocessed_packets( +<<<<<<< HEAD &bank, &msgs, +======= + working_bank, + &packet_batch, +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) &packet_indexes, my_pubkey, next_leader, @@ -1502,8 +1545,8 @@ impl BankingStage { cost_model, ); Self::push_unprocessed( - buffered_packets, - msgs, + buffered_packet_batches, + packet_batch, unprocessed_indexes, &mut dropped_packet_batches_count, &mut dropped_packets_count, @@ -1524,11 +1567,11 @@ impl BankingStage { debug!( "@{:?} done processing transaction batches: {} time: {:?}ms tx count: {} tx/s: {} total count: {} id: {}", timestamp(), - mms_len, + packet_batches_len, proc_start.as_ms(), new_tx_count, (new_tx_count as f32) / (proc_start.as_s()), - count, + packet_count, id, ); banking_stage_stats @@ -1536,7 +1579,7 @@ impl BankingStage { .fetch_add(proc_start.as_us(), Ordering::Relaxed); banking_stage_stats .process_packets_count - .fetch_add(count, Ordering::Relaxed); + .fetch_add(packet_count, Ordering::Relaxed); banking_stage_stats .new_tx_count .fetch_add(new_tx_count, Ordering::Relaxed); @@ -1551,9 +1594,12 @@ impl BankingStage { .fetch_add(newly_buffered_packets_count, Ordering::Relaxed); banking_stage_stats .current_buffered_packet_batches_count - .swap(buffered_packets.len(), Ordering::Relaxed); + .swap(buffered_packet_batches.len(), Ordering::Relaxed); banking_stage_stats.current_buffered_packets_count.swap( - buffered_packets.iter().map(|packets| packets.1.len()).sum(), + buffered_packet_batches + .iter() + .map(|packets| packets.1.len()) + .sum(), Ordering::Relaxed, ); *recv_start = Instant::now(); @@ -1561,8 +1607,8 @@ impl BankingStage { } fn push_unprocessed( - unprocessed_packets: &mut UnprocessedPackets, - packets: Packets, + unprocessed_packet_batches: &mut UnprocessedPacketBatches, + packet_batch: PacketBatch, mut packet_indexes: Vec, dropped_packet_batches_count: &mut usize, dropped_packets_count: &mut usize, @@ -1577,7 +1623,7 @@ impl BankingStage { let mut duplicates = duplicates.lock().unwrap(); let (cache, hasher) = duplicates.deref_mut(); packet_indexes.retain(|i| { - let packet_hash = hasher.hash_packet(&packets.packets[*i]); + let packet_hash = hasher.hash_packet(&packet_batch.packets[*i]); match cache.get_mut(&packet_hash) { Some(_hash) => false, None => { @@ -1598,14 +1644,14 @@ impl BankingStage { ); } if Self::packet_has_more_unprocessed_transactions(&packet_indexes) { - if unprocessed_packets.len() >= batch_limit { + if unprocessed_packet_batches.len() >= batch_limit { *dropped_packet_batches_count += 1; - if let Some(dropped_batch) = unprocessed_packets.pop_front() { + if let Some(dropped_batch) = unprocessed_packet_batches.pop_front() { *dropped_packets_count += dropped_batch.1.len(); } } *newly_buffered_packets_count += packet_indexes.len(); - unprocessed_packets.push_back((packets, packet_indexes, false)); + unprocessed_packet_batches.push_back((packet_batch, packet_indexes, false)); } } @@ -1675,7 +1721,7 @@ mod tests { get_tmp_ledger_path, leader_schedule_cache::LeaderScheduleCache, }, - solana_perf::packet::to_packets_chunked, + solana_perf::packet::to_packet_batches, solana_poh::{ poh_recorder::{create_test_recorder, Record, WorkingBank, WorkingBankEntry}, poh_service::PohService, @@ -1812,7 +1858,9 @@ mod tests { Blockstore::destroy(&ledger_path).unwrap(); } - pub fn convert_from_old_verified(mut with_vers: Vec<(Packets, Vec)>) -> Vec { + pub fn convert_from_old_verified( + mut with_vers: Vec<(PacketBatch, Vec)>, + ) -> Vec { with_vers.iter_mut().for_each(|(b, v)| { b.packets .iter_mut() @@ -1884,18 +1932,18 @@ mod tests { let tx_anf = system_transaction::transfer(&keypair, &to3, 1, start_hash); // send 'em over - let packets = to_packets_chunked(&[tx_no_ver, tx_anf, tx], 3); + let packet_batches = to_packet_batches(&[tx_no_ver, tx_anf, tx], 3); // glad they all fit - assert_eq!(packets.len(), 1); + assert_eq!(packet_batches.len(), 1); - let packets = packets + let packet_batches = packet_batches .into_iter() - .map(|packets| (packets, vec![0u8, 1u8, 1u8])) + .map(|batch| (batch, vec![0u8, 1u8, 1u8])) .collect(); - let packets = convert_from_old_verified(packets); + let packet_batches = convert_from_old_verified(packet_batches); verified_sender // no_ver, anf, tx - .send(packets) + .send(packet_batches) .unwrap(); drop(verified_sender); @@ -1961,24 +2009,24 @@ mod tests { let tx = system_transaction::transfer(&mint_keypair, &alice.pubkey(), 2, genesis_config.hash()); - let packets = to_packets_chunked(&[tx], 1); - let packets = packets + let packet_batches = to_packet_batches(&[tx], 1); + let packet_batches = packet_batches .into_iter() - .map(|packets| (packets, vec![1u8])) + .map(|batch| (batch, vec![1u8])) .collect(); - let packets = convert_from_old_verified(packets); - verified_sender.send(packets).unwrap(); + let packet_batches = convert_from_old_verified(packet_batches); + verified_sender.send(packet_batches).unwrap(); // Process a second batch that uses the same from account, so conflicts with above TX let tx = system_transaction::transfer(&mint_keypair, &alice.pubkey(), 1, genesis_config.hash()); - let packets = to_packets_chunked(&[tx], 1); - let packets = packets + let packet_batches = to_packet_batches(&[tx], 1); + let packet_batches = packet_batches .into_iter() - .map(|packets| (packets, vec![1u8])) + .map(|batch| (batch, vec![1u8])) .collect(); - let packets = convert_from_old_verified(packets); - verified_sender.send(packets).unwrap(); + let packet_batches = convert_from_old_verified(packet_batches); + verified_sender.send(packet_batches).unwrap(); let (vote_sender, vote_receiver) = unbounded(); let (tpu_vote_sender, tpu_vote_receiver) = unbounded(); @@ -2520,9 +2568,9 @@ mod tests { fn test_filter_valid_packets() { solana_logger::setup(); - let mut all_packets = (0..16) + let mut packet_batches = (0..16) .map(|packets_id| { - let packets = Packets::new( + let packet_batch = PacketBatch::new( (0..32) .map(|packet_id| { let mut p = Packet::default(); @@ -2534,11 +2582,11 @@ mod tests { let valid_indexes = (0..32) .filter_map(|x| if x % 2 != 0 { Some(x as usize) } else { None }) .collect_vec(); - (packets, valid_indexes, false) + (packet_batch, valid_indexes, false) }) .collect_vec(); - let result = BankingStage::filter_valid_packets_for_forwarding(all_packets.iter()); + let result = BankingStage::filter_valid_packets_for_forwarding(packet_batches.iter()); assert_eq!(result.len(), 256); @@ -2552,8 +2600,8 @@ mod tests { }) .collect_vec(); - all_packets[0].2 = true; - let result = BankingStage::filter_valid_packets_for_forwarding(all_packets.iter()); + packet_batches[0].2 = true; + let result = BankingStage::filter_valid_packets_for_forwarding(packet_batches.iter()); assert_eq!(result.len(), 240); } @@ -2807,12 +2855,15 @@ mod tests { setup_conflicting_transactions(&ledger_path); let recorder = poh_recorder.lock().unwrap().recorder(); let num_conflicting_transactions = transactions.len(); - let mut packets_vec = to_packets_chunked(&transactions, num_conflicting_transactions); - assert_eq!(packets_vec.len(), 1); - assert_eq!(packets_vec[0].packets.len(), num_conflicting_transactions); - let all_packets = packets_vec.pop().unwrap(); - let mut buffered_packets: UnprocessedPackets = vec![( - all_packets, + let mut packet_batches = to_packet_batches(&transactions, num_conflicting_transactions); + assert_eq!(packet_batches.len(), 1); + assert_eq!( + packet_batches[0].packets.len(), + num_conflicting_transactions + ); + let packet_batch = packet_batches.pop().unwrap(); + let mut buffered_packet_batches: UnprocessedPacketBatches = vec![( + packet_batch, (0..num_conflicting_transactions).into_iter().collect(), false, )] @@ -2828,7 +2879,7 @@ mod tests { &Pubkey::default(), max_tx_processing_ns, &poh_recorder, - &mut buffered_packets, + &mut buffered_packet_batches, None, &gossip_vote_sender, None::>, @@ -2836,7 +2887,10 @@ mod tests { &recorder, &Arc::new(RwLock::new(CostModel::default())), ); - assert_eq!(buffered_packets[0].1.len(), num_conflicting_transactions); + assert_eq!( + buffered_packet_batches[0].1.len(), + num_conflicting_transactions + ); // When the poh recorder has a bank, should process all non conflicting buffered packets. // Processes one packet per iteration of the loop for num_expected_unprocessed in (0..num_conflicting_transactions).rev() { @@ -2845,7 +2899,7 @@ mod tests { &Pubkey::default(), max_tx_processing_ns, &poh_recorder, - &mut buffered_packets, + &mut buffered_packet_batches, None, &gossip_vote_sender, None::>, @@ -2854,9 +2908,9 @@ mod tests { &Arc::new(RwLock::new(CostModel::default())), ); if num_expected_unprocessed == 0 { - assert!(buffered_packets.is_empty()) + assert!(buffered_packet_batches.is_empty()) } else { - assert_eq!(buffered_packets[0].1.len(), num_expected_unprocessed); + assert_eq!(buffered_packet_batches[0].1.len(), num_expected_unprocessed); } } poh_recorder @@ -2876,12 +2930,12 @@ mod tests { let (transactions, bank, poh_recorder, _entry_receiver, poh_simulator) = setup_conflicting_transactions(&ledger_path); let num_conflicting_transactions = transactions.len(); - let packets_vec = to_packets_chunked(&transactions, 1); - assert_eq!(packets_vec.len(), num_conflicting_transactions); - for single_packets in &packets_vec { - assert_eq!(single_packets.packets.len(), 1); + let packet_batches = to_packet_batches(&transactions, 1); + assert_eq!(packet_batches.len(), num_conflicting_transactions); + for single_packet_batch in &packet_batches { + assert_eq!(single_packet_batch.packets.len(), 1); } - let mut buffered_packets: UnprocessedPackets = packets_vec + let mut buffered_packet_batches: UnprocessedPacketBatches = packet_batches .clone() .into_iter() .map(|single_packets| (single_packets, vec![0], false)) @@ -2895,8 +2949,8 @@ mod tests { continue_receiver.recv().unwrap(); }); // When the poh recorder has a bank, it should process all non conflicting buffered packets. - // Because each conflicting transaction is in it's own `Packet` within `packets_vec`, then - // each iteration of this loop will process one element of `packets_vec`per iteration of the + // Because each conflicting transaction is in it's own `Packet` within a `PacketBatch`, then + // each iteration of this loop will process one element of the batch per iteration of the // loop. let interrupted_iteration = 1; poh_recorder.lock().unwrap().set_bank(&bank); @@ -2911,7 +2965,7 @@ mod tests { &Pubkey::default(), std::u128::MAX, &poh_recorder_, - &mut buffered_packets, + &mut buffered_packet_batches, None, &gossip_vote_sender, test_fn, @@ -2923,13 +2977,13 @@ mod tests { // Check everything is correct. All indexes after `interrupted_iteration` // should still be unprocessed assert_eq!( - buffered_packets.len(), - packets_vec[interrupted_iteration + 1..].len() + buffered_packet_batches.len(), + packet_batches[interrupted_iteration + 1..].len() ); for ((remaining_unprocessed_packet, _, _forwarded), original_packet) in - buffered_packets + buffered_packet_batches .iter() - .zip(&packets_vec[interrupted_iteration + 1..]) + .zip(&packet_batches[interrupted_iteration + 1..]) { assert_eq!( remaining_unprocessed_packet.packets[0], @@ -2964,10 +3018,10 @@ mod tests { #[test] fn test_forwarder_budget() { solana_logger::setup(); - // Create `Packets` with 1 unprocessed element - let single_element_packets = Packets::new(vec![Packet::default()]); - let mut unprocessed_packets: UnprocessedPackets = - vec![(single_element_packets, vec![0], false)] + // Create `PacketBatch` with 1 unprocessed packet + let single_packet_batch = PacketBatch::new(vec![Packet::default()]); + let mut unprocessed_packets: UnprocessedPacketBatches = + vec![(single_packet_batch, vec![0], false)] .into_iter() .collect(); @@ -3013,14 +3067,16 @@ mod tests { #[test] fn test_push_unprocessed_batch_limit() { solana_logger::setup(); - // Create `Packets` with 2 unprocessed elements - let new_packets = Packets::new(vec![Packet::default(); 2]); - let mut unprocessed_packets: UnprocessedPackets = - vec![(new_packets, vec![0, 1], false)].into_iter().collect(); + // Create `PacketBatch` with 2 unprocessed packets + let new_packet_batch = PacketBatch::new(vec![Packet::default(); 2]); + let mut unprocessed_packets: UnprocessedPacketBatches = + vec![(new_packet_batch, vec![0, 1], false)] + .into_iter() + .collect(); // Set the limit to 2 let batch_limit = 2; - // Create some new unprocessed packets - let new_packets = Packets::new(vec![Packet::default()]); + // Create new unprocessed packets and add to a batch + let new_packet_batch = PacketBatch::new(vec![Packet::default()]); let packet_indexes = vec![]; let duplicates = Arc::new(Mutex::new(( @@ -3035,7 +3091,7 @@ mod tests { // packets are not added to the unprocessed queue BankingStage::push_unprocessed( &mut unprocessed_packets, - new_packets.clone(), + new_packet_batch.clone(), packet_indexes, &mut dropped_packet_batches_count, &mut dropped_packets_count, @@ -3054,7 +3110,7 @@ mod tests { let packet_indexes = vec![0]; BankingStage::push_unprocessed( &mut unprocessed_packets, - new_packets, + new_packet_batch, packet_indexes.clone(), &mut dropped_packet_batches_count, &mut dropped_packets_count, @@ -3070,7 +3126,7 @@ mod tests { // Because we've reached the batch limit, old unprocessed packets are // dropped and the new one is appended to the end - let new_packets = Packets::new(vec![Packet::from_data( + let new_packet_batch = PacketBatch::new(vec![Packet::from_data( Some(&SocketAddr::from(([127, 0, 0, 1], 8001))), 42, ) @@ -3078,7 +3134,7 @@ mod tests { assert_eq!(unprocessed_packets.len(), batch_limit); BankingStage::push_unprocessed( &mut unprocessed_packets, - new_packets.clone(), + new_packet_batch.clone(), packet_indexes.clone(), &mut dropped_packet_batches_count, &mut dropped_packets_count, @@ -3088,7 +3144,10 @@ mod tests { &banking_stage_stats, ); assert_eq!(unprocessed_packets.len(), 2); - assert_eq!(unprocessed_packets[1].0.packets[0], new_packets.packets[0]); + assert_eq!( + unprocessed_packets[1].0.packets[0], + new_packet_batch.packets[0] + ); assert_eq!(dropped_packet_batches_count, 1); assert_eq!(dropped_packets_count, 2); assert_eq!(newly_buffered_packets_count, 2); @@ -3096,7 +3155,7 @@ mod tests { // Check duplicates are dropped (newly buffered shouldn't change) BankingStage::push_unprocessed( &mut unprocessed_packets, - new_packets.clone(), + new_packet_batch.clone(), packet_indexes, &mut dropped_packet_batches_count, &mut dropped_packets_count, @@ -3106,7 +3165,10 @@ mod tests { &banking_stage_stats, ); assert_eq!(unprocessed_packets.len(), 2); - assert_eq!(unprocessed_packets[1].0.packets[0], new_packets.packets[0]); + assert_eq!( + unprocessed_packets[1].0.packets[0], + new_packet_batch.packets[0] + ); assert_eq!(dropped_packet_batches_count, 1); assert_eq!(dropped_packets_count, 2); assert_eq!(newly_buffered_packets_count, 2); @@ -3129,19 +3191,19 @@ mod tests { fn make_test_packets( transactions: Vec, vote_indexes: Vec, - ) -> (Packets, Vec) { + ) -> (PacketBatch, Vec) { let capacity = transactions.len(); - let mut packets = Packets::with_capacity(capacity); + let mut packet_batch = PacketBatch::with_capacity(capacity); let mut packet_indexes = Vec::with_capacity(capacity); - packets.packets.resize(capacity, Packet::default()); + packet_batch.packets.resize(capacity, Packet::default()); for (index, tx) in transactions.iter().enumerate() { - Packet::populate_packet(&mut packets.packets[index], None, tx).ok(); + Packet::populate_packet(&mut packet_batch.packets[index], None, tx).ok(); packet_indexes.push(index); } for index in vote_indexes.iter() { - packets.packets[*index].meta.is_simple_vote_tx = true; + packet_batch.packets[*index].meta.is_simple_vote_tx = true; } - (packets, packet_indexes) + (packet_batch, packet_indexes) } #[test] @@ -3162,10 +3224,11 @@ mod tests { // packets with no votes { let vote_indexes = vec![]; - let (packets, packet_indexes) = + let (packet_batch, packet_indexes) = make_test_packets(vec![transfer_tx.clone(), transfer_tx.clone()], vote_indexes); let mut votes_only = false; +<<<<<<< HEAD let (txs, tx_packet_index, _retryable_packet_indexes) = BankingStage::transactions_from_packets( &packets, @@ -3177,10 +3240,19 @@ mod tests { votes_only, &Arc::new(RwLock::new(CostModel::default())), ); +======= + let (txs, tx_packet_index) = BankingStage::transactions_from_packets( + &packet_batch, + &packet_indexes, + &Arc::new(FeatureSet::default()), + votes_only, + ); +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) assert_eq!(2, txs.len()); assert_eq!(vec![0, 1], tx_packet_index); votes_only = true; +<<<<<<< HEAD let (txs, tx_packet_index, _retryable_packet_indexes) = BankingStage::transactions_from_packets( &packets, @@ -3192,6 +3264,14 @@ mod tests { votes_only, &Arc::new(RwLock::new(CostModel::default())), ); +======= + let (txs, tx_packet_index) = BankingStage::transactions_from_packets( + &packet_batch, + &packet_indexes, + &Arc::new(FeatureSet::default()), + votes_only, + ); +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) assert_eq!(0, txs.len()); assert_eq!(0, tx_packet_index.len()); } @@ -3199,12 +3279,13 @@ mod tests { // packets with some votes { let vote_indexes = vec![0, 2]; - let (packets, packet_indexes) = make_test_packets( + let (packet_batch, packet_indexes) = make_test_packets( vec![vote_tx.clone(), transfer_tx, vote_tx.clone()], vote_indexes, ); let mut votes_only = false; +<<<<<<< HEAD let (txs, tx_packet_index, _retryable_packet_indexes) = BankingStage::transactions_from_packets( &packets, @@ -3216,10 +3297,19 @@ mod tests { votes_only, &Arc::new(RwLock::new(CostModel::default())), ); +======= + let (txs, tx_packet_index) = BankingStage::transactions_from_packets( + &packet_batch, + &packet_indexes, + &Arc::new(FeatureSet::default()), + votes_only, + ); +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) assert_eq!(3, txs.len()); assert_eq!(vec![0, 1, 2], tx_packet_index); votes_only = true; +<<<<<<< HEAD let (txs, tx_packet_index, _retryable_packet_indexes) = BankingStage::transactions_from_packets( &packets, @@ -3231,6 +3321,14 @@ mod tests { votes_only, &Arc::new(RwLock::new(CostModel::default())), ); +======= + let (txs, tx_packet_index) = BankingStage::transactions_from_packets( + &packet_batch, + &packet_indexes, + &Arc::new(FeatureSet::default()), + votes_only, + ); +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) assert_eq!(2, txs.len()); assert_eq!(vec![0, 2], tx_packet_index); } @@ -3238,12 +3336,13 @@ mod tests { // packets with all votes { let vote_indexes = vec![0, 1, 2]; - let (packets, packet_indexes) = make_test_packets( + let (packet_batch, packet_indexes) = make_test_packets( vec![vote_tx.clone(), vote_tx.clone(), vote_tx], vote_indexes, ); let mut votes_only = false; +<<<<<<< HEAD let (txs, tx_packet_index, _retryable_packet_indexes) = BankingStage::transactions_from_packets( &packets, @@ -3255,10 +3354,19 @@ mod tests { votes_only, &Arc::new(RwLock::new(CostModel::default())), ); +======= + let (txs, tx_packet_index) = BankingStage::transactions_from_packets( + &packet_batch, + &packet_indexes, + &Arc::new(FeatureSet::default()), + votes_only, + ); +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) assert_eq!(3, txs.len()); assert_eq!(vec![0, 1, 2], tx_packet_index); votes_only = true; +<<<<<<< HEAD let (txs, tx_packet_index, _retryable_packet_indexes) = BankingStage::transactions_from_packets( &packets, @@ -3270,6 +3378,14 @@ mod tests { votes_only, &Arc::new(RwLock::new(CostModel::default())), ); +======= + let (txs, tx_packet_index) = BankingStage::transactions_from_packets( + &packet_batch, + &packet_indexes, + &Arc::new(FeatureSet::default()), + votes_only, + ); +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) assert_eq!(3, txs.len()); assert_eq!(vec![0, 1, 2], tx_packet_index); } diff --git a/core/src/cluster_info_vote_listener.rs b/core/src/cluster_info_vote_listener.rs index 3a6c6b55448..a68efa17717 100644 --- a/core/src/cluster_info_vote_listener.rs +++ b/core/src/cluster_info_vote_listener.rs @@ -20,7 +20,7 @@ use { }, solana_ledger::blockstore::Blockstore, solana_metrics::inc_new_counter_debug, - solana_perf::packet::{self, Packets}, + solana_perf::packet::{self, PacketBatch}, solana_poh::poh_recorder::PohRecorder, solana_rpc::{ optimistically_confirmed_bank_tracker::{BankNotification, BankNotificationSender}, @@ -253,7 +253,7 @@ impl ClusterInfoVoteListener { pub fn new( exit: &Arc, cluster_info: Arc, - verified_packets_sender: CrossbeamSender>, + verified_packets_sender: CrossbeamSender>, poh_recorder: &Arc>, vote_tracker: Arc, bank_forks: Arc>, @@ -349,16 +349,22 @@ impl ClusterInfoVoteListener { } #[allow(clippy::type_complexity)] +<<<<<<< HEAD fn verify_votes( votes: Vec, labels: Vec, ) -> (Vec, Vec<(CrdsValueLabel, Slot, Packets)>) { let mut msgs = packet::to_packets_chunked(&votes, 1); +======= + fn verify_votes(votes: Vec) -> (Vec, Vec) { + let mut packet_batches = packet::to_packet_batches(&votes, 1); +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) // Votes should already be filtered by this point. let reject_non_vote = false; - sigverify::ed25519_verify_cpu(&mut msgs, reject_non_vote); + sigverify::ed25519_verify_cpu(&mut packet_batches, reject_non_vote); +<<<<<<< HEAD let (vote_txs, packets) = izip!(labels.into_iter(), votes.into_iter(), msgs,) .filter_map(|(label, vote, packet)| { let slot = vote_transaction::parse_vote_transaction(&vote) @@ -370,6 +376,33 @@ impl ClusterInfoVoteListener { Some((vote, (label, slot, packet))) } else { None +======= + let (vote_txs, vote_metadata) = izip!(votes.into_iter(), packet_batches) + .filter_map(|(vote_tx, packet_batch)| { + let (vote, vote_account_key) = vote_transaction::parse_vote_transaction(&vote_tx) + .and_then(|(vote_account_key, vote, _)| { + if vote.slots().is_empty() { + None + } else { + Some((vote, vote_account_key)) + } + })?; + + // to_packet_batches() above splits into 1 packet long batches + assert_eq!(packet_batch.packets.len(), 1); + if !packet_batch.packets[0].meta.discard { + if let Some(signature) = vote_tx.signatures.first().cloned() { + return Some(( + vote_tx, + VerifiedVoteMetadata { + vote_account_key, + vote, + packet_batch, + signature, + }, + )); + } +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) } }) .unzip(); @@ -380,7 +413,7 @@ impl ClusterInfoVoteListener { exit: Arc, verified_vote_label_packets_receiver: VerifiedLabelVotePacketsReceiver, poh_recorder: Arc>, - verified_packets_sender: &CrossbeamSender>, + verified_packets_sender: &CrossbeamSender>, ) -> Result<()> { let mut verified_vote_packets = VerifiedVotePackets::default(); let mut time_since_lock = Instant::now(); @@ -430,6 +463,61 @@ impl ClusterInfoVoteListener { } } +<<<<<<< HEAD +======= + fn check_for_leader_bank_and_send_votes( + bank_vote_sender_state_option: &mut Option, + current_working_bank: Arc, + verified_packets_sender: &CrossbeamSender>, + verified_vote_packets: &VerifiedVotePackets, + ) -> Result<()> { + // We will take this lock at most once every `BANK_SEND_VOTES_LOOP_SLEEP_MS` + if let Some(bank_vote_sender_state) = bank_vote_sender_state_option { + if bank_vote_sender_state.bank.slot() != current_working_bank.slot() { + bank_vote_sender_state.report_metrics(); + *bank_vote_sender_state_option = + Some(BankVoteSenderState::new(current_working_bank)); + } + } else { + *bank_vote_sender_state_option = Some(BankVoteSenderState::new(current_working_bank)); + } + + let bank_vote_sender_state = bank_vote_sender_state_option.as_mut().unwrap(); + let BankVoteSenderState { + ref bank, + ref mut bank_send_votes_stats, + ref mut previously_sent_to_bank_votes, + } = bank_vote_sender_state; + + // This logic may run multiple times for the same leader bank, + // we just have to ensure that the same votes are not sent + // to the bank multiple times, which is guaranteed by + // `previously_sent_to_bank_votes` + let gossip_votes_iterator = ValidatorGossipVotesIterator::new( + bank.clone(), + verified_vote_packets, + previously_sent_to_bank_votes, + ); + + let mut filter_gossip_votes_timing = Measure::start("filter_gossip_votes"); + + // Send entire batch at a time so that there is no partial processing of + // a single validator's votes by two different banks. This might happen + // if we sent each vote individually, for instance if we creaed two different + // leader banks from the same common parent, one leader bank may process + // only the later votes and ignore the earlier votes. + for single_validator_votes in gossip_votes_iterator { + bank_send_votes_stats.num_votes_sent += single_validator_votes.len(); + bank_send_votes_stats.num_batches_sent += 1; + verified_packets_sender.send(single_validator_votes)?; + } + filter_gossip_votes_timing.stop(); + bank_send_votes_stats.total_elapsed += filter_gossip_votes_timing.as_us(); + + Ok(()) + } + +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) #[allow(clippy::too_many_arguments)] fn process_votes_loop( exit: Arc, @@ -876,9 +964,9 @@ mod tests { use bincode::serialized_size; info!("max vote size {}", serialized_size(&vote_tx).unwrap()); - let msgs = packet::to_packets_chunked(&[vote_tx], 1); // panics if won't fit + let packet_batches = packet::to_packet_batches(&[vote_tx], 1); // panics if won't fit - assert_eq!(msgs.len(), 1); + assert_eq!(packet_batches.len(), 1); } fn run_vote_contains_authorized_voter(hash: Option) { @@ -1706,8 +1794,16 @@ mod tests { assert!(packets.is_empty()); } +<<<<<<< HEAD fn verify_packets_len(packets: &[(CrdsValueLabel, Slot, Packets)], ref_value: usize) { let num_packets: usize = packets.iter().map(|(_, _, p)| p.packets.len()).sum(); +======= + fn verify_packets_len(packets: &[VerifiedVoteMetadata], ref_value: usize) { + let num_packets: usize = packets + .iter() + .map(|vote_metadata| vote_metadata.packet_batch.packets.len()) + .sum(); +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) assert_eq!(num_packets, ref_value); } diff --git a/core/src/fetch_stage.rs b/core/src/fetch_stage.rs index 17a5cdafe5a..8cfa9745d0b 100644 --- a/core/src/fetch_stage.rs +++ b/core/src/fetch_stage.rs @@ -6,10 +6,10 @@ use { result::{Error, Result}, }, solana_metrics::{inc_new_counter_debug, inc_new_counter_info}, - solana_perf::{packet::PacketsRecycler, recycler::Recycler}, + solana_perf::{packet::PacketBatchRecycler, recycler::Recycler}, solana_poh::poh_recorder::PohRecorder, solana_sdk::clock::DEFAULT_TICKS_PER_SLOT, - solana_streamer::streamer::{self, PacketReceiver, PacketSender}, + solana_streamer::streamer::{self, PacketBatchReceiver, PacketBatchSender}, std::{ net::UdpSocket, sync::{ @@ -34,7 +34,7 @@ impl FetchStage { exit: &Arc, poh_recorder: &Arc>, coalesce_ms: u64, - ) -> (Self, PacketReceiver, PacketReceiver) { + ) -> (Self, PacketBatchReceiver, PacketBatchReceiver) { let (sender, receiver) = channel(); let (vote_sender, vote_receiver) = channel(); ( @@ -58,8 +58,8 @@ impl FetchStage { tpu_forwards_sockets: Vec, tpu_vote_sockets: Vec, exit: &Arc, - sender: &PacketSender, - vote_sender: &PacketSender, + sender: &PacketBatchSender, + vote_sender: &PacketBatchSender, poh_recorder: &Arc>, coalesce_ms: u64, ) -> Self { @@ -79,18 +79,18 @@ impl FetchStage { } fn handle_forwarded_packets( - recvr: &PacketReceiver, - sendr: &PacketSender, + recvr: &PacketBatchReceiver, + sendr: &PacketBatchSender, poh_recorder: &Arc>, ) -> Result<()> { - let msgs = recvr.recv()?; - let mut len = msgs.packets.len(); - let mut batch = vec![msgs]; - while let Ok(more) = recvr.try_recv() { - len += more.packets.len(); - batch.push(more); + let packet_batch = recvr.recv()?; + let mut num_packets = packet_batch.packets.len(); + let mut packet_batches = vec![packet_batch]; + while let Ok(packet_batch) = recvr.try_recv() { + num_packets += packet_batch.packets.len(); + packet_batches.push(packet_batch); // Read at most 1K transactions in a loop - if len > 1024 { + if num_packets > 1024 { break; } } @@ -100,14 +100,21 @@ impl FetchStage { .unwrap() .would_be_leader(HOLD_TRANSACTIONS_SLOT_OFFSET.saturating_mul(DEFAULT_TICKS_PER_SLOT)) { +<<<<<<< HEAD inc_new_counter_debug!("fetch_stage-honor_forwards", len); for packets in batch { if sendr.send(packets).is_err() { +======= + inc_new_counter_debug!("fetch_stage-honor_forwards", num_packets); + for packet_batch in packet_batches { + #[allow(clippy::question_mark)] + if sendr.send(packet_batch).is_err() { +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) return Err(Error::Send); } } } else { - inc_new_counter_info!("fetch_stage-discard_forwards", len); + inc_new_counter_info!("fetch_stage-discard_forwards", num_packets); } Ok(()) @@ -118,12 +125,12 @@ impl FetchStage { tpu_forwards_sockets: Vec>, tpu_vote_sockets: Vec>, exit: &Arc, - sender: &PacketSender, - vote_sender: &PacketSender, + sender: &PacketBatchSender, + vote_sender: &PacketBatchSender, poh_recorder: &Arc>, coalesce_ms: u64, ) -> Self { - let recycler: PacketsRecycler = Recycler::warmed(1000, 1024); + let recycler: PacketBatchRecycler = Recycler::warmed(1000, 1024); let tpu_threads = sockets.into_iter().map(|socket| { streamer::receiver( diff --git a/core/src/retransmit_stage.rs b/core/src/retransmit_stage.rs index 256b2c1c25e..1ba00336585 100644 --- a/core/src/retransmit_stage.rs +++ b/core/src/retransmit_stage.rs @@ -21,7 +21,7 @@ use { blockstore::Blockstore, leader_schedule_cache::LeaderScheduleCache, shred::Shred, }, solana_measure::measure::Measure, - solana_perf::packet::Packets, + solana_perf::packet::PacketBatch, solana_rayon_threadlimit::get_thread_count, solana_rpc::{max_slots::MaxSlots, rpc_subscriptions::RpcSubscriptions}, solana_runtime::{bank::Bank, bank_forks::BankForks}, @@ -438,7 +438,7 @@ impl RetransmitStage { cluster_info: Arc, retransmit_sockets: Arc>, repair_socket: Arc, - verified_receiver: Receiver>, + verified_receiver: Receiver>, exit: Arc, cluster_slots_update_receiver: ClusterSlotsUpdateReceiver, epoch_schedule: EpochSchedule, @@ -603,10 +603,10 @@ mod tests { let shred = Shred::new_from_data(0, 0, 0, None, true, true, 0, 0x20, 0); // it should send this over the sockets. retransmit_sender.send(vec![shred]).unwrap(); - let mut packets = Packets::new(vec![]); - solana_streamer::packet::recv_from(&mut packets, &me_retransmit, 1).unwrap(); - assert_eq!(packets.packets.len(), 1); - assert!(!packets.packets[0].meta.repair); + let mut packet_batch = PacketBatch::new(vec![]); + solana_streamer::packet::recv_from(&mut packet_batch, &me_retransmit, 1).unwrap(); + assert_eq!(packet_batch.packets.len(), 1); + assert!(!packet_batch.packets[0].meta.repair); } #[test] diff --git a/core/src/serve_repair.rs b/core/src/serve_repair.rs index 63a45fc84b2..20a43d1df31 100644 --- a/core/src/serve_repair.rs +++ b/core/src/serve_repair.rs @@ -23,14 +23,14 @@ use { }, solana_measure::measure::Measure, solana_metrics::inc_new_counter_debug, - solana_perf::packet::{limited_deserialize, Packets, PacketsRecycler}, + solana_perf::packet::{limited_deserialize, PacketBatch, PacketBatchRecycler}, solana_sdk::{ clock::Slot, pubkey::Pubkey, signature::{Keypair, Signer}, timing::duration_as_ms, }, - solana_streamer::streamer::{PacketReceiver, PacketSender}, + solana_streamer::streamer::{PacketBatchReceiver, PacketBatchSender}, std::{ collections::HashSet, net::SocketAddr, @@ -183,12 +183,12 @@ impl ServeRepair { fn handle_repair( me: &Arc>, - recycler: &PacketsRecycler, + recycler: &PacketBatchRecycler, from_addr: &SocketAddr, blockstore: Option<&Arc>, request: RepairProtocol, stats: &mut ServeRepairStats, - ) -> Option { + ) -> Option { let now = Instant::now(); //TODO verify from is signed @@ -264,10 +264,10 @@ impl ServeRepair { /// Process messages from the network fn run_listen( obj: &Arc>, - recycler: &PacketsRecycler, + recycler: &PacketBatchRecycler, blockstore: Option<&Arc>, - requests_receiver: &PacketReceiver, - response_sender: &PacketSender, + requests_receiver: &PacketBatchReceiver, + response_sender: &PacketBatchSender, stats: &mut ServeRepairStats, max_packets: &mut usize, ) -> Result<()> { @@ -336,12 +336,12 @@ impl ServeRepair { pub fn listen( me: Arc>, blockstore: Option>, - requests_receiver: PacketReceiver, - response_sender: PacketSender, + requests_receiver: PacketBatchReceiver, + response_sender: PacketBatchSender, exit: &Arc, ) -> JoinHandle<()> { let exit = exit.clone(); - let recycler = PacketsRecycler::default(); + let recycler = PacketBatchRecycler::default(); Builder::new() .name("solana-repair-listen".to_string()) .spawn(move || { @@ -376,14 +376,14 @@ impl ServeRepair { fn handle_packets( me: &Arc>, - recycler: &PacketsRecycler, + recycler: &PacketBatchRecycler, blockstore: Option<&Arc>, - packets: Packets, - response_sender: &PacketSender, + packet_batch: PacketBatch, + response_sender: &PacketBatchSender, stats: &mut ServeRepairStats, ) { // iter over the packets - packets.packets.iter().for_each(|packet| { + packet_batch.packets.iter().for_each(|packet| { let from_addr = packet.meta.addr(); limited_deserialize(&packet.data[..packet.meta.size]) .into_iter() @@ -526,7 +526,7 @@ impl ServeRepair { } fn run_window_request( - recycler: &PacketsRecycler, + recycler: &PacketBatchRecycler, from: &ContactInfo, from_addr: &SocketAddr, blockstore: Option<&Arc>, @@ -534,7 +534,7 @@ impl ServeRepair { slot: Slot, shred_index: u64, nonce: Nonce, - ) -> Option { + ) -> Option { if let Some(blockstore) = blockstore { // Try to find the requested index in one of the slots let packet = repair_response::repair_response_packet( @@ -547,7 +547,7 @@ impl ServeRepair { if let Some(packet) = packet { inc_new_counter_debug!("serve_repair-window-request-ledger", 1); - return Some(Packets::new_unpinned_with_recycler_data( + return Some(PacketBatch::new_unpinned_with_recycler_data( recycler, "run_window_request", vec![packet], @@ -568,13 +568,13 @@ impl ServeRepair { } fn run_highest_window_request( - recycler: &PacketsRecycler, + recycler: &PacketBatchRecycler, from_addr: &SocketAddr, blockstore: Option<&Arc>, slot: Slot, highest_index: u64, nonce: Nonce, - ) -> Option { + ) -> Option { let blockstore = blockstore?; // Try to find the requested index in one of the slots let meta = blockstore.meta(slot).ok()??; @@ -587,7 +587,7 @@ impl ServeRepair { from_addr, nonce, )?; - return Some(Packets::new_unpinned_with_recycler_data( + return Some(PacketBatch::new_unpinned_with_recycler_data( recycler, "run_highest_window_request", vec![packet], @@ -597,14 +597,14 @@ impl ServeRepair { } fn run_orphan( - recycler: &PacketsRecycler, + recycler: &PacketBatchRecycler, from_addr: &SocketAddr, blockstore: Option<&Arc>, mut slot: Slot, max_responses: usize, nonce: Nonce, - ) -> Option { - let mut res = Packets::new_unpinned_with_recycler(recycler.clone(), 64, "run_orphan"); + ) -> Option { + let mut res = PacketBatch::new_unpinned_with_recycler(recycler.clone(), 64, "run_orphan"); if let Some(blockstore) = blockstore { // Try to find the next "n" parent slots of the input slot while let Ok(Some(meta)) = blockstore.meta(slot) { @@ -635,6 +635,43 @@ impl ServeRepair { } Some(res) } +<<<<<<< HEAD +======= + + fn run_ancestor_hashes( + recycler: &PacketBatchRecycler, + from_addr: &SocketAddr, + blockstore: Option<&Arc>, + slot: Slot, + nonce: Nonce, + ) -> Option { + let blockstore = blockstore?; + let ancestor_slot_hashes = if blockstore.is_duplicate_confirmed(slot) { + let ancestor_iterator = + AncestorIteratorWithHash::from(AncestorIterator::new_inclusive(slot, blockstore)); + ancestor_iterator.take(MAX_ANCESTOR_RESPONSES).collect() + } else { + // If this slot is not duplicate confirmed, return nothing + vec![] + }; + let response = AncestorHashesResponseVersion::Current(ancestor_slot_hashes); + let serialized_response = serialize(&response).ok()?; + + // Could probably directly write response into packet via `serialize_into()` + // instead of incurring extra copy in `repair_response_packet_from_bytes`, but + // serialize_into doesn't return the written size... + let packet = repair_response::repair_response_packet_from_bytes( + serialized_response, + from_addr, + nonce, + )?; + Some(PacketBatch::new_unpinned_with_recycler_data( + recycler, + "run_ancestor_hashes", + vec![packet], + )) + } +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) } #[cfg(test)] @@ -661,7 +698,7 @@ mod tests { /// test run_window_request responds with the right shred, and do not overrun fn run_highest_window_request(slot: Slot, num_slots: u64, nonce: Nonce) { - let recycler = PacketsRecycler::default(); + let recycler = PacketBatchRecycler::default(); solana_logger::setup(); let ledger_path = get_tmp_ledger_path!(); { @@ -731,7 +768,7 @@ mod tests { /// test window requests respond with the right shred, and do not overrun fn run_window_request(slot: Slot, nonce: Nonce) { - let recycler = PacketsRecycler::default(); + let recycler = PacketBatchRecycler::default(); solana_logger::setup(); let ledger_path = get_tmp_ledger_path!(); { @@ -900,7 +937,7 @@ mod tests { fn run_orphan(slot: Slot, num_slots: u64, nonce: Nonce) { solana_logger::setup(); - let recycler = PacketsRecycler::default(); + let recycler = PacketBatchRecycler::default(); let ledger_path = get_tmp_ledger_path!(); { let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap()); @@ -974,7 +1011,7 @@ mod tests { #[test] fn run_orphan_corrupted_shred_size() { solana_logger::setup(); - let recycler = PacketsRecycler::default(); + let recycler = PacketBatchRecycler::default(); let ledger_path = get_tmp_ledger_path!(); { let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap()); @@ -1033,6 +1070,92 @@ mod tests { } #[test] +<<<<<<< HEAD +======= + fn test_run_ancestor_hashes() { + solana_logger::setup(); + let recycler = PacketBatchRecycler::default(); + let ledger_path = get_tmp_ledger_path!(); + { + let slot = 0; + let num_slots = MAX_ANCESTOR_RESPONSES as u64; + let nonce = 10; + + let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap()); + + // Create slots [slot, slot + num_slots) with 5 shreds apiece + let (shreds, _) = make_many_slot_entries(slot, num_slots, 5); + + blockstore + .insert_shreds(shreds, None, false) + .expect("Expect successful ledger write"); + + // We don't have slot `slot + num_slots`, so we return empty + let rv = ServeRepair::run_ancestor_hashes( + &recycler, + &socketaddr_any!(), + Some(&blockstore), + slot + num_slots, + nonce, + ) + .expect("run_ancestor_hashes packets") + .packets; + assert_eq!(rv.len(), 1); + let packet = &rv[0]; + let ancestor_hashes_response: AncestorHashesResponseVersion = + limited_deserialize(&packet.data[..packet.meta.size - SIZE_OF_NONCE]).unwrap(); + assert!(ancestor_hashes_response.into_slot_hashes().is_empty()); + + // `slot + num_slots - 1` is not marked duplicate confirmed so nothing should return + // empty + let rv = ServeRepair::run_ancestor_hashes( + &recycler, + &socketaddr_any!(), + Some(&blockstore), + slot + num_slots - 1, + nonce, + ) + .expect("run_ancestor_hashes packets") + .packets; + assert_eq!(rv.len(), 1); + let packet = &rv[0]; + let ancestor_hashes_response: AncestorHashesResponseVersion = + limited_deserialize(&packet.data[..packet.meta.size - SIZE_OF_NONCE]).unwrap(); + assert!(ancestor_hashes_response.into_slot_hashes().is_empty()); + + // Set duplicate confirmed + let mut expected_ancestors = Vec::with_capacity(num_slots as usize); + expected_ancestors.resize(num_slots as usize, (0, Hash::default())); + for (i, duplicate_confirmed_slot) in (slot..slot + num_slots).enumerate() { + let frozen_hash = Hash::new_unique(); + expected_ancestors[num_slots as usize - i - 1] = + (duplicate_confirmed_slot, frozen_hash); + blockstore.insert_bank_hash(duplicate_confirmed_slot, frozen_hash, true); + } + let rv = ServeRepair::run_ancestor_hashes( + &recycler, + &socketaddr_any!(), + Some(&blockstore), + slot + num_slots - 1, + nonce, + ) + .expect("run_ancestor_hashes packets") + .packets; + assert_eq!(rv.len(), 1); + let packet = &rv[0]; + let ancestor_hashes_response: AncestorHashesResponseVersion = + limited_deserialize(&packet.data[..packet.meta.size - SIZE_OF_NONCE]).unwrap(); + assert_eq!( + ancestor_hashes_response.into_slot_hashes(), + expected_ancestors + ); + } + + Blockstore::destroy(&ledger_path).expect("Expected successful database destruction"); + } + + #[test] +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) fn test_repair_with_repair_validators() { let cluster_slots = ClusterSlots::default(); let me = ContactInfo::new_localhost(&solana_sdk::pubkey::new_rand(), timestamp()); diff --git a/core/src/shred_fetch_stage.rs b/core/src/shred_fetch_stage.rs index 9ac40d29912..4beaf3cc1b8 100644 --- a/core/src/shred_fetch_stage.rs +++ b/core/src/shred_fetch_stage.rs @@ -6,12 +6,12 @@ use { solana_ledger::shred::{get_shred_slot_index_type, ShredFetchStats}, solana_perf::{ cuda_runtime::PinnedVec, - packet::{Packet, PacketsRecycler}, + packet::{Packet, PacketBatchRecycler}, recycler::Recycler, }, solana_runtime::bank_forks::BankForks, solana_sdk::clock::{Slot, DEFAULT_MS_PER_SLOT}, - solana_streamer::streamer::{self, PacketReceiver, PacketSender}, + solana_streamer::streamer::{self, PacketBatchReceiver, PacketBatchSender}, std::{ net::UdpSocket, sync::{atomic::AtomicBool, mpsc::channel, Arc, RwLock}, @@ -63,8 +63,8 @@ impl ShredFetchStage { // updates packets received on a channel and sends them on another channel fn modify_packets( - recvr: PacketReceiver, - sendr: PacketSender, + recvr: PacketBatchReceiver, + sendr: PacketBatchSender, bank_forks: Option>>, name: &'static str, modify: F, @@ -83,7 +83,7 @@ impl ShredFetchStage { let mut stats = ShredFetchStats::default(); let mut packet_hasher = PacketHasher::default(); - while let Some(mut p) = recvr.iter().next() { + while let Some(mut packet_batch) = recvr.iter().next() { if last_updated.elapsed().as_millis() as u64 > DEFAULT_MS_PER_SLOT { last_updated = Instant::now(); packet_hasher.reset(); @@ -97,8 +97,13 @@ impl ShredFetchStage { slots_per_epoch = root_bank.get_slots_in_epoch(root_bank.epoch()); } } +<<<<<<< HEAD stats.shred_count += p.packets.len(); p.packets.iter_mut().for_each(|mut packet| { +======= + stats.shred_count += packet_batch.packets.len(); + packet_batch.packets.iter_mut().for_each(|packet| { +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) Self::process_packet( &mut packet, &mut shreds_received, @@ -124,7 +129,7 @@ impl ShredFetchStage { stats = ShredFetchStats::default(); last_stats = Instant::now(); } - if sendr.send(p).is_err() { + if sendr.send(packet_batch).is_err() { break; } } @@ -133,7 +138,7 @@ impl ShredFetchStage { fn packet_modifier( sockets: Vec>, exit: &Arc, - sender: PacketSender, + sender: PacketBatchSender, recycler: Recycler>, bank_forks: Option>>, name: &'static str, @@ -169,11 +174,11 @@ impl ShredFetchStage { sockets: Vec>, forward_sockets: Vec>, repair_socket: Arc, - sender: &PacketSender, + sender: &PacketBatchSender, bank_forks: Option>>, exit: &Arc, ) -> Self { - let recycler: PacketsRecycler = Recycler::warmed(100, 1024); + let recycler: PacketBatchRecycler = Recycler::warmed(100, 1024); let (mut tvu_threads, tvu_filter) = Self::packet_modifier( sockets, diff --git a/core/src/sigverify.rs b/core/src/sigverify.rs index 8ffa30bb841..74dbf5bdfc8 100644 --- a/core/src/sigverify.rs +++ b/core/src/sigverify.rs @@ -5,11 +5,11 @@ //! pub use solana_perf::sigverify::{ - batch_size, ed25519_verify_cpu, ed25519_verify_disabled, init, TxOffset, + count_packets_in_batches, ed25519_verify_cpu, ed25519_verify_disabled, init, TxOffset, }; use { crate::sigverify_stage::SigVerifier, - solana_perf::{cuda_runtime::PinnedVec, packet::Packets, recycler::Recycler, sigverify}, + solana_perf::{cuda_runtime::PinnedVec, packet::PacketBatch, recycler::Recycler, sigverify}, }; #[derive(Clone)] @@ -40,13 +40,13 @@ impl Default for TransactionSigVerifier { } impl SigVerifier for TransactionSigVerifier { - fn verify_batch(&self, mut batch: Vec) -> Vec { + fn verify_batches(&self, mut batches: Vec) -> Vec { sigverify::ed25519_verify( - &mut batch, + &mut batches, &self.recycler, &self.recycler_out, self.reject_non_vote, ); - batch + batches } } diff --git a/core/src/sigverify_shreds.rs b/core/src/sigverify_shreds.rs index 174811ccee1..e9309b434f4 100644 --- a/core/src/sigverify_shreds.rs +++ b/core/src/sigverify_shreds.rs @@ -5,7 +5,7 @@ use { leader_schedule_cache::LeaderScheduleCache, shred::Shred, sigverify_shreds::verify_shreds_gpu, }, - solana_perf::{self, packet::Packets, recycler_cache::RecyclerCache}, + solana_perf::{self, packet::PacketBatch, recycler_cache::RecyclerCache}, solana_runtime::bank_forks::BankForks, std::{ collections::{HashMap, HashSet}, @@ -32,7 +32,7 @@ impl ShredSigVerifier { recycler_cache: RecyclerCache::warmed(), } } - fn read_slots(batches: &[Packets]) -> HashSet { + fn read_slots(batches: &[PacketBatch]) -> HashSet { batches .iter() .flat_map(|batch| batch.packets.iter().filter_map(Shred::get_slot_from_packet)) @@ -41,7 +41,7 @@ impl ShredSigVerifier { } impl SigVerifier for ShredSigVerifier { - fn verify_batch(&self, mut batches: Vec) -> Vec { + fn verify_batches(&self, mut batches: Vec) -> Vec { let r_bank = self.bank_forks.read().unwrap().working_bank(); let slots: HashSet = Self::read_slots(&batches); let mut leader_slots: HashMap = slots @@ -88,13 +88,13 @@ pub mod tests { 0, 0xc0de, ); - let mut batch = [Packets::default(), Packets::default()]; + let mut batches = [PacketBatch::default(), PacketBatch::default()]; let keypair = Keypair::new(); Shredder::sign_shred(&keypair, &mut shred); - batch[0].packets.resize(1, Packet::default()); - batch[0].packets[0].data[0..shred.payload.len()].copy_from_slice(&shred.payload); - batch[0].packets[0].meta.size = shred.payload.len(); + batches[0].packets.resize(1, Packet::default()); + batches[0].packets[0].data[0..shred.payload.len()].copy_from_slice(&shred.payload); + batches[0].packets[0].meta.size = shred.payload.len(); let mut shred = Shred::new_from_data( 0xc0de_dead, @@ -108,16 +108,16 @@ pub mod tests { 0xc0de, ); Shredder::sign_shred(&keypair, &mut shred); - batch[1].packets.resize(1, Packet::default()); - batch[1].packets[0].data[0..shred.payload.len()].copy_from_slice(&shred.payload); - batch[1].packets[0].meta.size = shred.payload.len(); + batches[1].packets.resize(1, Packet::default()); + batches[1].packets[0].data[0..shred.payload.len()].copy_from_slice(&shred.payload); + batches[1].packets[0].meta.size = shred.payload.len(); let expected: HashSet = [0xc0de_dead, 0xdead_c0de].iter().cloned().collect(); - assert_eq!(ShredSigVerifier::read_slots(&batch), expected); + assert_eq!(ShredSigVerifier::read_slots(&batches), expected); } #[test] - fn test_sigverify_shreds_verify_batch() { + fn test_sigverify_shreds_verify_batches() { let leader_keypair = Arc::new(Keypair::new()); let leader_pubkey = leader_keypair.pubkey(); let bank = @@ -126,8 +126,8 @@ pub mod tests { let bf = Arc::new(RwLock::new(BankForks::new(bank))); let verifier = ShredSigVerifier::new(bf, cache); - let mut batch = vec![Packets::default()]; - batch[0].packets.resize(2, Packet::default()); + let mut batches = vec![PacketBatch::default()]; + batches[0].packets.resize(2, Packet::default()); let mut shred = Shred::new_from_data( 0, @@ -141,8 +141,8 @@ pub mod tests { 0xc0de, ); Shredder::sign_shred(&leader_keypair, &mut shred); - batch[0].packets[0].data[0..shred.payload.len()].copy_from_slice(&shred.payload); - batch[0].packets[0].meta.size = shred.payload.len(); + batches[0].packets[0].data[0..shred.payload.len()].copy_from_slice(&shred.payload); + batches[0].packets[0].meta.size = shred.payload.len(); let mut shred = Shred::new_from_data( 0, @@ -157,10 +157,10 @@ pub mod tests { ); let wrong_keypair = Keypair::new(); Shredder::sign_shred(&wrong_keypair, &mut shred); - batch[0].packets[1].data[0..shred.payload.len()].copy_from_slice(&shred.payload); - batch[0].packets[1].meta.size = shred.payload.len(); + batches[0].packets[1].data[0..shred.payload.len()].copy_from_slice(&shred.payload); + batches[0].packets[1].meta.size = shred.payload.len(); - let rv = verifier.verify_batch(batch); + let rv = verifier.verify_batches(batches); assert!(!rv[0].packets[0].meta.discard); assert!(rv[0].packets[1].meta.discard); } diff --git a/core/src/sigverify_stage.rs b/core/src/sigverify_stage.rs index 596cdcd2111..3b6f88e2a70 100644 --- a/core/src/sigverify_stage.rs +++ b/core/src/sigverify_stage.rs @@ -9,9 +9,9 @@ use { crate::sigverify, crossbeam_channel::{SendError, Sender as CrossbeamSender}, solana_measure::measure::Measure, - solana_perf::packet::Packets, + solana_perf::packet::PacketBatch, solana_sdk::timing, - solana_streamer::streamer::{self, PacketReceiver, StreamerError}, + solana_streamer::streamer::{self, PacketBatchReceiver, StreamerError}, std::{ collections::HashMap, sync::mpsc::{Receiver, RecvTimeoutError}, @@ -26,7 +26,7 @@ const MAX_SIGVERIFY_BATCH: usize = 10_000; #[derive(Error, Debug)] pub enum SigVerifyServiceError { #[error("send packets batch error")] - Send(#[from] SendError>), + Send(#[from] SendError>), #[error("streamer error")] Streamer(#[from] StreamerError), @@ -39,7 +39,7 @@ pub struct SigVerifyStage { } pub trait SigVerifier { - fn verify_batch(&self, batch: Vec) -> Vec; + fn verify_batches(&self, batches: Vec) -> Vec; } #[derive(Default, Clone)] @@ -49,7 +49,7 @@ pub struct DisabledSigVerifier {} struct SigVerifierStats { recv_batches_us_hist: histogram::Histogram, // time to call recv_batch verify_batches_pp_us_hist: histogram::Histogram, // per-packet time to call verify_batch - batches_hist: histogram::Histogram, // number of Packets structures per verify call + batches_hist: histogram::Histogram, // number of packet batches per verify call packets_hist: histogram::Histogram, // number of packets per verify call total_batches: usize, total_packets: usize, @@ -122,24 +122,24 @@ impl SigVerifierStats { } impl SigVerifier for DisabledSigVerifier { - fn verify_batch(&self, mut batch: Vec) -> Vec { - sigverify::ed25519_verify_disabled(&mut batch); - batch + fn verify_batches(&self, mut batches: Vec) -> Vec { + sigverify::ed25519_verify_disabled(&mut batches); + batches } } impl SigVerifyStage { #[allow(clippy::new_ret_no_self)] pub fn new( - packet_receiver: Receiver, - verified_sender: CrossbeamSender>, + packet_receiver: Receiver, + verified_sender: CrossbeamSender>, verifier: T, ) -> Self { let thread_hdl = Self::verifier_services(packet_receiver, verified_sender, verifier); Self { thread_hdl } } - pub fn discard_excess_packets(batches: &mut Vec, max_packets: usize) { + pub fn discard_excess_packets(batches: &mut Vec, max_packets: usize) { let mut received_ips = HashMap::new(); for (batch_index, batch) in batches.iter().enumerate() { for (packet_index, packets) in batch.packets.iter().enumerate() { @@ -169,11 +169,12 @@ impl SigVerifyStage { } fn verifier( - recvr: &PacketReceiver, - sendr: &CrossbeamSender>, + recvr: &PacketBatchReceiver, + sendr: &CrossbeamSender>, verifier: &T, stats: &mut SigVerifierStats, ) -> Result<()> { +<<<<<<< HEAD let (mut batches, len, recv_time) = streamer::recv_batch(recvr)?; let mut verify_batch_time = Measure::start("sigverify_batch_time"); @@ -183,6 +184,22 @@ impl SigVerifyStage { Self::discard_excess_packets(&mut batches, MAX_SIGVERIFY_BATCH); } sendr.send(verifier.verify_batch(batches))?; +======= + let (mut batches, num_packets, recv_duration) = streamer::recv_packet_batches(recvr)?; + + let batches_len = batches.len(); + debug!( + "@{:?} verifier: verifying: {}", + timing::timestamp(), + num_packets, + ); + if num_packets > MAX_SIGVERIFY_BATCH { + Self::discard_excess_packets(&mut batches, MAX_SIGVERIFY_BATCH); + } + + let mut verify_batch_time = Measure::start("sigverify_batch_time"); + sendr.send(verifier.verify_batches(batches))?; +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) verify_batch_time.stop(); debug!( @@ -219,8 +236,8 @@ impl SigVerifyStage { } fn verifier_service( - packet_receiver: PacketReceiver, - verified_sender: CrossbeamSender>, + packet_receiver: PacketBatchReceiver, + verified_sender: CrossbeamSender>, verifier: &T, ) -> JoinHandle<()> { let verifier = verifier.clone(); @@ -255,8 +272,8 @@ impl SigVerifyStage { } fn verifier_services( - packet_receiver: PacketReceiver, - verified_sender: CrossbeamSender>, + packet_receiver: PacketBatchReceiver, + verified_sender: CrossbeamSender>, verifier: T, ) -> JoinHandle<()> { Self::verifier_service(packet_receiver, verified_sender, &verifier) @@ -271,11 +288,12 @@ impl SigVerifyStage { mod tests { use {super::*, solana_perf::packet::Packet}; - fn count_non_discard(packets: &[Packets]) -> usize { - packets + fn count_non_discard(packet_batches: &[PacketBatch]) -> usize { + packet_batches .iter() - .map(|pp| { - pp.packets + .map(|batch| { + batch + .packets .iter() .map(|p| if p.meta.discard { 0 } else { 1 }) .sum::() @@ -286,14 +304,14 @@ mod tests { #[test] fn test_packet_discard() { solana_logger::setup(); - let mut p = Packets::default(); - p.packets.resize(10, Packet::default()); - p.packets[3].meta.addr = [1u16; 8]; - let mut packets = vec![p]; + let mut batch = PacketBatch::default(); + batch.packets.resize(10, Packet::default()); + batch.packets[3].meta.addr = [1u16; 8]; + let mut batches = vec![batch]; let max = 3; - SigVerifyStage::discard_excess_packets(&mut packets, max); - assert_eq!(count_non_discard(&packets), max); - assert!(!packets[0].packets[0].meta.discard); - assert!(!packets[0].packets[3].meta.discard); + SigVerifyStage::discard_excess_packets(&mut batches, max); + assert_eq!(count_non_discard(&batches), max); + assert!(!batches[0].packets[0].meta.discard); + assert!(!batches[0].packets[3].meta.discard); } } diff --git a/core/src/verified_vote_packets.rs b/core/src/verified_vote_packets.rs index 55c7577c877..0f8efed6b9a 100644 --- a/core/src/verified_vote_packets.rs +++ b/core/src/verified_vote_packets.rs @@ -1,14 +1,145 @@ use { crate::{cluster_info_vote_listener::VerifiedLabelVotePacketsReceiver, result::Result}, +<<<<<<< HEAD solana_gossip::crds_value::CrdsValueLabel, solana_perf::packet::Packets, solana_sdk::clock::Slot, +======= + crossbeam_channel::Select, + solana_perf::packet::PacketBatch, + solana_runtime::bank::Bank, + solana_sdk::{ + account::from_account, clock::Slot, hash::Hash, pubkey::Pubkey, signature::Signature, + slot_hashes::SlotHashes, sysvar, + }, + solana_vote_program::vote_state::VoteTransaction, +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) std::{ collections::{hash_map::Entry, HashMap}, time::Duration, }, }; +<<<<<<< HEAD +======= +const MAX_VOTES_PER_VALIDATOR: usize = 1000; + +pub struct VerifiedVoteMetadata { + pub vote_account_key: Pubkey, + pub vote: Box, + pub packet_batch: PacketBatch, + pub signature: Signature, +} + +pub struct ValidatorGossipVotesIterator<'a> { + my_leader_bank: Arc, + slot_hashes: SlotHashes, + verified_vote_packets: &'a VerifiedVotePackets, + vote_account_keys: Vec, + previously_sent_to_bank_votes: &'a mut HashSet, +} + +impl<'a> ValidatorGossipVotesIterator<'a> { + pub fn new( + my_leader_bank: Arc, + verified_vote_packets: &'a VerifiedVotePackets, + previously_sent_to_bank_votes: &'a mut HashSet, + ) -> Self { + let slot_hashes_account = my_leader_bank.get_account(&sysvar::slot_hashes::id()); + + if slot_hashes_account.is_none() { + warn!( + "Slot hashes sysvar doesn't exist on bank {}", + my_leader_bank.slot() + ); + } + + let slot_hashes_account = slot_hashes_account.unwrap_or_default(); + let slot_hashes = from_account::(&slot_hashes_account).unwrap_or_default(); + + // TODO: my_leader_bank.vote_accounts() may not contain zero-staked validators + // in this epoch, but those validators may have stake warming up in the next epoch + let vote_account_keys: Vec = + my_leader_bank.vote_accounts().keys().copied().collect(); + + Self { + my_leader_bank, + slot_hashes, + verified_vote_packets, + vote_account_keys, + previously_sent_to_bank_votes, + } + } +} + +/// Each iteration returns all of the missing votes for a single validator, the votes +/// ordered from smallest to largest. +/// +/// Iterator is done after iterating through all vote accounts +impl<'a> Iterator for ValidatorGossipVotesIterator<'a> { + type Item = Vec; + + fn next(&mut self) -> Option { + // TODO: Maybe prioritize by stake weight + while !self.vote_account_keys.is_empty() { + let vote_account_key = self.vote_account_keys.pop().unwrap(); + // Get all the gossip votes we've queued up for this validator + // that are: + // 1) missing from the current leader bank + // 2) on the same fork + let validator_votes = self + .verified_vote_packets + .0 + .get(&vote_account_key) + .and_then(|validator_gossip_votes| { + // Fetch the validator's vote state from the bank + self.my_leader_bank + .vote_accounts() + .get(&vote_account_key) + .and_then(|(_stake, vote_account)| { + vote_account.vote_state().as_ref().ok().map(|vote_state| { + let start_vote_slot = + vote_state.last_voted_slot().map(|x| x + 1).unwrap_or(0); + // Filter out the votes that are outdated + validator_gossip_votes + .range((start_vote_slot, Hash::default())..) + .filter_map(|((slot, hash), (packet, tx_signature))| { + if self.previously_sent_to_bank_votes.contains(tx_signature) + { + return None; + } + // Don't send the same vote to the same bank multiple times + self.previously_sent_to_bank_votes.insert(*tx_signature); + // Filter out votes on the wrong fork (or too old to be) + // on this fork + if self + .slot_hashes + .get(slot) + .map(|found_hash| found_hash == hash) + .unwrap_or(false) + { + Some(packet.clone()) + } else { + None + } + }) + .collect::>() + }) + }) + }); + if let Some(validator_votes) = validator_votes { + if !validator_votes.is_empty() { + return Some(validator_votes); + } + } + } + None + } +} + +pub type SingleValidatorVotes = BTreeMap<(Slot, Hash), (PacketBatch, Signature)>; + +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) #[derive(Default)] pub struct VerifiedVotePackets(HashMap); @@ -32,8 +163,32 @@ impl VerifiedVotePackets { } while let Ok(vote_packets) = vote_packets_receiver.try_recv() { if would_be_leader { +<<<<<<< HEAD for (label, slot, packet) in vote_packets { self.0.insert(label, (*last_update_version, slot, packet)); +======= + for verfied_vote_metadata in gossip_votes { + let VerifiedVoteMetadata { + vote_account_key, + vote, + packet_batch, + signature, + } = verfied_vote_metadata; + if vote.is_empty() { + error!("Empty votes should have been filtered out earlier in the pipeline"); + continue; + } + let slot = vote.last_voted_slot().unwrap(); + let hash = vote.hash(); + + let validator_votes = self.0.entry(vote_account_key).or_default(); + validator_votes.insert((slot, hash), (packet_batch, signature)); + + if validator_votes.len() > MAX_VOTES_PER_VALIDATOR { + let smallest_key = validator_votes.keys().next().cloned().unwrap(); + validator_votes.remove(&smallest_key).unwrap(); + } +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) } } } @@ -90,6 +245,7 @@ mod tests { let label2 = CrdsValueLabel::Vote(1, pubkey); let mut verified_vote_packets = VerifiedVotePackets(HashMap::new()); +<<<<<<< HEAD let data = Packet { meta: Meta { repair: true, @@ -103,6 +259,282 @@ mod tests { verified_vote_packets .0 .insert(label1, (2, 42, none_empty_packets)); +======= + // Send a vote from `vote_account_key`, check that it was inserted + let vote_slot = 0; + let vote_hash = Hash::new_unique(); + let vote = Vote::new(vec![vote_slot], vote_hash); + s.send(vec![VerifiedVoteMetadata { + vote_account_key, + vote: Box::new(vote.clone()), + packet_batch: PacketBatch::default(), + signature: Signature::new(&[1u8; 64]), + }]) + .unwrap(); + verified_vote_packets + .receive_and_process_vote_packets(&r, true) + .unwrap(); + assert_eq!( + verified_vote_packets + .0 + .get(&vote_account_key) + .unwrap() + .len(), + 1 + ); + + // Same slot, same hash, should not be inserted + s.send(vec![VerifiedVoteMetadata { + vote_account_key, + vote: Box::new(vote), + packet_batch: PacketBatch::default(), + signature: Signature::new(&[1u8; 64]), + }]) + .unwrap(); + verified_vote_packets + .receive_and_process_vote_packets(&r, true) + .unwrap(); + assert_eq!( + verified_vote_packets + .0 + .get(&vote_account_key) + .unwrap() + .len(), + 1 + ); + + // Same slot, different hash, should still be inserted + let new_vote_hash = Hash::new_unique(); + let vote = Vote::new(vec![vote_slot], new_vote_hash); + s.send(vec![VerifiedVoteMetadata { + vote_account_key, + vote: Box::new(vote), + packet_batch: PacketBatch::default(), + signature: Signature::new(&[1u8; 64]), + }]) + .unwrap(); + verified_vote_packets + .receive_and_process_vote_packets(&r, true) + .unwrap(); + assert_eq!( + verified_vote_packets + .0 + .get(&vote_account_key) + .unwrap() + .len(), + 2 + ); + + // Different vote slot, should be inserted + let vote_slot = 1; + let vote_hash = Hash::new_unique(); + let vote = Vote::new(vec![vote_slot], vote_hash); + s.send(vec![VerifiedVoteMetadata { + vote_account_key, + vote: Box::new(vote), + packet_batch: PacketBatch::default(), + signature: Signature::new(&[2u8; 64]), + }]) + .unwrap(); + verified_vote_packets + .receive_and_process_vote_packets(&r, true) + .unwrap(); + assert_eq!( + verified_vote_packets + .0 + .get(&vote_account_key) + .unwrap() + .len(), + 3 + ); + + // No new messages, should time out + assert_matches!( + verified_vote_packets.receive_and_process_vote_packets(&r, true), + Err(Error::ReadyTimeout) + ); + } + + #[test] + fn test_verified_vote_packets_receive_and_process_vote_packets_max_len() { + let (s, r) = unbounded(); + let vote_account_key = solana_sdk::pubkey::new_rand(); + + // Construct the buffer + let mut verified_vote_packets = VerifiedVotePackets(HashMap::new()); + + // Send many more votes than the upper limit per validator + for _ in 0..2 * MAX_VOTES_PER_VALIDATOR { + let vote_slot = 0; + let vote_hash = Hash::new_unique(); + let vote = Vote::new(vec![vote_slot], vote_hash); + s.send(vec![VerifiedVoteMetadata { + vote_account_key, + vote: Box::new(vote), + packet_batch: PacketBatch::default(), + signature: Signature::new(&[1u8; 64]), + }]) + .unwrap(); + } + + // At most `MAX_VOTES_PER_VALIDATOR` should be stored per validator + verified_vote_packets + .receive_and_process_vote_packets(&r, true) + .unwrap(); + assert_eq!( + verified_vote_packets + .0 + .get(&vote_account_key) + .unwrap() + .len(), + MAX_VOTES_PER_VALIDATOR + ); + } + + #[test] + fn test_verified_vote_packets_validator_gossip_votes_iterator_wrong_fork() { + let (s, r) = unbounded(); + let vote_simulator = VoteSimulator::new(1); + let my_leader_bank = vote_simulator.bank_forks.read().unwrap().root_bank(); + let vote_account_key = vote_simulator.vote_pubkeys[0]; + + // Create a bunch of votes with random vote hashes, which should all be ignored + // since they are not on the same fork as `my_leader_bank`, i.e. their hashes do + // not exist in the SlotHashes sysvar for `my_leader_bank` + for _ in 0..MAX_VOTES_PER_VALIDATOR { + let vote_slot = 0; + let vote_hash = Hash::new_unique(); + let vote = Vote::new(vec![vote_slot], vote_hash); + s.send(vec![VerifiedVoteMetadata { + vote_account_key, + vote: Box::new(vote), + packet_batch: PacketBatch::default(), + signature: Signature::new_unique(), + }]) + .unwrap(); + } + + // Ingest the votes into the buffer + let mut verified_vote_packets = VerifiedVotePackets(HashMap::new()); + verified_vote_packets + .receive_and_process_vote_packets(&r, true) + .unwrap(); + + // Create tracker for previously sent bank votes + let mut previously_sent_to_bank_votes = HashSet::new(); + let mut gossip_votes_iterator = ValidatorGossipVotesIterator::new( + my_leader_bank, + &verified_vote_packets, + &mut previously_sent_to_bank_votes, + ); + + // Wrong fork, we should get no hashes + assert!(gossip_votes_iterator.next().is_none()); + } + + #[test] + fn test_verified_vote_packets_validator_gossip_votes_iterator_correct_fork() { + let (s, r) = unbounded(); + let num_validators = 2; + let vote_simulator = VoteSimulator::new(2); + let mut my_leader_bank = vote_simulator.bank_forks.read().unwrap().root_bank(); + + // Create a set of valid ancestor hashes for this fork + for _ in 0..MAX_ENTRIES { + my_leader_bank = Arc::new(Bank::new_from_parent( + &my_leader_bank, + &Pubkey::default(), + my_leader_bank.slot() + 1, + )); + } + let slot_hashes_account = my_leader_bank + .get_account(&sysvar::slot_hashes::id()) + .expect("Slot hashes sysvar must exist"); + let slot_hashes = from_account::(&slot_hashes_account).unwrap(); + + // Create valid votes + for i in 0..num_validators { + let vote_account_key = vote_simulator.vote_pubkeys[i]; + // Used to uniquely identify the packets for each validator + let num_packets = i + 1; + for (vote_slot, vote_hash) in slot_hashes.slot_hashes().iter() { + let vote = Vote::new(vec![*vote_slot], *vote_hash); + s.send(vec![VerifiedVoteMetadata { + vote_account_key, + vote: Box::new(vote), + packet_batch: PacketBatch::new(vec![Packet::default(); num_packets]), + signature: Signature::new_unique(), + }]) + .unwrap(); + } + } + + // Ingest the votes into the buffer + let mut verified_vote_packets = VerifiedVotePackets(HashMap::new()); + verified_vote_packets + .receive_and_process_vote_packets(&r, true) + .unwrap(); + + // Check we get two batches, one for each validator. Each batch + // should only contain a packets structure with the specific number + // of packets associated with that batch + assert_eq!(verified_vote_packets.0.len(), 2); + // Every validator should have `slot_hashes.slot_hashes().len()` votes + assert!(verified_vote_packets + .0 + .values() + .all(|validator_votes| validator_votes.len() == slot_hashes.slot_hashes().len())); + + let mut previously_sent_to_bank_votes = HashSet::new(); + let mut gossip_votes_iterator = ValidatorGossipVotesIterator::new( + my_leader_bank.clone(), + &verified_vote_packets, + &mut previously_sent_to_bank_votes, + ); + + // Get and verify batches + let num_expected_batches = 2; + for _ in 0..num_expected_batches { + let validator_batch: Vec = gossip_votes_iterator.next().unwrap(); + assert_eq!(validator_batch.len(), slot_hashes.slot_hashes().len()); + let expected_len = validator_batch[0].packets.len(); + assert!(validator_batch + .iter() + .all(|batch| batch.packets.len() == expected_len)); + } + + // Should be empty now + assert!(gossip_votes_iterator.next().is_none()); + + // If we construct another iterator, should return nothing because `previously_sent_to_bank_votes` + // should filter out everything + let mut gossip_votes_iterator = ValidatorGossipVotesIterator::new( + my_leader_bank.clone(), + &verified_vote_packets, + &mut previously_sent_to_bank_votes, + ); + assert!(gossip_votes_iterator.next().is_none()); + + // If we add a new vote, we should return it + my_leader_bank.freeze(); + let vote_slot = my_leader_bank.slot(); + let vote_hash = my_leader_bank.hash(); + let my_leader_bank = Arc::new(Bank::new_from_parent( + &my_leader_bank, + &Pubkey::default(), + my_leader_bank.slot() + 1, + )); + let vote_account_key = vote_simulator.vote_pubkeys[1]; + let vote = Box::new(Vote::new(vec![vote_slot], vote_hash)); + s.send(vec![VerifiedVoteMetadata { + vote_account_key, + vote, + packet_batch: PacketBatch::default(), + signature: Signature::new_unique(), + }]) + .unwrap(); + // Ingest the votes into the buffer +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) verified_vote_packets .0 .insert(label2, (1, 23, Packets::default())); diff --git a/core/src/window_service.rs b/core/src/window_service.rs index 2e8344cfe85..ef4f3460a88 100644 --- a/core/src/window_service.rs +++ b/core/src/window_service.rs @@ -22,7 +22,7 @@ use { }, solana_measure::measure::Measure, solana_metrics::{inc_new_counter_debug, inc_new_counter_error}, - solana_perf::packet::{Packet, Packets}, + solana_perf::packet::{Packet, PacketBatch}, solana_rayon_threadlimit::get_thread_count, solana_runtime::{bank::Bank, bank_forks::BankForks}, solana_sdk::{clock::Slot, packet::PACKET_DATA_SIZE, pubkey::Pubkey}, @@ -348,7 +348,7 @@ fn recv_window( blockstore: &Blockstore, bank_forks: &RwLock, insert_shred_sender: &CrossbeamSender<(Vec, Vec>)>, - verified_receiver: &CrossbeamReceiver>, + verified_receiver: &CrossbeamReceiver>, retransmit_sender: &Sender>, shred_filter: F, thread_pool: &ThreadPool, @@ -453,8 +453,12 @@ impl WindowService { #[allow(clippy::too_many_arguments)] pub(crate) fn new( blockstore: Arc, +<<<<<<< HEAD cluster_info: Arc, verified_receiver: CrossbeamReceiver>, +======= + verified_receiver: CrossbeamReceiver>, +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) retransmit_sender: Sender>, repair_socket: Arc, exit: Arc, @@ -624,7 +628,7 @@ impl WindowService { exit: Arc, blockstore: Arc, insert_sender: CrossbeamSender<(Vec, Vec>)>, - verified_receiver: CrossbeamReceiver>, + verified_receiver: CrossbeamReceiver>, shred_filter: F, bank_forks: Arc>, retransmit_sender: Sender>, diff --git a/gossip/src/cluster_info.rs b/gossip/src/cluster_info.rs index b5fd389addd..ab9991492ad 100644 --- a/gossip/src/cluster_info.rs +++ b/gossip/src/cluster_info.rs @@ -47,8 +47,8 @@ use { solana_perf::{ data_budget::DataBudget, packet::{ - limited_deserialize, to_packets_with_destination, Packet, Packets, PacketsRecycler, - PACKET_DATA_SIZE, + limited_deserialize, to_packet_batch_with_destination, Packet, PacketBatch, + PacketBatchRecycler, PACKET_DATA_SIZE, }, }, solana_rayon_threadlimit::get_thread_count, @@ -67,7 +67,7 @@ use { packet, sendmmsg::{multi_target_send, SendPktsError}, socket::SocketAddrSpace, - streamer::{PacketReceiver, PacketSender}, + streamer::{PacketBatchReceiver, PacketBatchSender}, }, solana_vote_program::{ vote_state::MAX_LOCKOUT_HISTORY, vote_transaction::parse_vote_transaction, @@ -1561,9 +1561,9 @@ impl ClusterInfo { &self, thread_pool: &ThreadPool, gossip_validators: Option<&HashSet>, - recycler: &PacketsRecycler, + recycler: &PacketBatchRecycler, stakes: &HashMap, - sender: &PacketSender, + sender: &PacketBatchSender, generate_pull_requests: bool, require_stake_for_gossip: bool, ) -> Result<(), GossipError> { @@ -1575,11 +1575,11 @@ impl ClusterInfo { require_stake_for_gossip, ); if !reqs.is_empty() { - let packets = to_packets_with_destination(recycler.clone(), &reqs); + let packet_batch = to_packet_batch_with_destination(recycler.clone(), &reqs); self.stats .packets_sent_gossip_requests_count - .add_relaxed(packets.packets.len() as u64); - sender.send(packets)?; + .add_relaxed(packet_batch.packets.len() as u64); + sender.send(packet_batch)?; } Ok(()) } @@ -1673,7 +1673,7 @@ impl ClusterInfo { pub fn gossip( self: Arc, bank_forks: Option>>, - sender: PacketSender, + sender: PacketBatchSender, gossip_validators: Option>, exit: Arc, ) -> JoinHandle<()> { @@ -1689,7 +1689,7 @@ impl ClusterInfo { let mut last_contact_info_trace = timestamp(); let mut last_contact_info_save = timestamp(); let mut entrypoints_processed = false; - let recycler = PacketsRecycler::default(); + let recycler = PacketBatchRecycler::default(); let crds_data = vec![ CrdsData::Version(Version::new(self.id())), CrdsData::NodeInstance(self.instance.with_wallclock(timestamp())), @@ -1815,10 +1815,14 @@ impl ClusterInfo { // from address, crds filter, caller contact info requests: Vec<(SocketAddr, CrdsFilter, CrdsValue)>, thread_pool: &ThreadPool, - recycler: &PacketsRecycler, + recycler: &PacketBatchRecycler, stakes: &HashMap, +<<<<<<< HEAD response_sender: &PacketSender, require_stake_for_gossip: bool, +======= + response_sender: &PacketBatchSender, +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) ) { let _st = ScopedTimer::from(&self.stats.handle_batch_pull_requests_time); if requests.is_empty() { @@ -1886,7 +1890,7 @@ impl ClusterInfo { &'a self, now: Instant, mut rng: &'a mut R, - packets: &'a mut Packets, + packet_batch: &'a mut PacketBatch, ) -> impl FnMut(&PullData) -> bool + 'a where R: Rng + CryptoRng, @@ -1899,7 +1903,7 @@ impl ClusterInfo { if let Some(ping) = ping { let ping = Protocol::PingMessage(ping); match Packet::from_data(Some(&node.1), ping) { - Ok(packet) => packets.packets.push(packet), + Ok(packet) => packet_batch.packets.push(packet), Err(err) => error!("failed to write ping packet: {:?}", err), }; } @@ -1926,11 +1930,15 @@ impl ClusterInfo { fn handle_pull_requests( &self, thread_pool: &ThreadPool, - recycler: &PacketsRecycler, + recycler: &PacketBatchRecycler, requests: Vec, stakes: &HashMap, +<<<<<<< HEAD require_stake_for_gossip: bool, ) -> Packets { +======= + ) -> PacketBatch { +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) const DEFAULT_EPOCH_DURATION_MS: u64 = DEFAULT_SLOTS_PER_EPOCH * DEFAULT_MS_PER_SLOT; let mut time = Measure::start("handle_pull_requests"); let callers = crds_value::filter_current(requests.iter().map(|r| &r.caller)); @@ -1938,12 +1946,12 @@ impl ClusterInfo { .process_pull_requests(callers.cloned(), timestamp()); let output_size_limit = self.update_data_budget(stakes.len()) / PULL_RESPONSE_MIN_SERIALIZED_SIZE; - let mut packets = - Packets::new_unpinned_with_recycler(recycler.clone(), 64, "handle_pull_requests"); + let mut packet_batch = + PacketBatch::new_unpinned_with_recycler(recycler.clone(), 64, "handle_pull_requests"); let (caller_and_filters, addrs): (Vec<_>, Vec<_>) = { let mut rng = rand::thread_rng(); let check_pull_request = - self.check_pull_request(Instant::now(), &mut rng, &mut packets); + self.check_pull_request(Instant::now(), &mut rng, &mut packet_batch); requests .into_iter() .filter(check_pull_request) @@ -1987,7 +1995,7 @@ impl ClusterInfo { }) .unzip(); if responses.is_empty() { - return packets; + return packet_batch; } let mut rng = rand::thread_rng(); let shuffle = WeightedShuffle::new(&mut rng, &scores).unwrap(); @@ -2001,7 +2009,7 @@ impl ClusterInfo { Ok(packet) => { if self.outbound_budget.take(packet.meta.size) { total_bytes += packet.meta.size; - packets.packets.push(packet); + packet_batch.packets.push(packet); sent += 1; } else { inc_new_counter_info!("gossip_pull_request-no_budget", 1); @@ -2021,7 +2029,7 @@ impl ClusterInfo { responses.len(), total_bytes ); - packets + packet_batch } fn handle_batch_pull_responses( @@ -2142,8 +2150,8 @@ impl ClusterInfo { fn handle_batch_ping_messages( &self, pings: I, - recycler: &PacketsRecycler, - response_sender: &PacketSender, + recycler: &PacketBatchRecycler, + response_sender: &PacketBatchSender, ) where I: IntoIterator, { @@ -2153,7 +2161,11 @@ impl ClusterInfo { } } - fn handle_ping_messages(&self, pings: I, recycler: &PacketsRecycler) -> Option + fn handle_ping_messages( + &self, + pings: I, + recycler: &PacketBatchRecycler, + ) -> Option where I: IntoIterator, { @@ -2174,9 +2186,12 @@ impl ClusterInfo { if packets.is_empty() { None } else { - let packets = - Packets::new_unpinned_with_recycler_data(recycler, "handle_ping_messages", packets); - Some(packets) + let packet_batch = PacketBatch::new_unpinned_with_recycler_data( + recycler, + "handle_ping_messages", + packets, + ); + Some(packet_batch) } } @@ -2199,10 +2214,14 @@ impl ClusterInfo { &self, messages: Vec<(Pubkey, Vec)>, thread_pool: &ThreadPool, - recycler: &PacketsRecycler, + recycler: &PacketBatchRecycler, stakes: &HashMap, +<<<<<<< HEAD response_sender: &PacketSender, require_stake_for_gossip: bool, +======= + response_sender: &PacketBatchSender, +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) ) { let _st = ScopedTimer::from(&self.stats.handle_batch_push_messages_time); if messages.is_empty() { @@ -2273,17 +2292,22 @@ impl ClusterInfo { if prune_messages.is_empty() { return; } - let mut packets = to_packets_with_destination(recycler.clone(), &prune_messages); - let num_prune_packets = packets.packets.len(); + let mut packet_batch = to_packet_batch_with_destination(recycler.clone(), &prune_messages); + let num_prune_packets = packet_batch.packets.len(); self.stats .push_response_count +<<<<<<< HEAD .add_relaxed(packets.packets.len() as u64); let new_push_requests = self.new_push_requests(stakes, require_stake_for_gossip); +======= + .add_relaxed(packet_batch.packets.len() as u64); + let new_push_requests = self.new_push_requests(stakes); +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) inc_new_counter_debug!("cluster_info-push_message-pushes", new_push_requests.len()); for (address, request) in new_push_requests { if ContactInfo::is_valid_address(&address, &self.socket_addr_space) { match Packet::from_data(Some(&address), &request) { - Ok(packet) => packets.packets.push(packet), + Ok(packet) => packet_batch.packets.push(packet), Err(err) => error!("failed to write push-request packet: {:?}", err), } } else { @@ -2295,8 +2319,8 @@ impl ClusterInfo { .add_relaxed(num_prune_packets as u64); self.stats .packets_sent_push_messages_count - .add_relaxed((packets.packets.len() - num_prune_packets) as u64); - let _ = response_sender.send(packets); + .add_relaxed((packet_batch.packets.len() - num_prune_packets) as u64); + let _ = response_sender.send(packet_batch); } fn require_stake_for_gossip( @@ -2330,8 +2354,8 @@ impl ClusterInfo { &self, packets: VecDeque<(/*from:*/ SocketAddr, Protocol)>, thread_pool: &ThreadPool, - recycler: &PacketsRecycler, - response_sender: &PacketSender, + recycler: &PacketBatchRecycler, + response_sender: &PacketBatchSender, stakes: &HashMap, feature_set: Option<&FeatureSet>, epoch_duration: Duration, @@ -2449,15 +2473,15 @@ impl ClusterInfo { // handling of requests/messages. fn run_socket_consume( &self, - receiver: &PacketReceiver, + receiver: &PacketBatchReceiver, sender: &Sender>, thread_pool: &ThreadPool, ) -> Result<(), GossipError> { const RECV_TIMEOUT: Duration = Duration::from_secs(1); let packets: Vec<_> = receiver.recv_timeout(RECV_TIMEOUT)?.packets.into(); let mut packets = VecDeque::from(packets); - for payload in receiver.try_iter() { - packets.extend(payload.packets.iter().cloned()); + for packet_batch in receiver.try_iter() { + packets.extend(packet_batch.packets.iter().cloned()); let excess_count = packets.len().saturating_sub(MAX_GOSSIP_TRAFFIC); if excess_count > 0 { packets.drain(0..excess_count); @@ -2489,10 +2513,10 @@ impl ClusterInfo { /// Process messages from the network fn run_listen( &self, - recycler: &PacketsRecycler, + recycler: &PacketBatchRecycler, bank_forks: Option<&RwLock>, receiver: &Receiver>, - response_sender: &PacketSender, + response_sender: &PacketBatchSender, thread_pool: &ThreadPool, last_print: &mut Instant, should_check_duplicate_instance: bool, @@ -2540,7 +2564,7 @@ impl ClusterInfo { pub(crate) fn start_socket_consume_thread( self: Arc, - receiver: PacketReceiver, + receiver: PacketBatchReceiver, sender: Sender>, exit: Arc, ) -> JoinHandle<()> { @@ -2570,12 +2594,12 @@ impl ClusterInfo { self: Arc, bank_forks: Option>>, requests_receiver: Receiver>, - response_sender: PacketSender, + response_sender: PacketBatchSender, should_check_duplicate_instance: bool, exit: Arc, ) -> JoinHandle<()> { let mut last_print = Instant::now(); - let recycler = PacketsRecycler::default(); + let recycler = PacketBatchRecycler::default(); let thread_pool = ThreadPoolBuilder::new() .num_threads(get_thread_count().min(8)) .thread_name(|i| format!("sol-gossip-work-{}", i)) @@ -2944,9 +2968,9 @@ pub fn push_messages_to_peer( let reqs: Vec<_> = ClusterInfo::split_gossip_messages(PUSH_MESSAGE_MAX_PAYLOAD_SIZE, messages) .map(move |payload| (peer_gossip, Protocol::PushMessage(self_id, payload))) .collect(); - let packets = to_packets_with_destination(PacketsRecycler::default(), &reqs); + let packet_batch = to_packet_batch_with_destination(PacketBatchRecycler::default(), &reqs); let sock = UdpSocket::bind("0.0.0.0:0").unwrap(); - packet::send_to(&packets, &sock, socket_addr_space)?; + packet::send_to(&packet_batch, &sock, socket_addr_space)?; Ok(()) } @@ -3189,7 +3213,7 @@ mod tests { .iter() .map(|ping| Pong::new(ping, &this_node).unwrap()) .collect(); - let recycler = PacketsRecycler::default(); + let recycler = PacketBatchRecycler::default(); let packets = cluster_info .handle_ping_messages( remote_nodes diff --git a/ledger/benches/sigverify_shreds.rs b/ledger/benches/sigverify_shreds.rs index 4a3de44fffe..16bfd7200ef 100644 --- a/ledger/benches/sigverify_shreds.rs +++ b/ledger/benches/sigverify_shreds.rs @@ -7,7 +7,7 @@ use { sigverify_shreds::{sign_shreds_cpu, sign_shreds_gpu, sign_shreds_gpu_pinned_keypair}, }, solana_perf::{ - packet::{Packet, Packets}, + packet::{Packet, PacketBatch}, recycler_cache::RecyclerCache, }, solana_sdk::signature::Keypair, @@ -21,13 +21,13 @@ const NUM_BATCHES: usize = 1; fn bench_sigverify_shreds_sign_gpu(bencher: &mut Bencher) { let recycler_cache = RecyclerCache::default(); - let mut packets = Packets::default(); - packets.packets.set_pinnable(); + let mut packet_batch = PacketBatch::default(); + packet_batch.packets.set_pinnable(); let slot = 0xdead_c0de; // need to pin explicitly since the resize will not cause re-allocation - packets.packets.reserve_and_pin(NUM_PACKETS); - packets.packets.resize(NUM_PACKETS, Packet::default()); - for p in packets.packets.iter_mut() { + packet_batch.packets.reserve_and_pin(NUM_PACKETS); + packet_batch.packets.resize(NUM_PACKETS, Packet::default()); + for p in packet_batch.packets.iter_mut() { let shred = Shred::new_from_data( slot, 0xc0de, @@ -41,25 +41,25 @@ fn bench_sigverify_shreds_sign_gpu(bencher: &mut Bencher) { ); shred.copy_to_packet(p); } - let mut batch = vec![packets; NUM_BATCHES]; + let mut batches = vec![packet_batch; NUM_BATCHES]; let keypair = Keypair::new(); let pinned_keypair = sign_shreds_gpu_pinned_keypair(&keypair, &recycler_cache); let pinned_keypair = Some(Arc::new(pinned_keypair)); //warmup for _ in 0..100 { - sign_shreds_gpu(&keypair, &pinned_keypair, &mut batch, &recycler_cache); + sign_shreds_gpu(&keypair, &pinned_keypair, &mut batches, &recycler_cache); } bencher.iter(|| { - sign_shreds_gpu(&keypair, &pinned_keypair, &mut batch, &recycler_cache); + sign_shreds_gpu(&keypair, &pinned_keypair, &mut batches, &recycler_cache); }) } #[bench] fn bench_sigverify_shreds_sign_cpu(bencher: &mut Bencher) { - let mut packets = Packets::default(); + let mut packet_batch = PacketBatch::default(); let slot = 0xdead_c0de; - packets.packets.resize(NUM_PACKETS, Packet::default()); - for p in packets.packets.iter_mut() { + packet_batch.packets.resize(NUM_PACKETS, Packet::default()); + for p in packet_batch.packets.iter_mut() { let shred = Shred::new_from_data( slot, 0xc0de, @@ -73,9 +73,9 @@ fn bench_sigverify_shreds_sign_cpu(bencher: &mut Bencher) { ); shred.copy_to_packet(p); } - let mut batch = vec![packets; NUM_BATCHES]; + let mut batches = vec![packet_batch; NUM_BATCHES]; let keypair = Keypair::new(); bencher.iter(|| { - sign_shreds_cpu(&keypair, &mut batch); + sign_shreds_cpu(&keypair, &mut batches); }) } diff --git a/ledger/src/entry.rs b/ledger/src/entry.rs index 272262f571b..feb2f564486 100644 --- a/ledger/src/entry.rs +++ b/ledger/src/entry.rs @@ -13,7 +13,17 @@ use { solana_measure::measure::Measure, solana_merkle_tree::MerkleTree, solana_metrics::*, +<<<<<<< HEAD:ledger/src/entry.rs solana_perf::{cuda_runtime::PinnedVec, perf_libs, recycler::Recycler}, +======= + solana_perf::{ + cuda_runtime::PinnedVec, + packet::{Packet, PacketBatch, PacketBatchRecycler, PACKETS_PER_BATCH}, + perf_libs, + recycler::Recycler, + sigverify, + }, +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)):entry/src/entry.rs solana_rayon_threadlimit::get_thread_count, solana_runtime::hashed_transaction::HashedTransaction, solana_sdk::{ @@ -267,6 +277,12 @@ pub struct EntryVerificationState { pub struct VerifyRecyclers { hash_recycler: Recycler>, tick_count_recycler: Recycler>, +<<<<<<< HEAD:ledger/src/entry.rs +======= + packet_recycler: PacketBatchRecycler, + out_recycler: Recycler>, + tx_offset_recycler: Recycler, +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)):entry/src/entry.rs } #[derive(PartialEq, Clone, Copy, Debug)] @@ -332,6 +348,199 @@ impl EntryVerificationState { } } +<<<<<<< HEAD:ledger/src/entry.rs +======= +pub fn verify_transactions( + entries: Vec, + verify: Arc Result + Send + Sync>, +) -> Result> { + PAR_THREAD_POOL.with(|thread_pool| { + thread_pool.borrow().install(|| { + entries + .into_par_iter() + .map(|entry| { + if entry.transactions.is_empty() { + Ok(EntryType::Tick(entry.hash)) + } else { + Ok(EntryType::Transactions( + entry + .transactions + .into_par_iter() + .map(verify.as_ref()) + .collect::>>()?, + )) + } + }) + .collect() + }) + }) +} + +pub fn start_verify_transactions( + entries: Vec, + skip_verification: bool, + verify_recyclers: VerifyRecyclers, + verify: Arc< + dyn Fn(VersionedTransaction, TransactionVerificationMode) -> Result + + Send + + Sync, + >, +) -> Result { + let api = perf_libs::api(); + + // Use the CPU if we have too few transactions for GPU signature verification to be worth it. + // We will also use the CPU if no acceleration API is used or if we're skipping + // the signature verification as we'd have nothing to do on the GPU in that case. + // TODO: make the CPU-to GPU crossover point dynamic, perhaps based on similar future + // heuristics to what might be used in sigverify::ed25519_verify when a dynamic crossover + // is introduced for that function (see TODO in sigverify::ed25519_verify) + let use_cpu = skip_verification + || api.is_none() + || entries + .iter() + .try_fold(0, |accum: usize, entry: &Entry| -> Option { + if accum.saturating_add(entry.transactions.len()) < 512 { + Some(accum.saturating_add(entry.transactions.len())) + } else { + None + } + }) + .is_some(); + + if use_cpu { + let verify_func = { + let verification_mode = if skip_verification { + TransactionVerificationMode::HashOnly + } else { + TransactionVerificationMode::FullVerification + }; + move |versioned_tx: VersionedTransaction| -> Result { + verify(versioned_tx, verification_mode) + } + }; + + let entries = verify_transactions(entries, Arc::new(verify_func)); + + match entries { + Ok(entries_val) => { + return Ok(EntrySigVerificationState { + verification_status: EntryVerificationStatus::Success, + entries: Some(entries_val), + device_verification_data: DeviceSigVerificationData::Cpu(), + gpu_verify_duration_us: 0, + }); + } + Err(err) => { + return Err(err); + } + } + } + + let verify_func = { + move |versioned_tx: VersionedTransaction| -> Result { + verify( + versioned_tx, + TransactionVerificationMode::HashAndVerifyPrecompiles, + ) + } + }; + let entries = verify_transactions(entries, Arc::new(verify_func)); + match entries { + Ok(entries) => { + let num_transactions: usize = entries + .iter() + .map(|entry: &EntryType| -> usize { + match entry { + EntryType::Transactions(transactions) => transactions.len(), + EntryType::Tick(_) => 0, + } + }) + .sum(); + + if num_transactions == 0 { + return Ok(EntrySigVerificationState { + verification_status: EntryVerificationStatus::Success, + entries: Some(entries), + device_verification_data: DeviceSigVerificationData::Cpu(), + gpu_verify_duration_us: 0, + }); + } + let entry_txs: Vec<&SanitizedTransaction> = entries + .iter() + .filter_map(|entry_type| match entry_type { + EntryType::Tick(_) => None, + EntryType::Transactions(transactions) => Some(transactions), + }) + .flatten() + .collect::>(); + let mut packet_batches = entry_txs + .par_iter() + .chunks(PACKETS_PER_BATCH) + .map(|slice| { + let vec_size = slice.len(); + let mut packet_batch = PacketBatch::new_with_recycler( + verify_recyclers.packet_recycler.clone(), + vec_size, + "entry-sig-verify", + ); + // We use set_len here instead of resize(num_transactions, Packet::default()), to save + // memory bandwidth and avoid writing a large amount of data that will be overwritten + // soon afterwards. As well, Packet::default() actually leaves the packet data + // uninitialized anyway, so the initilization would simply write junk into + // the vector anyway. + unsafe { + packet_batch.packets.set_len(vec_size); + } + let entry_tx_iter = slice + .into_par_iter() + .map(|tx| tx.to_versioned_transaction()); + + let res = packet_batch + .packets + .par_iter_mut() + .zip(entry_tx_iter) + .all(|pair| { + pair.0.meta = Meta::default(); + Packet::populate_packet(pair.0, None, &pair.1).is_ok() + }); + if res { + Ok(packet_batch) + } else { + Err(TransactionError::SanitizeFailure) + } + }) + .collect::>>()?; + + let tx_offset_recycler = verify_recyclers.tx_offset_recycler; + let out_recycler = verify_recyclers.out_recycler; + let gpu_verify_thread = thread::spawn(move || { + let mut verify_time = Measure::start("sigverify"); + sigverify::ed25519_verify( + &mut packet_batches, + &tx_offset_recycler, + &out_recycler, + false, + ); + let verified = packet_batches + .iter() + .all(|batch| batch.packets.iter().all(|p| !p.meta.discard)); + verify_time.stop(); + (verified, verify_time.as_us()) + }); + Ok(EntrySigVerificationState { + verification_status: EntryVerificationStatus::Pending, + entries: Some(entries), + device_verification_data: DeviceSigVerificationData::Gpu(GpuSigVerificationData { + thread_h: Some(gpu_verify_thread), + }), + gpu_verify_duration_us: 0, + }) + } + Err(err) => Err(err), + } +} + +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)):entry/src/entry.rs fn compare_hashes(computed_hash: Hash, ref_entry: &Entry) -> bool { let actual = if !ref_entry.transactions.is_empty() { let tx_hash = hash_transactions(&ref_entry.transactions); diff --git a/ledger/src/sigverify_shreds.rs b/ledger/src/sigverify_shreds.rs index 279d54e9033..76bf4eadbec 100644 --- a/ledger/src/sigverify_shreds.rs +++ b/ledger/src/sigverify_shreds.rs @@ -12,10 +12,10 @@ use { solana_metrics::inc_new_counter_debug, solana_perf::{ cuda_runtime::PinnedVec, - packet::{limited_deserialize, Packet, Packets}, + packet::{limited_deserialize, Packet, PacketBatch}, perf_libs, recycler_cache::RecyclerCache, - sigverify::{self, batch_size, TxOffset}, + sigverify::{self, count_packets_in_batches, TxOffset}, }, solana_rayon_threadlimit::get_thread_count, solana_sdk::{ @@ -76,22 +76,26 @@ pub fn verify_shred_cpu(packet: &Packet, slot_leaders: &HashMap) Some(1) } -fn verify_shreds_cpu(batches: &[Packets], slot_leaders: &HashMap) -> Vec> { +fn verify_shreds_cpu( + batches: &[PacketBatch], + slot_leaders: &HashMap, +) -> Vec> { use rayon::prelude::*; - let count = batch_size(batches); - debug!("CPU SHRED ECDSA for {}", count); + let packet_count = count_packets_in_batches(batches); + debug!("CPU SHRED ECDSA for {}", packet_count); let rv = SIGVERIFY_THREAD_POOL.install(|| { batches .into_par_iter() - .map(|p| { - p.packets + .map(|batch| { + batch + .packets .par_iter() .map(|p| verify_shred_cpu(p, slot_leaders).unwrap_or(0)) .collect() }) .collect() }); - inc_new_counter_debug!("ed25519_shred_verify_cpu", count); + inc_new_counter_debug!("ed25519_shred_verify_cpu", packet_count); rv } @@ -99,7 +103,7 @@ fn slot_key_data_for_gpu< T: Sync + Sized + Default + std::fmt::Debug + Eq + std::hash::Hash + Clone + Copy + AsRef<[u8]>, >( offset_start: usize, - batches: &[Packets], + batches: &[PacketBatch], slot_keys: &HashMap, recycler_cache: &RecyclerCache, ) -> (PinnedVec, TxOffset, usize) { @@ -108,8 +112,9 @@ fn slot_key_data_for_gpu< let slots: Vec> = SIGVERIFY_THREAD_POOL.install(|| { batches .into_par_iter() - .map(|p| { - p.packets + .map(|batch| { + batch + .packets .iter() .map(|packet| { let slot_start = size_of::() + size_of::(); @@ -173,7 +178,7 @@ fn vec_size_in_packets(keyvec: &PinnedVec) -> usize { } fn resize_vec(keyvec: &mut PinnedVec) -> usize { - //HACK: Pubkeys vector is passed along as a `Packets` buffer to the GPU + //HACK: Pubkeys vector is passed along as a `PacketBatch` buffer to the GPU //TODO: GPU needs a more opaque interface, which can handle variable sized structures for data //Pad the Pubkeys buffer such that it is bigger than a buffer of Packet sized elems let num_in_packets = (keyvec.len() + (size_of::() - 1)) / size_of::(); @@ -183,7 +188,7 @@ fn resize_vec(keyvec: &mut PinnedVec) -> usize { fn shred_gpu_offsets( mut pubkeys_end: usize, - batches: &[Packets], + batches: &[PacketBatch], recycler_cache: &RecyclerCache, ) -> (TxOffset, TxOffset, TxOffset, Vec>) { let mut signature_offsets = recycler_cache.offsets().allocate("shred_signatures"); @@ -221,7 +226,7 @@ fn shred_gpu_offsets( } pub fn verify_shreds_gpu( - batches: &[Packets], + batches: &[PacketBatch], slot_leaders: &HashMap, recycler_cache: &RecyclerCache, ) -> Vec> { @@ -233,10 +238,10 @@ pub fn verify_shreds_gpu( let mut elems = Vec::new(); let mut rvs = Vec::new(); - let count = batch_size(batches); + let packet_count = count_packets_in_batches(batches); let (pubkeys, pubkey_offsets, mut num_packets) = slot_key_data_for_gpu(0, batches, slot_leaders, recycler_cache); - //HACK: Pubkeys vector is passed along as a `Packets` buffer to the GPU + //HACK: Pubkeys vector is passed along as a `PacketBatch` buffer to the GPU //TODO: GPU needs a more opaque interface, which can handle variable sized structures for data let pubkeys_len = num_packets * size_of::(); trace!("num_packets: {}", num_packets); @@ -251,15 +256,15 @@ pub fn verify_shreds_gpu( num: num_packets as u32, }); - for p in batches { + for batch in batches { elems.push(perf_libs::Elems { - elems: p.packets.as_ptr(), - num: p.packets.len() as u32, + elems: batch.packets.as_ptr(), + num: batch.packets.len() as u32, }); let mut v = Vec::new(); - v.resize(p.packets.len(), 0); + v.resize(batch.packets.len(), 0); rvs.push(v); - num_packets += p.packets.len(); + num_packets += batch.packets.len(); } out.resize(signature_offsets.len(), 0); @@ -290,7 +295,7 @@ pub fn verify_shreds_gpu( sigverify::copy_return_values(&v_sig_lens, &out, &mut rvs); - inc_new_counter_debug!("ed25519_shred_verify_gpu", count); + inc_new_counter_debug!("ed25519_shred_verify_gpu", packet_count); rvs } @@ -316,18 +321,18 @@ fn sign_shred_cpu(keypair: &Keypair, packet: &mut Packet) { packet.data[0..sig_end].copy_from_slice(signature.as_ref()); } -pub fn sign_shreds_cpu(keypair: &Keypair, batches: &mut [Packets]) { +pub fn sign_shreds_cpu(keypair: &Keypair, batches: &mut [PacketBatch]) { use rayon::prelude::*; - let count = batch_size(batches); - debug!("CPU SHRED ECDSA for {}", count); + let packet_count = count_packets_in_batches(batches); + debug!("CPU SHRED ECDSA for {}", packet_count); SIGVERIFY_THREAD_POOL.install(|| { - batches.par_iter_mut().for_each(|p| { - p.packets[..] + batches.par_iter_mut().for_each(|batch| { + batch.packets[..] .par_iter_mut() .for_each(|mut p| sign_shred_cpu(keypair, &mut p)); }); }); - inc_new_counter_debug!("ed25519_shred_verify_cpu", count); + inc_new_counter_debug!("ed25519_shred_verify_cpu", packet_count); } pub fn sign_shreds_gpu_pinned_keypair(keypair: &Keypair, cache: &RecyclerCache) -> PinnedVec { @@ -350,14 +355,14 @@ pub fn sign_shreds_gpu_pinned_keypair(keypair: &Keypair, cache: &RecyclerCache) pub fn sign_shreds_gpu( keypair: &Keypair, pinned_keypair: &Option>>, - batches: &mut [Packets], + batches: &mut [PacketBatch], recycler_cache: &RecyclerCache, ) { let sig_size = size_of::(); let pubkey_size = size_of::(); let api = perf_libs::api(); - let count = batch_size(batches); - if api.is_none() || count < SIGN_SHRED_GPU_MIN || pinned_keypair.is_none() { + let packet_count = count_packets_in_batches(batches); + if api.is_none() || packet_count < SIGN_SHRED_GPU_MIN || pinned_keypair.is_none() { return sign_shreds_cpu(keypair, batches); } let api = api.unwrap(); @@ -370,10 +375,10 @@ pub fn sign_shreds_gpu( //should be zero let mut pubkey_offsets = recycler_cache.offsets().allocate("pubkey offsets"); - pubkey_offsets.resize(count, 0); + pubkey_offsets.resize(packet_count, 0); let mut secret_offsets = recycler_cache.offsets().allocate("secret_offsets"); - secret_offsets.resize(count, pubkey_size as u32); + secret_offsets.resize(packet_count, pubkey_size as u32); trace!("offset: {}", offset); let (signature_offsets, msg_start_offsets, msg_sizes, _v_sig_lens) = @@ -388,14 +393,14 @@ pub fn sign_shreds_gpu( num: num_keypair_packets as u32, }); - for p in batches.iter() { + for batch in batches.iter() { elems.push(perf_libs::Elems { - elems: p.packets.as_ptr(), - num: p.packets.len() as u32, + elems: batch.packets.as_ptr(), + num: batch.packets.len() as u32, }); let mut v = Vec::new(); - v.resize(p.packets.len(), 0); - num_packets += p.packets.len(); + v.resize(batch.packets.len(), 0); + num_packets += batch.packets.len(); } trace!("Starting verify num packets: {}", num_packets); @@ -447,7 +452,7 @@ pub fn sign_shreds_gpu( }); }); }); - inc_new_counter_debug!("ed25519_shred_sign_gpu", count); + inc_new_counter_debug!("ed25519_shred_sign_gpu", packet_count); } #[cfg(test)] @@ -506,7 +511,7 @@ pub mod tests { fn run_test_sigverify_shreds_cpu(slot: Slot) { solana_logger::setup(); - let mut batch = [Packets::default()]; + let mut batches = [PacketBatch::default()]; let mut shred = Shred::new_from_data( slot, 0xc0de, @@ -520,15 +525,15 @@ pub mod tests { ); let keypair = Keypair::new(); Shredder::sign_shred(&keypair, &mut shred); - batch[0].packets.resize(1, Packet::default()); - batch[0].packets[0].data[0..shred.payload.len()].copy_from_slice(&shred.payload); - batch[0].packets[0].meta.size = shred.payload.len(); + batches[0].packets.resize(1, Packet::default()); + batches[0].packets[0].data[0..shred.payload.len()].copy_from_slice(&shred.payload); + batches[0].packets[0].meta.size = shred.payload.len(); let leader_slots = [(slot, keypair.pubkey().to_bytes())] .iter() .cloned() .collect(); - let rv = verify_shreds_cpu(&batch, &leader_slots); + let rv = verify_shreds_cpu(&batches, &leader_slots); assert_eq!(rv, vec![vec![1]]); let wrong_keypair = Keypair::new(); @@ -536,19 +541,19 @@ pub mod tests { .iter() .cloned() .collect(); - let rv = verify_shreds_cpu(&batch, &leader_slots); + let rv = verify_shreds_cpu(&batches, &leader_slots); assert_eq!(rv, vec![vec![0]]); let leader_slots = HashMap::new(); - let rv = verify_shreds_cpu(&batch, &leader_slots); + let rv = verify_shreds_cpu(&batches, &leader_slots); assert_eq!(rv, vec![vec![0]]); let leader_slots = [(slot, keypair.pubkey().to_bytes())] .iter() .cloned() .collect(); - batch[0].packets[0].meta.size = 0; - let rv = verify_shreds_cpu(&batch, &leader_slots); + batches[0].packets[0].meta.size = 0; + let rv = verify_shreds_cpu(&batches, &leader_slots); assert_eq!(rv, vec![vec![0]]); } @@ -561,7 +566,7 @@ pub mod tests { solana_logger::setup(); let recycler_cache = RecyclerCache::default(); - let mut batch = [Packets::default()]; + let mut batches = [PacketBatch::default()]; let mut shred = Shred::new_from_data( slot, 0xc0de, @@ -575,9 +580,9 @@ pub mod tests { ); let keypair = Keypair::new(); Shredder::sign_shred(&keypair, &mut shred); - batch[0].packets.resize(1, Packet::default()); - batch[0].packets[0].data[0..shred.payload.len()].copy_from_slice(&shred.payload); - batch[0].packets[0].meta.size = shred.payload.len(); + batches[0].packets.resize(1, Packet::default()); + batches[0].packets[0].data[0..shred.payload.len()].copy_from_slice(&shred.payload); + batches[0].packets[0].meta.size = shred.payload.len(); let leader_slots = [ (std::u64::MAX, Pubkey::default().to_bytes()), @@ -586,7 +591,7 @@ pub mod tests { .iter() .cloned() .collect(); - let rv = verify_shreds_gpu(&batch, &leader_slots, &recycler_cache); + let rv = verify_shreds_gpu(&batches, &leader_slots, &recycler_cache); assert_eq!(rv, vec![vec![1]]); let wrong_keypair = Keypair::new(); @@ -597,14 +602,14 @@ pub mod tests { .iter() .cloned() .collect(); - let rv = verify_shreds_gpu(&batch, &leader_slots, &recycler_cache); + let rv = verify_shreds_gpu(&batches, &leader_slots, &recycler_cache); assert_eq!(rv, vec![vec![0]]); let leader_slots = [(std::u64::MAX, [0u8; 32])].iter().cloned().collect(); - let rv = verify_shreds_gpu(&batch, &leader_slots, &recycler_cache); + let rv = verify_shreds_gpu(&batches, &leader_slots, &recycler_cache); assert_eq!(rv, vec![vec![0]]); - batch[0].packets[0].meta.size = 0; + batches[0].packets[0].meta.size = 0; let leader_slots = [ (std::u64::MAX, Pubkey::default().to_bytes()), (slot, keypair.pubkey().to_bytes()), @@ -612,7 +617,7 @@ pub mod tests { .iter() .cloned() .collect(); - let rv = verify_shreds_gpu(&batch, &leader_slots, &recycler_cache); + let rv = verify_shreds_gpu(&batches, &leader_slots, &recycler_cache); assert_eq!(rv, vec![vec![0]]); } @@ -625,11 +630,11 @@ pub mod tests { solana_logger::setup(); let recycler_cache = RecyclerCache::default(); - let mut packets = Packets::default(); + let mut packet_batch = PacketBatch::default(); let num_packets = 32; let num_batches = 100; - packets.packets.resize(num_packets, Packet::default()); - for (i, p) in packets.packets.iter_mut().enumerate() { + packet_batch.packets.resize(num_packets, Packet::default()); + for (i, p) in packet_batch.packets.iter_mut().enumerate() { let shred = Shred::new_from_data( slot, 0xc0de, @@ -643,7 +648,7 @@ pub mod tests { ); shred.copy_to_packet(p); } - let mut batch = vec![packets; num_batches]; + let mut batches = vec![packet_batch; num_batches]; let keypair = Keypair::new(); let pinned_keypair = sign_shreds_gpu_pinned_keypair(&keypair, &recycler_cache); let pinned_keypair = Some(Arc::new(pinned_keypair)); @@ -655,14 +660,14 @@ pub mod tests { .cloned() .collect(); //unsigned - let rv = verify_shreds_gpu(&batch, &pubkeys, &recycler_cache); + let rv = verify_shreds_gpu(&batches, &pubkeys, &recycler_cache); assert_eq!(rv, vec![vec![0; num_packets]; num_batches]); //signed - sign_shreds_gpu(&keypair, &pinned_keypair, &mut batch, &recycler_cache); - let rv = verify_shreds_cpu(&batch, &pubkeys); + sign_shreds_gpu(&keypair, &pinned_keypair, &mut batches, &recycler_cache); + let rv = verify_shreds_cpu(&batches, &pubkeys); assert_eq!(rv, vec![vec![1; num_packets]; num_batches]); - let rv = verify_shreds_gpu(&batch, &pubkeys, &recycler_cache); + let rv = verify_shreds_gpu(&batches, &pubkeys, &recycler_cache); assert_eq!(rv, vec![vec![1; num_packets]; num_batches]); } @@ -674,7 +679,7 @@ pub mod tests { fn run_test_sigverify_shreds_sign_cpu(slot: Slot) { solana_logger::setup(); - let mut batch = [Packets::default()]; + let mut batches = [PacketBatch::default()]; let keypair = Keypair::new(); let shred = Shred::new_from_data( slot, @@ -687,9 +692,9 @@ pub mod tests { 0, 0xc0de, ); - batch[0].packets.resize(1, Packet::default()); - batch[0].packets[0].data[0..shred.payload.len()].copy_from_slice(&shred.payload); - batch[0].packets[0].meta.size = shred.payload.len(); + batches[0].packets.resize(1, Packet::default()); + batches[0].packets[0].data[0..shred.payload.len()].copy_from_slice(&shred.payload); + batches[0].packets[0].meta.size = shred.payload.len(); let pubkeys = [ (slot, keypair.pubkey().to_bytes()), (std::u64::MAX, Pubkey::default().to_bytes()), @@ -698,11 +703,11 @@ pub mod tests { .cloned() .collect(); //unsigned - let rv = verify_shreds_cpu(&batch, &pubkeys); + let rv = verify_shreds_cpu(&batches, &pubkeys); assert_eq!(rv, vec![vec![0]]); //signed - sign_shreds_cpu(&keypair, &mut batch); - let rv = verify_shreds_cpu(&batch, &pubkeys); + sign_shreds_cpu(&keypair, &mut batches); + let rv = verify_shreds_cpu(&batches, &pubkeys); assert_eq!(rv, vec![vec![1]]); } diff --git a/perf/benches/recycler.rs b/perf/benches/recycler.rs index 63410ffc856..0533e4a11eb 100644 --- a/perf/benches/recycler.rs +++ b/perf/benches/recycler.rs @@ -3,7 +3,7 @@ extern crate test; use { - solana_perf::{packet::PacketsRecycler, recycler::Recycler}, + solana_perf::{packet::PacketBatchRecycler, recycler::Recycler}, test::Bencher, }; @@ -11,7 +11,7 @@ use { fn bench_recycler(bencher: &mut Bencher) { solana_logger::setup(); - let recycler: PacketsRecycler = Recycler::default(); + let recycler: PacketBatchRecycler = Recycler::default(); for _ in 0..1000 { let _packet = recycler.allocate(""); diff --git a/perf/benches/sigverify.rs b/perf/benches/sigverify.rs index a3211cade62..7c60f362b7a 100644 --- a/perf/benches/sigverify.rs +++ b/perf/benches/sigverify.rs @@ -3,7 +3,7 @@ extern crate test; use { - solana_perf::{packet::to_packets_chunked, recycler::Recycler, sigverify, test_tx::test_tx}, + solana_perf::{packet::to_packet_batches, recycler::Recycler, sigverify, test_tx::test_tx}, test::Bencher, }; @@ -12,7 +12,7 @@ fn bench_sigverify(bencher: &mut Bencher) { let tx = test_tx(); // generate packet vector - let mut batches = to_packets_chunked(&std::iter::repeat(tx).take(128).collect::>(), 128); + let mut batches = to_packet_batches(&std::iter::repeat(tx).take(128).collect::>(), 128); let recycler = Recycler::default(); let recycler_out = Recycler::default(); @@ -28,7 +28,7 @@ fn bench_get_offsets(bencher: &mut Bencher) { // generate packet vector let mut batches = - to_packets_chunked(&std::iter::repeat(tx).take(1024).collect::>(), 1024); + to_packet_batches(&std::iter::repeat(tx).take(1024).collect::>(), 1024); let recycler = Recycler::default(); // verify packets diff --git a/perf/src/packet.rs b/perf/src/packet.rs index 59f9d8f7dfc..d8c163a7af0 100644 --- a/perf/src/packet.rs +++ b/perf/src/packet.rs @@ -13,13 +13,13 @@ pub const PACKETS_PER_BATCH: usize = 128; pub const NUM_RCVMMSGS: usize = 128; #[derive(Debug, Default, Clone)] -pub struct Packets { +pub struct PacketBatch { pub packets: PinnedVec, } -pub type PacketsRecycler = Recycler>; +pub type PacketBatchRecycler = Recycler>; -impl Packets { +impl PacketBatch { pub fn new(packets: Vec) -> Self { let packets = PinnedVec::from_vec(packets); Self { packets } @@ -27,48 +27,52 @@ impl Packets { pub fn with_capacity(capacity: usize) -> Self { let packets = PinnedVec::with_capacity(capacity); - Packets { packets } + PacketBatch { packets } } pub fn new_unpinned_with_recycler( - recycler: PacketsRecycler, + recycler: PacketBatchRecycler, size: usize, name: &'static str, ) -> Self { let mut packets = recycler.allocate(name); packets.reserve(size); - Packets { packets } + PacketBatch { packets } } - pub fn new_with_recycler(recycler: PacketsRecycler, size: usize, name: &'static str) -> Self { + pub fn new_with_recycler( + recycler: PacketBatchRecycler, + size: usize, + name: &'static str, + ) -> Self { let mut packets = recycler.allocate(name); packets.reserve_and_pin(size); - Packets { packets } + PacketBatch { packets } } pub fn new_with_recycler_data( - recycler: &PacketsRecycler, + recycler: &PacketBatchRecycler, name: &'static str, mut packets: Vec, ) -> Self { - let mut vec = Self::new_with_recycler(recycler.clone(), packets.len(), name); - vec.packets.append(&mut packets); - vec + let mut batch = Self::new_with_recycler(recycler.clone(), packets.len(), name); + batch.packets.append(&mut packets); + batch } pub fn new_unpinned_with_recycler_data( - recycler: &PacketsRecycler, + recycler: &PacketBatchRecycler, name: &'static str, mut packets: Vec, ) -> Self { - let mut vec = Self::new_unpinned_with_recycler(recycler.clone(), packets.len(), name); - vec.packets.append(&mut packets); - vec + let mut batch = Self::new_unpinned_with_recycler(recycler.clone(), packets.len(), name); + batch.packets.append(&mut packets); + batch } pub fn set_addr(&mut self, addr: &SocketAddr) { - for m in self.packets.iter_mut() { - m.meta.set_addr(addr); + for p in self.packets.iter_mut() { + p.meta.set_addr(addr); } } @@ -77,32 +81,32 @@ impl Packets { } } -pub fn to_packets_chunked(xs: &[T], chunks: usize) -> Vec { +pub fn to_packet_batches(xs: &[T], chunks: usize) -> Vec { let mut out = vec![]; for x in xs.chunks(chunks) { - let mut p = Packets::with_capacity(x.len()); - p.packets.resize(x.len(), Packet::default()); - for (i, o) in x.iter().zip(p.packets.iter_mut()) { - Packet::populate_packet(o, None, i).expect("serialize request"); + let mut batch = PacketBatch::with_capacity(x.len()); + batch.packets.resize(x.len(), Packet::default()); + for (i, packet) in x.iter().zip(batch.packets.iter_mut()) { + Packet::populate_packet(packet, None, i).expect("serialize request"); } - out.push(p); + out.push(batch); } out } #[cfg(test)] -pub fn to_packets(xs: &[T]) -> Vec { - to_packets_chunked(xs, NUM_PACKETS) +pub fn to_packet_batches_for_tests(xs: &[T]) -> Vec { + to_packet_batches(xs, NUM_PACKETS) } -pub fn to_packets_with_destination( - recycler: PacketsRecycler, +pub fn to_packet_batch_with_destination( + recycler: PacketBatchRecycler, dests_and_data: &[(SocketAddr, T)], -) -> Packets { - let mut out = Packets::new_unpinned_with_recycler( +) -> PacketBatch { + let mut out = PacketBatch::new_unpinned_with_recycler( recycler, dests_and_data.len(), - "to_packets_with_destination", + "to_packet_batch_with_destination", ); out.packets.resize(dests_and_data.len(), Packet::default()); for (dest_and_data, o) in dests_and_data.iter().zip(out.packets.iter_mut()) { @@ -143,21 +147,21 @@ mod tests { }; #[test] - fn test_to_packets() { + fn test_to_packet_batches() { let keypair = Keypair::new(); let hash = Hash::new(&[1; 32]); let tx = system_transaction::transfer(&keypair, &keypair.pubkey(), 1, hash); - let rv = to_packets(&[tx.clone(); 1]); + let rv = to_packet_batches_for_tests(&[tx.clone(); 1]); assert_eq!(rv.len(), 1); assert_eq!(rv[0].packets.len(), 1); #[allow(clippy::useless_vec)] - let rv = to_packets(&vec![tx.clone(); NUM_PACKETS]); + let rv = to_packet_batches_for_tests(&vec![tx.clone(); NUM_PACKETS]); assert_eq!(rv.len(), 1); assert_eq!(rv[0].packets.len(), NUM_PACKETS); #[allow(clippy::useless_vec)] - let rv = to_packets(&vec![tx; NUM_PACKETS + 1]); + let rv = to_packet_batches_for_tests(&vec![tx; NUM_PACKETS + 1]); assert_eq!(rv.len(), 2); assert_eq!(rv[0].packets.len(), NUM_PACKETS); assert_eq!(rv[1].packets.len(), 1); @@ -165,9 +169,10 @@ mod tests { #[test] fn test_to_packets_pinning() { - let recycler = PacketsRecycler::default(); + let recycler = PacketBatchRecycler::default(); for i in 0..2 { - let _first_packets = Packets::new_with_recycler(recycler.clone(), i + 1, "first one"); + let _first_packets = + PacketBatch::new_with_recycler(recycler.clone(), i + 1, "first one"); } } } diff --git a/perf/src/recycler.rs b/perf/src/recycler.rs index 381bf76cf19..d925a4eee75 100644 --- a/perf/src/recycler.rs +++ b/perf/src/recycler.rs @@ -183,7 +183,7 @@ impl RecyclerX { #[cfg(test)] mod tests { - use {super::*, crate::packet::PacketsRecycler, std::iter::repeat_with}; + use {super::*, crate::packet::PacketBatchRecycler, std::iter::repeat_with}; impl Reset for u64 { fn reset(&mut self) { @@ -210,7 +210,7 @@ mod tests { #[test] fn test_recycler_shrink() { let mut rng = rand::thread_rng(); - let recycler = PacketsRecycler::default(); + let recycler = PacketBatchRecycler::default(); // Allocate a burst of packets. const NUM_PACKETS: usize = RECYCLER_SHRINK_SIZE * 2; { diff --git a/perf/src/sigverify.rs b/perf/src/sigverify.rs index 1b605112190..66881bc9840 100644 --- a/perf/src/sigverify.rs +++ b/perf/src/sigverify.rs @@ -9,7 +9,7 @@ use solana_sdk::transaction::Transaction; use { crate::{ cuda_runtime::PinnedVec, - packet::{Packet, Packets}, + packet::{Packet, PacketBatch}, perf_libs, recycler::Recycler, }, @@ -156,8 +156,8 @@ fn verify_packet(packet: &mut Packet, reject_non_vote: bool) { } } -pub fn batch_size(batches: &[Packets]) -> usize { - batches.iter().map(|p| p.packets.len()).sum() +pub fn count_packets_in_batches(batches: &[PacketBatch]) -> usize { + batches.iter().map(|batch| batch.packets.len()).sum() } // internal function to be unit-tested; should be used only by get_packet_offsets @@ -336,7 +336,7 @@ fn check_for_simple_vote_transaction( } pub fn generate_offsets( - batches: &mut [Packets], + batches: &mut [PacketBatch], recycler: &Recycler, reject_non_vote: bool, ) -> TxOffsets { @@ -351,9 +351,9 @@ pub fn generate_offsets( msg_sizes.set_pinnable(); let mut current_offset: usize = 0; let mut v_sig_lens = Vec::new(); - batches.iter_mut().for_each(|p| { + batches.iter_mut().for_each(|batch| { let mut sig_lens = Vec::new(); - p.packets.iter_mut().for_each(|packet| { + batch.packets.iter_mut().for_each(|packet| { let packet_offsets = get_packet_offsets(packet, current_offset, reject_non_vote); sig_lens.push(packet_offsets.sig_len); @@ -388,30 +388,32 @@ pub fn generate_offsets( ) } -pub fn ed25519_verify_cpu(batches: &mut [Packets], reject_non_vote: bool) { +pub fn ed25519_verify_cpu(batches: &mut [PacketBatch], reject_non_vote: bool) { use rayon::prelude::*; - let count = batch_size(batches); - debug!("CPU ECDSA for {}", batch_size(batches)); + let packet_count = count_packets_in_batches(batches); + debug!("CPU ECDSA for {}", packet_count); PAR_THREAD_POOL.install(|| { - batches.into_par_iter().for_each(|p| { - p.packets + batches.into_par_iter().for_each(|batch| { + batch + .packets .par_iter_mut() .for_each(|p| verify_packet(p, reject_non_vote)) }) }); - inc_new_counter_debug!("ed25519_verify_cpu", count); + inc_new_counter_debug!("ed25519_verify_cpu", packet_count); } -pub fn ed25519_verify_disabled(batches: &mut [Packets]) { +pub fn ed25519_verify_disabled(batches: &mut [PacketBatch]) { use rayon::prelude::*; - let count = batch_size(batches); - debug!("disabled ECDSA for {}", batch_size(batches)); - batches.into_par_iter().for_each(|p| { - p.packets + let packet_count = count_packets_in_batches(batches); + debug!("disabled ECDSA for {}", packet_count); + batches.into_par_iter().for_each(|batch| { + batch + .packets .par_iter_mut() .for_each(|p| p.meta.discard = false) }); - inc_new_counter_debug!("ed25519_verify_disabled", count); + inc_new_counter_debug!("ed25519_verify_disabled", packet_count); } pub fn copy_return_values(sig_lens: &[Vec], out: &PinnedVec, rvs: &mut Vec>) { @@ -465,7 +467,7 @@ pub fn get_checked_scalar(scalar: &[u8; 32]) -> Result<[u8; 32], PacketError> { Ok(out) } -pub fn mark_disabled(batches: &mut [Packets], r: &[Vec]) { +pub fn mark_disabled(batches: &mut [PacketBatch], r: &[Vec]) { batches.iter_mut().zip(r).for_each(|(b, v)| { b.packets.iter_mut().zip(v).for_each(|(p, f)| { p.meta.discard = *f == 0; @@ -474,7 +476,7 @@ pub fn mark_disabled(batches: &mut [Packets], r: &[Vec]) { } pub fn ed25519_verify( - batches: &mut [Packets], + batches: &mut [PacketBatch], recycler: &Recycler, recycler_out: &Recycler>, reject_non_vote: bool, @@ -486,21 +488,21 @@ pub fn ed25519_verify( let api = api.unwrap(); use crate::packet::PACKET_DATA_SIZE; - let count = batch_size(batches); + let packet_count = count_packets_in_batches(batches); // micro-benchmarks show GPU time for smallest batch around 15-20ms // and CPU speed for 64-128 sigverifies around 10-20ms. 64 is a nice // power-of-two number around that accounting for the fact that the CPU // may be busy doing other things while being a real validator // TODO: dynamically adjust this crossover - if count < 64 { + if packet_count < 64 { return ed25519_verify_cpu(batches, reject_non_vote); } let (signature_offsets, pubkey_offsets, msg_start_offsets, msg_sizes, sig_lens) = generate_offsets(batches, recycler, reject_non_vote); - debug!("CUDA ECDSA for {}", batch_size(batches)); + debug!("CUDA ECDSA for {}", packet_count); debug!("allocating out.."); let mut out = recycler_out.allocate("out_buffer"); out.set_pinnable(); @@ -508,15 +510,15 @@ pub fn ed25519_verify( let mut rvs = Vec::new(); let mut num_packets: usize = 0; - for p in batches.iter() { + for batch in batches.iter() { elems.push(perf_libs::Elems { - elems: p.packets.as_ptr(), - num: p.packets.len() as u32, + elems: batch.packets.as_ptr(), + num: batch.packets.len() as u32, }); let mut v = Vec::new(); - v.resize(p.packets.len(), 0); + v.resize(batch.packets.len(), 0); rvs.push(v); - num_packets = num_packets.saturating_add(p.packets.len()); + num_packets = num_packets.saturating_add(batch.packets.len()); } out.resize(signature_offsets.len(), 0); trace!("Starting verify num packets: {}", num_packets); @@ -545,7 +547,7 @@ pub fn ed25519_verify( trace!("done verify"); copy_return_values(&sig_lens, &out, &mut rvs); mark_disabled(batches, &rvs); - inc_new_counter_debug!("ed25519_verify_gpu", count); + inc_new_counter_debug!("ed25519_verify_gpu", packet_count); } #[cfg(test)] @@ -565,7 +567,7 @@ mod tests { use { super::*, crate::{ - packet::{Packet, Packets}, + packet::{Packet, PacketBatch}, sigverify::{self, PacketOffsets}, test_tx::{test_multisig_tx, test_tx, vote_tx}, }, @@ -593,9 +595,9 @@ mod tests { #[test] fn test_mark_disabled() { - let mut batch = Packets::default(); + let mut batch = PacketBatch::default(); batch.packets.push(Packet::default()); - let mut batches: Vec = vec![batch]; + let mut batches: Vec = vec![batch]; mark_disabled(&mut batches, &[vec![0]]); assert!(batches[0].packets[0].meta.discard); mark_disabled(&mut batches, &[vec![1]]); @@ -717,7 +719,7 @@ mod tests { assert!(packet.meta.discard); packet.meta.discard = false; - let mut batches = generate_packet_vec(&packet, 1, 1); + let mut batches = generate_packet_batches(&packet, 1, 1); ed25519_verify(&mut batches); assert!(batches[0].packets[0].meta.discard); } @@ -753,7 +755,7 @@ mod tests { assert!(packet.meta.discard); packet.meta.discard = false; - let mut batches = generate_packet_vec(&packet, 1, 1); + let mut batches = generate_packet_batches(&packet, 1, 1); ed25519_verify(&mut batches); assert!(batches[0].packets[0].meta.discard); } @@ -878,21 +880,21 @@ mod tests { ); } - fn generate_packet_vec( + fn generate_packet_batches( packet: &Packet, num_packets_per_batch: usize, num_batches: usize, - ) -> Vec { + ) -> Vec { // generate packet vector let batches: Vec<_> = (0..num_batches) .map(|_| { - let mut packets = Packets::default(); - packets.packets.resize(0, Packet::default()); + let mut packet_batch = PacketBatch::default(); + packet_batch.packets.resize(0, Packet::default()); for _ in 0..num_packets_per_batch { - packets.packets.push(packet.clone()); + packet_batch.packets.push(packet.clone()); } - assert_eq!(packets.packets.len(), num_packets_per_batch); - packets + assert_eq!(packet_batch.packets.len(), num_packets_per_batch); + packet_batch }) .collect(); assert_eq!(batches.len(), num_batches); @@ -909,7 +911,7 @@ mod tests { packet.data[20] = packet.data[20].wrapping_add(10); } - let mut batches = generate_packet_vec(&packet, n, 2); + let mut batches = generate_packet_batches(&packet, n, 2); // verify packets ed25519_verify(&mut batches); @@ -918,11 +920,11 @@ mod tests { let should_discard = modify_data; assert!(batches .iter() - .flat_map(|p| &p.packets) + .flat_map(|batch| &batch.packets) .all(|p| p.meta.discard == should_discard)); } - fn ed25519_verify(batches: &mut [Packets]) { + fn ed25519_verify(batches: &mut [PacketBatch]) { let recycler = Recycler::default(); let recycler_out = Recycler::default(); sigverify::ed25519_verify(batches, &recycler, &recycler_out, false); @@ -935,13 +937,13 @@ mod tests { tx.signatures.pop(); let packet = sigverify::make_packet_from_transaction(tx); - let mut batches = generate_packet_vec(&packet, 1, 1); + let mut batches = generate_packet_batches(&packet, 1, 1); // verify packets ed25519_verify(&mut batches); assert!(batches .iter() - .flat_map(|p| &p.packets) + .flat_map(|batch| &batch.packets) .all(|p| p.meta.discard)); } @@ -969,7 +971,7 @@ mod tests { let n = 4; let num_batches = 3; - let mut batches = generate_packet_vec(&packet, n, num_batches); + let mut batches = generate_packet_batches(&packet, n, num_batches); packet.data[40] = packet.data[40].wrapping_add(8); @@ -984,7 +986,7 @@ mod tests { ref_vec[0].push(0u8); assert!(batches .iter() - .flat_map(|p| &p.packets) + .flat_map(|batch| &batch.packets) .zip(ref_vec.into_iter().flatten()) .all(|(p, discard)| { if discard == 0 { @@ -1008,7 +1010,7 @@ mod tests { for _ in 0..50 { let n = thread_rng().gen_range(1, 30); let num_batches = thread_rng().gen_range(2, 30); - let mut batches = generate_packet_vec(&packet, n, num_batches); + let mut batches = generate_packet_batches(&packet, n, num_batches); let num_modifications = thread_rng().gen_range(0, 5); for _ in 0..num_modifications { @@ -1029,8 +1031,8 @@ mod tests { // check result batches .iter() - .flat_map(|p| &p.packets) - .zip(batches_cpu.iter().flat_map(|p| &p.packets)) + .flat_map(|batch| &batch.packets) + .zip(batches_cpu.iter().flat_map(|batch| &batch.packets)) .for_each(|(p1, p2)| assert_eq!(p1, p2)); } } @@ -1182,7 +1184,7 @@ mod tests { solana_logger::setup(); let mut current_offset = 0usize; - let mut batch = Packets::default(); + let mut batch = PacketBatch::default(); batch .packets .push(sigverify::make_packet_from_transaction(test_tx())); diff --git a/streamer/src/packet.rs b/streamer/src/packet.rs index 58688ef80ec..b0abe551a3a 100644 --- a/streamer/src/packet.rs +++ b/streamer/src/packet.rs @@ -9,13 +9,13 @@ use { }; pub use { solana_perf::packet::{ - limited_deserialize, to_packets_chunked, Packets, PacketsRecycler, NUM_PACKETS, + limited_deserialize, to_packet_batches, PacketBatch, PacketBatchRecycler, NUM_PACKETS, PACKETS_PER_BATCH, }, solana_sdk::packet::{Meta, Packet, PACKET_DATA_SIZE}, }; -pub fn recv_from(obj: &mut Packets, socket: &UdpSocket, max_wait_ms: u64) -> Result { +pub fn recv_from(batch: &mut PacketBatch, socket: &UdpSocket, max_wait_ms: u64) -> Result { let mut i = 0; //DOCUMENTED SIDE-EFFECT //Performance out of the IO without poll @@ -27,11 +27,11 @@ pub fn recv_from(obj: &mut Packets, socket: &UdpSocket, max_wait_ms: u64) -> Res trace!("receiving on {}", socket.local_addr().unwrap()); let start = Instant::now(); loop { - obj.packets.resize( + batch.packets.resize( std::cmp::min(i + NUM_RCVMMSGS, PACKETS_PER_BATCH), Packet::default(), ); - match recv_mmsg(socket, &mut obj.packets[i..]) { + match recv_mmsg(socket, &mut batch.packets[i..]) { Err(_) if i > 0 => { if start.elapsed().as_millis() as u64 > max_wait_ms { break; @@ -55,17 +55,17 @@ pub fn recv_from(obj: &mut Packets, socket: &UdpSocket, max_wait_ms: u64) -> Res } } } - obj.packets.truncate(i); + batch.packets.truncate(i); inc_new_counter_debug!("packets-recv_count", i); Ok(i) } pub fn send_to( - obj: &Packets, + batch: &PacketBatch, socket: &UdpSocket, socket_addr_space: &SocketAddrSpace, ) -> Result<()> { - for p in &obj.packets { + for p in &batch.packets { let addr = p.meta.addr(); if socket_addr_space.check(&addr) { socket.send_to(&p.data[..p.meta.size], &addr)?; @@ -90,9 +90,9 @@ mod tests { // test that the address is actually being updated let send_addr: SocketAddr = "127.0.0.1:123".parse().unwrap(); let packets = vec![Packet::default()]; - let mut msgs = Packets::new(packets); - msgs.set_addr(&send_addr); - assert_eq!(msgs.packets[0].meta.addr(), send_addr); + let mut packet_batch = PacketBatch::new(packets); + packet_batch.set_addr(&send_addr); + assert_eq!(packet_batch.packets[0].meta.addr(), send_addr); } #[test] @@ -102,21 +102,21 @@ mod tests { let addr = recv_socket.local_addr().unwrap(); let send_socket = UdpSocket::bind("127.0.0.1:0").expect("bind"); let saddr = send_socket.local_addr().unwrap(); - let mut p = Packets::default(); + let mut batch = PacketBatch::default(); - p.packets.resize(10, Packet::default()); + batch.packets.resize(10, Packet::default()); - for m in p.packets.iter_mut() { + for m in batch.packets.iter_mut() { m.meta.set_addr(&addr); m.meta.size = PACKET_DATA_SIZE; } - send_to(&p, &send_socket, &SocketAddrSpace::Unspecified).unwrap(); + send_to(&batch, &send_socket, &SocketAddrSpace::Unspecified).unwrap(); - let recvd = recv_from(&mut p, &recv_socket, 1).unwrap(); + let recvd = recv_from(&mut batch, &recv_socket, 1).unwrap(); - assert_eq!(recvd, p.packets.len()); + assert_eq!(recvd, batch.packets.len()); - for m in &p.packets { + for m in &batch.packets { assert_eq!(m.meta.size, PACKET_DATA_SIZE); assert_eq!(m.meta.addr(), saddr); } @@ -125,7 +125,7 @@ mod tests { #[test] pub fn debug_trait() { write!(io::sink(), "{:?}", Packet::default()).unwrap(); - write!(io::sink(), "{:?}", Packets::default()).unwrap(); + write!(io::sink(), "{:?}", PacketBatch::default()).unwrap(); } #[test] @@ -151,25 +151,25 @@ mod tests { let recv_socket = UdpSocket::bind("127.0.0.1:0").expect("bind"); let addr = recv_socket.local_addr().unwrap(); let send_socket = UdpSocket::bind("127.0.0.1:0").expect("bind"); - let mut p = Packets::default(); - p.packets.resize(PACKETS_PER_BATCH, Packet::default()); + let mut batch = PacketBatch::default(); + batch.packets.resize(PACKETS_PER_BATCH, Packet::default()); // Should only get PACKETS_PER_BATCH packets per iteration even // if a lot more were sent, and regardless of packet size for _ in 0..2 * PACKETS_PER_BATCH { - let mut p = Packets::default(); - p.packets.resize(1, Packet::default()); - for m in p.packets.iter_mut() { + let mut batch = PacketBatch::default(); + batch.packets.resize(1, Packet::default()); + for m in batch.packets.iter_mut() { m.meta.set_addr(&addr); m.meta.size = 1; } - send_to(&p, &send_socket, &SocketAddrSpace::Unspecified).unwrap(); + send_to(&batch, &send_socket, &SocketAddrSpace::Unspecified).unwrap(); } - let recvd = recv_from(&mut p, &recv_socket, 100).unwrap(); + let recvd = recv_from(&mut batch, &recv_socket, 100).unwrap(); // Check we only got PACKETS_PER_BATCH packets assert_eq!(recvd, PACKETS_PER_BATCH); - assert_eq!(p.packets.capacity(), PACKETS_PER_BATCH); + assert_eq!(batch.packets.capacity(), PACKETS_PER_BATCH); } } diff --git a/streamer/src/streamer.rs b/streamer/src/streamer.rs index a6eebf8ad54..0e659dbe970 100644 --- a/streamer/src/streamer.rs +++ b/streamer/src/streamer.rs @@ -3,7 +3,7 @@ use { crate::{ - packet::{self, send_to, Packets, PacketsRecycler, PACKETS_PER_BATCH}, + packet::{self, send_to, PacketBatch, PacketBatchRecycler, PACKETS_PER_BATCH}, recvmmsg::NUM_RCVMMSGS, socket::SocketAddrSpace, }, @@ -21,8 +21,8 @@ use { thiserror::Error, }; -pub type PacketReceiver = Receiver; -pub type PacketSender = Sender; +pub type PacketBatchReceiver = Receiver; +pub type PacketBatchSender = Sender; #[derive(Error, Debug)] pub enum StreamerError { @@ -33,7 +33,7 @@ pub enum StreamerError { RecvTimeout(#[from] RecvTimeoutError), #[error("send packets error")] - Send(#[from] SendError), + Send(#[from] SendError), } pub type Result = std::result::Result; @@ -41,8 +41,8 @@ pub type Result = std::result::Result; fn recv_loop( sock: &UdpSocket, exit: Arc, - channel: &PacketSender, - recycler: &PacketsRecycler, + channel: &PacketBatchSender, + recycler: &PacketBatchRecycler, name: &'static str, coalesce_ms: u64, use_pinned_memory: bool, @@ -52,10 +52,10 @@ fn recv_loop( let mut now = Instant::now(); let mut num_max_received = 0; // Number of times maximum packets were received loop { - let mut msgs = if use_pinned_memory { - Packets::new_with_recycler(recycler.clone(), PACKETS_PER_BATCH, name) + let mut packet_batch = if use_pinned_memory { + PacketBatch::new_with_recycler(recycler.clone(), PACKETS_PER_BATCH, name) } else { - Packets::with_capacity(PACKETS_PER_BATCH) + PacketBatch::with_capacity(PACKETS_PER_BATCH) }; loop { // Check for exit signal, even if socket is busy @@ -63,14 +63,14 @@ fn recv_loop( if exit.load(Ordering::Relaxed) { return Ok(()); } - if let Ok(len) = packet::recv_from(&mut msgs, sock, coalesce_ms) { + if let Ok(len) = packet::recv_from(&mut packet_batch, sock, coalesce_ms) { if len == NUM_RCVMMSGS { num_max_received += 1; } recv_count += len; call_count += 1; if len > 0 { - channel.send(msgs)?; + channel.send(packet_batch)?; } break; } @@ -94,8 +94,8 @@ fn recv_loop( pub fn receiver( sock: Arc, exit: &Arc, - packet_sender: PacketSender, - recycler: PacketsRecycler, + packet_sender: PacketBatchSender, + recycler: PacketBatchRecycler, name: &'static str, coalesce_ms: u64, use_pinned_memory: bool, @@ -123,40 +123,55 @@ pub fn receiver( fn recv_send( sock: &UdpSocket, - r: &PacketReceiver, + r: &PacketBatchReceiver, socket_addr_space: &SocketAddrSpace, ) -> Result<()> { let timer = Duration::new(1, 0); - let msgs = r.recv_timeout(timer)?; - send_to(&msgs, sock, socket_addr_space)?; + let packet_batch = r.recv_timeout(timer)?; + send_to(&packet_batch, sock, socket_addr_space)?; Ok(()) } +<<<<<<< HEAD pub fn recv_batch(recvr: &PacketReceiver) -> Result<(Vec, usize, u64)> { +======= +pub fn recv_packet_batches( + recvr: &PacketBatchReceiver, +) -> Result<(Vec, usize, Duration)> { +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) let timer = Duration::new(1, 0); - let msgs = recvr.recv_timeout(timer)?; + let packet_batch = recvr.recv_timeout(timer)?; let recv_start = Instant::now(); - trace!("got msgs"); - let mut len = msgs.packets.len(); - let mut batch = vec![msgs]; - while let Ok(more) = recvr.try_recv() { - trace!("got more msgs"); - len += more.packets.len(); - batch.push(more); + trace!("got packets"); + let mut num_packets = packet_batch.packets.len(); + let mut packet_batches = vec![packet_batch]; + while let Ok(packet_batch) = recvr.try_recv() { + trace!("got more packets"); + num_packets += packet_batch.packets.len(); + packet_batches.push(packet_batch); } let recv_duration = recv_start.elapsed(); +<<<<<<< HEAD trace!("batch len {}", batch.len()); Ok(( batch, len, solana_sdk::timing::duration_as_ms(&recv_duration), )) +======= + trace!( + "packet batches len: {}, num packets: {}", + packet_batches.len(), + num_packets + ); + Ok((packet_batches, num_packets, recv_duration)) +>>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) } pub fn responder( name: &'static str, sock: Arc, - r: PacketReceiver, + r: PacketBatchReceiver, socket_addr_space: SocketAddrSpace, ) -> JoinHandle<()> { Builder::new() @@ -193,7 +208,7 @@ mod test { use { super::*, crate::{ - packet::{Packet, Packets, PACKET_DATA_SIZE}, + packet::{Packet, PacketBatch, PACKET_DATA_SIZE}, streamer::{receiver, responder}, }, solana_perf::recycler::Recycler, @@ -210,16 +225,16 @@ mod test { }, }; - fn get_msgs(r: PacketReceiver, num: &mut usize) { + fn get_packet_batches(r: PacketBatchReceiver, num_packets: &mut usize) { for _ in 0..10 { - let m = r.recv_timeout(Duration::new(1, 0)); - if m.is_err() { + let packet_batch_res = r.recv_timeout(Duration::new(1, 0)); + if packet_batch_res.is_err() { continue; } - *num -= m.unwrap().packets.len(); + *num_packets -= packet_batch_res.unwrap().packets.len(); - if *num == 0 { + if *num_packets == 0 { break; } } @@ -228,7 +243,7 @@ mod test { #[test] fn streamer_debug() { write!(io::sink(), "{:?}", Packet::default()).unwrap(); - write!(io::sink(), "{:?}", Packets::default()).unwrap(); + write!(io::sink(), "{:?}", PacketBatch::default()).unwrap(); } #[test] fn streamer_send_test() { @@ -256,23 +271,23 @@ mod test { r_responder, SocketAddrSpace::Unspecified, ); - let mut msgs = Packets::default(); + let mut packet_batch = PacketBatch::default(); for i in 0..5 { - let mut b = Packet::default(); + let mut p = Packet::default(); { - b.data[0] = i as u8; - b.meta.size = PACKET_DATA_SIZE; - b.meta.set_addr(&addr); + p.data[0] = i as u8; + p.meta.size = PACKET_DATA_SIZE; + p.meta.set_addr(&addr); } - msgs.packets.push(b); + packet_batch.packets.push(p); } - s_responder.send(msgs).expect("send"); + s_responder.send(packet_batch).expect("send"); t_responder }; - let mut num = 5; - get_msgs(r_reader, &mut num); - assert_eq!(num, 0); + let mut packets_remaining = 5; + get_packet_batches(r_reader, &mut packets_remaining); + assert_eq!(packets_remaining, 0); exit.store(true, Ordering::Relaxed); t_receiver.join().expect("join"); t_responder.join().expect("join"); From d14d45c7b8846281c401a2b11e1657bea0fdfd11 Mon Sep 17 00:00:00 2001 From: Justin Starry Date: Sat, 11 Dec 2021 10:31:22 -0500 Subject: [PATCH 2/2] resolve conflicts --- core/src/ancestor_hashes_service.rs | 1548 ------------------------ core/src/banking_stage.rs | 119 +- core/src/cluster_info_vote_listener.rs | 125 +- core/src/fetch_stage.rs | 7 - core/src/serve_repair.rs | 123 -- core/src/shred_fetch_stage.rs | 7 +- core/src/sigverify_stage.rs | 17 - core/src/verified_vote_packets.rs | 454 +------ core/src/window_service.rs | 4 - gossip/src/cluster_info.rs | 23 +- ledger/src/entry.rs | 209 ---- streamer/src/streamer.rs | 23 +- 12 files changed, 50 insertions(+), 2609 deletions(-) delete mode 100644 core/src/ancestor_hashes_service.rs diff --git a/core/src/ancestor_hashes_service.rs b/core/src/ancestor_hashes_service.rs deleted file mode 100644 index 4ccdb33338e..00000000000 --- a/core/src/ancestor_hashes_service.rs +++ /dev/null @@ -1,1548 +0,0 @@ -use { - crate::{ - cluster_slots::ClusterSlots, - duplicate_repair_status::{DeadSlotAncestorRequestStatus, DuplicateAncestorDecision}, - outstanding_requests::OutstandingRequests, - repair_response::{self}, - repair_service::{DuplicateSlotsResetSender, RepairInfo, RepairStatsGroup}, - replay_stage::DUPLICATE_THRESHOLD, - result::{Error, Result}, - serve_repair::{AncestorHashesRepairType, ServeRepair}, - }, - crossbeam_channel::{unbounded, Receiver, Sender}, - dashmap::{mapref::entry::Entry::Occupied, DashMap}, - solana_ledger::{blockstore::Blockstore, shred::SIZE_OF_NONCE}, - solana_measure::measure::Measure, - solana_perf::{ - packet::{limited_deserialize, Packet, PacketBatch}, - recycler::Recycler, - }, - solana_runtime::bank::Bank, - solana_sdk::{ - clock::{Slot, SLOT_MS}, - pubkey::Pubkey, - timing::timestamp, - }, - solana_streamer::streamer::{self, PacketBatchReceiver}, - std::{ - collections::HashSet, - net::UdpSocket, - sync::{ - atomic::{AtomicBool, Ordering}, - mpsc::channel, - Arc, RwLock, - }, - thread::{self, sleep, Builder, JoinHandle}, - time::{Duration, Instant}, - }, -}; - -#[derive(Debug, PartialEq)] -pub enum AncestorHashesReplayUpdate { - Dead(Slot), - DeadDuplicateConfirmed(Slot), -} - -impl AncestorHashesReplayUpdate { - fn slot(&self) -> Slot { - match self { - AncestorHashesReplayUpdate::Dead(slot) => *slot, - AncestorHashesReplayUpdate::DeadDuplicateConfirmed(slot) => *slot, - } - } -} - -pub const MAX_ANCESTOR_HASHES_SLOT_REQUESTS_PER_SECOND: usize = 2; - -pub type AncestorHashesReplayUpdateSender = Sender; -pub type AncestorHashesReplayUpdateReceiver = Receiver; - -type RetryableSlotsSender = Sender; -type RetryableSlotsReceiver = Receiver; -type OutstandingAncestorHashesRepairs = OutstandingRequests; - -#[derive(Default)] -pub struct AncestorHashesResponsesStats { - pub total_packets: usize, - pub dropped_packets: usize, - pub invalid_packets: usize, - pub processed: usize, -} - -impl AncestorHashesResponsesStats { - fn report(&mut self) { - inc_new_counter_info!( - "ancestor_hashes_responses-total_packets", - self.total_packets - ); - inc_new_counter_info!("ancestor_hashes_responses-processed", self.processed); - inc_new_counter_info!( - "ancestor_hashes_responses-dropped_packets", - self.dropped_packets - ); - inc_new_counter_info!( - "ancestor_hashes_responses-invalid_packets", - self.invalid_packets - ); - *self = AncestorHashesResponsesStats::default(); - } -} - -pub struct AncestorRepairRequestsStats { - pub ancestor_requests: RepairStatsGroup, - last_report: Instant, -} - -impl Default for AncestorRepairRequestsStats { - fn default() -> Self { - AncestorRepairRequestsStats { - ancestor_requests: RepairStatsGroup::default(), - last_report: Instant::now(), - } - } -} - -impl AncestorRepairRequestsStats { - fn report(&mut self) { - let slot_to_count: Vec<_> = self - .ancestor_requests - .slot_pubkeys - .iter() - .map(|(slot, slot_repairs)| { - ( - slot, - slot_repairs - .pubkey_repairs() - .iter() - .map(|(_key, count)| count) - .sum::(), - ) - }) - .collect(); - - let repair_total = self.ancestor_requests.count; - if self.last_report.elapsed().as_secs() > 2 && repair_total > 0 { - info!("ancestor_repair_requests_stats: {:?}", slot_to_count); - datapoint_info!( - "ancestor-repair", - ("ancestor-repair-count", self.ancestor_requests.count, i64) - ); - - *self = AncestorRepairRequestsStats::default(); - } - } -} - -pub struct AncestorHashesService { - thread_hdls: Vec>, -} - -impl AncestorHashesService { - pub fn new( - exit: Arc, - blockstore: Arc, - ancestor_hashes_request_socket: Arc, - repair_info: RepairInfo, - ancestor_hashes_replay_update_receiver: AncestorHashesReplayUpdateReceiver, - ) -> Self { - let outstanding_requests: Arc> = - Arc::new(RwLock::new(OutstandingAncestorHashesRepairs::default())); - let (response_sender, response_receiver) = channel(); - let t_receiver = streamer::receiver( - ancestor_hashes_request_socket.clone(), - &exit, - response_sender, - Recycler::default(), - "ancestor_hashes_response_receiver", - 1, - false, - ); - - let ancestor_hashes_request_statuses: Arc> = - Arc::new(DashMap::new()); - let (retryable_slots_sender, retryable_slots_receiver) = unbounded(); - - // Listen for responses to our ancestor requests - let t_ancestor_hashes_responses = Self::run_responses_listener( - ancestor_hashes_request_statuses.clone(), - response_receiver, - blockstore, - outstanding_requests.clone(), - exit.clone(), - repair_info.duplicate_slots_reset_sender.clone(), - retryable_slots_sender, - ); - - // Generate ancestor requests for dead slots that are repairable - let t_ancestor_requests = Self::run_manage_ancestor_requests( - ancestor_hashes_request_statuses, - ancestor_hashes_request_socket, - repair_info, - outstanding_requests, - exit, - ancestor_hashes_replay_update_receiver, - retryable_slots_receiver, - ); - let thread_hdls = vec![t_receiver, t_ancestor_hashes_responses, t_ancestor_requests]; - Self { thread_hdls } - } - - pub fn join(self) -> thread::Result<()> { - for thread_hdl in self.thread_hdls { - thread_hdl.join()?; - } - Ok(()) - } - - /// Listen for responses to our ancestors hashes repair requests - fn run_responses_listener( - ancestor_hashes_request_statuses: Arc>, - response_receiver: PacketBatchReceiver, - blockstore: Arc, - outstanding_requests: Arc>, - exit: Arc, - duplicate_slots_reset_sender: DuplicateSlotsResetSender, - retryable_slots_sender: RetryableSlotsSender, - ) -> JoinHandle<()> { - Builder::new() - .name("solana-ancestor-hashes-responses-service".to_string()) - .spawn(move || { - let mut last_stats_report = Instant::now(); - let mut stats = AncestorHashesResponsesStats::default(); - let mut max_packets = 1024; - loop { - let result = Self::process_new_packets_from_channel( - &ancestor_hashes_request_statuses, - &response_receiver, - &blockstore, - &outstanding_requests, - &mut stats, - &mut max_packets, - &duplicate_slots_reset_sender, - &retryable_slots_sender, - ); - match result { - Err(Error::RecvTimeout(_)) | Ok(_) => {} - Err(err) => info!("ancestors hashes reponses listener error: {:?}", err), - }; - if exit.load(Ordering::Relaxed) { - return; - } - if last_stats_report.elapsed().as_secs() > 2 { - stats.report(); - last_stats_report = Instant::now(); - } - } - }) - .unwrap() - } - - /// Process messages from the network - fn process_new_packets_from_channel( - ancestor_hashes_request_statuses: &DashMap, - response_receiver: &PacketBatchReceiver, - blockstore: &Blockstore, - outstanding_requests: &RwLock, - stats: &mut AncestorHashesResponsesStats, - max_packets: &mut usize, - duplicate_slots_reset_sender: &DuplicateSlotsResetSender, - retryable_slots_sender: &RetryableSlotsSender, - ) -> Result<()> { - let timeout = Duration::new(1, 0); - let mut packet_batches = vec![response_receiver.recv_timeout(timeout)?]; - let mut total_packets = packet_batches[0].packets.len(); - - let mut dropped_packets = 0; - while let Ok(batch) = response_receiver.try_recv() { - total_packets += batch.packets.len(); - if total_packets < *max_packets { - // Drop the rest in the channel in case of DOS - packet_batches.push(batch); - } else { - dropped_packets += batch.packets.len(); - } - } - - stats.dropped_packets += dropped_packets; - stats.total_packets += total_packets; - - let mut time = Measure::start("ancestor_hashes::handle_packets"); - for packet_batch in packet_batches { - Self::process_packet_batch( - ancestor_hashes_request_statuses, - packet_batch, - stats, - outstanding_requests, - blockstore, - duplicate_slots_reset_sender, - retryable_slots_sender, - ); - } - time.stop(); - if total_packets >= *max_packets { - if time.as_ms() > 1000 { - *max_packets = (*max_packets * 9) / 10; - } else { - *max_packets = (*max_packets * 10) / 9; - } - } - Ok(()) - } - - fn process_packet_batch( - ancestor_hashes_request_statuses: &DashMap, - packet_batch: PacketBatch, - stats: &mut AncestorHashesResponsesStats, - outstanding_requests: &RwLock, - blockstore: &Blockstore, - duplicate_slots_reset_sender: &DuplicateSlotsResetSender, - retryable_slots_sender: &RetryableSlotsSender, - ) { - packet_batch.packets.iter().for_each(|packet| { - let decision = Self::verify_and_process_ancestor_response( - packet, - ancestor_hashes_request_statuses, - stats, - outstanding_requests, - blockstore, - ); - if let Some((slot, decision)) = decision { - Self::handle_ancestor_request_decision( - slot, - decision, - duplicate_slots_reset_sender, - retryable_slots_sender, - ); - } - }); - } - - /// Returns `Some((request_slot, decision))`, where `decision` is an actionable - /// result after processing sufficient responses for the subject of the query, - /// `request_slot` - fn verify_and_process_ancestor_response( - packet: &Packet, - ancestor_hashes_request_statuses: &DashMap, - stats: &mut AncestorHashesResponsesStats, - outstanding_requests: &RwLock, - blockstore: &Blockstore, - ) -> Option<(Slot, DuplicateAncestorDecision)> { - let from_addr = packet.meta.addr(); - limited_deserialize(&packet.data[..packet.meta.size - SIZE_OF_NONCE]) - .ok() - .and_then(|ancestor_hashes_response| { - // Verify the response - let request_slot = repair_response::nonce(&packet.data[..packet.meta.size]) - .and_then(|nonce| { - outstanding_requests.write().unwrap().register_response( - nonce, - &ancestor_hashes_response, - timestamp(), - // If the response is valid, return the slot the request - // was for - |ancestor_hashes_request| ancestor_hashes_request.0, - ) - }); - - if request_slot.is_none() { - stats.invalid_packets += 1; - return None; - } - - // If was a valid response, there must be a valid `request_slot` - let request_slot = request_slot.unwrap(); - stats.processed += 1; - - if let Occupied(mut ancestor_hashes_status_ref) = - ancestor_hashes_request_statuses.entry(request_slot) - { - let decision = ancestor_hashes_status_ref.get_mut().add_response( - &from_addr, - ancestor_hashes_response.into_slot_hashes(), - blockstore, - ); - if decision.is_some() { - // Once a request is completed, remove it from the map so that new - // requests for the same slot can be made again if necessary. It's - // important to hold the `write` lock here via - // `ancestor_hashes_status_ref` so that we don't race with deletion + - // insertion from the `t_ancestor_requests` thread, which may - // 1) Remove expired statuses from `ancestor_hashes_request_statuses` - // 2) Insert another new one via `manage_ancestor_requests()`. - // In which case we wouldn't want to delete the newly inserted entry here. - ancestor_hashes_status_ref.remove(); - } - decision.map(|decision| (request_slot, decision)) - } else { - None - } - }) - } - - fn handle_ancestor_request_decision( - slot: Slot, - decision: DuplicateAncestorDecision, - duplicate_slots_reset_sender: &DuplicateSlotsResetSender, - retryable_slots_sender: &RetryableSlotsSender, - ) { - if decision.is_retryable() { - let _ = retryable_slots_sender.send(slot); - } - let potential_slots_to_dump = { - // TODO: In the case of DuplicateAncestorDecision::ContinueSearch - // This means all the ancestors were mismatched, which - // means the earliest mismatched ancestor has yet to be found. - // - // In the best case scenario, this means after ReplayStage dumps - // the earliest known ancestor `A` here, and then repairs `A`, - // because we may still have the incorrect version of some ancestor - // of `A`, we will mark `A` as dead and then continue the search - // protocol through another round of ancestor repairs. - // - // However this process is a bit slow, so in an ideal world, the - // protocol could be extended to keep searching by making - // another ancestor repair request from the earliest returned - // ancestor from this search. - decision - .repair_status() - .map(|status| status.correct_ancestors_to_repair.clone()) - }; - - // Now signal ReplayStage about the new updated slots. It's important to do this - // AFTER we've removed the ancestor_hashes_status_ref in case replay - // then sends us another dead slot signal based on the updates we are - // about to send. - if let Some(potential_slots_to_dump) = potential_slots_to_dump { - // Signal ReplayStage to dump the fork that is descended from - // `earliest_mismatched_slot_to_dump`. - if !potential_slots_to_dump.is_empty() { - let _ = duplicate_slots_reset_sender.send(potential_slots_to_dump); - } - } - } - - fn process_replay_updates( - ancestor_hashes_replay_update_receiver: &AncestorHashesReplayUpdateReceiver, - ancestor_hashes_request_statuses: &DashMap, - dead_slot_pool: &mut HashSet, - repairable_dead_slot_pool: &mut HashSet, - root_slot: Slot, - ) { - for update in ancestor_hashes_replay_update_receiver.try_iter() { - let slot = update.slot(); - if slot <= root_slot || ancestor_hashes_request_statuses.contains_key(&slot) { - return; - } - match update { - AncestorHashesReplayUpdate::Dead(dead_slot) => { - if repairable_dead_slot_pool.contains(&dead_slot) { - return; - } else { - dead_slot_pool.insert(dead_slot); - } - } - AncestorHashesReplayUpdate::DeadDuplicateConfirmed(dead_slot) => { - dead_slot_pool.remove(&dead_slot); - repairable_dead_slot_pool.insert(dead_slot); - } - } - } - } - - fn run_manage_ancestor_requests( - ancestor_hashes_request_statuses: Arc>, - ancestor_hashes_request_socket: Arc, - repair_info: RepairInfo, - outstanding_requests: Arc>, - exit: Arc, - ancestor_hashes_replay_update_receiver: AncestorHashesReplayUpdateReceiver, - retryable_slots_receiver: RetryableSlotsReceiver, - ) -> JoinHandle<()> { - let serve_repair = ServeRepair::new(repair_info.cluster_info.clone()); - let mut repair_stats = AncestorRepairRequestsStats::default(); - - let mut dead_slot_pool = HashSet::new(); - let mut repairable_dead_slot_pool = HashSet::new(); - - // Sliding window that limits the number of slots repaired via AncestorRepair - // to MAX_ANCESTOR_HASHES_SLOT_REQUESTS_PER_SECOND/second - let mut request_throttle = vec![]; - Builder::new() - .name("solana-manage-ancestor-requests".to_string()) - .spawn(move || loop { - if exit.load(Ordering::Relaxed) { - return; - } - - Self::manage_ancestor_requests( - &ancestor_hashes_request_statuses, - &ancestor_hashes_request_socket, - &repair_info, - &outstanding_requests, - &ancestor_hashes_replay_update_receiver, - &retryable_slots_receiver, - &serve_repair, - &mut repair_stats, - &mut dead_slot_pool, - &mut repairable_dead_slot_pool, - &mut request_throttle, - ); - - sleep(Duration::from_millis(SLOT_MS)); - }) - .unwrap() - } - - #[allow(clippy::too_many_arguments)] - fn manage_ancestor_requests( - ancestor_hashes_request_statuses: &DashMap, - ancestor_hashes_request_socket: &UdpSocket, - repair_info: &RepairInfo, - outstanding_requests: &RwLock, - ancestor_hashes_replay_update_receiver: &AncestorHashesReplayUpdateReceiver, - retryable_slots_receiver: &RetryableSlotsReceiver, - serve_repair: &ServeRepair, - repair_stats: &mut AncestorRepairRequestsStats, - dead_slot_pool: &mut HashSet, - repairable_dead_slot_pool: &mut HashSet, - request_throttle: &mut Vec, - ) { - let root_bank = repair_info.bank_forks.read().unwrap().root_bank(); - for slot in retryable_slots_receiver.try_iter() { - datapoint_info!("ancestor-repair-retry", ("slot", slot, i64)); - repairable_dead_slot_pool.insert(slot); - } - - Self::process_replay_updates( - ancestor_hashes_replay_update_receiver, - ancestor_hashes_request_statuses, - dead_slot_pool, - repairable_dead_slot_pool, - root_bank.slot(), - ); - - Self::find_epoch_slots_frozen_dead_slots( - &repair_info.cluster_slots, - dead_slot_pool, - repairable_dead_slot_pool, - &root_bank, - ); - - dead_slot_pool.retain(|slot| *slot > root_bank.slot()); - - repairable_dead_slot_pool.retain(|slot| *slot > root_bank.slot()); - - ancestor_hashes_request_statuses.retain(|slot, status| { - if *slot <= root_bank.slot() { - false - } else if status.is_expired() { - // Add the slot back to the repairable pool to retry - repairable_dead_slot_pool.insert(*slot); - false - } else { - true - } - }); - - // Keep around the last second of requests in the throttler. - request_throttle.retain(|request_time| *request_time > (timestamp() - 1000)); - - let number_of_allowed_requests = - MAX_ANCESTOR_HASHES_SLOT_REQUESTS_PER_SECOND.saturating_sub(request_throttle.len()); - - // Find dead slots for which it's worthwhile to ask the network for their - // ancestors - for _ in 0..number_of_allowed_requests { - let slot = repairable_dead_slot_pool.iter().next().cloned(); - if let Some(slot) = slot { - warn!( - "Cluster froze slot: {}, but we marked it as dead. - Initiating protocol to sample cluster for dead slot ancestors.", - slot - ); - - if Self::initiate_ancestor_hashes_requests_for_duplicate_slot( - ancestor_hashes_request_statuses, - ancestor_hashes_request_socket, - &repair_info.cluster_slots, - serve_repair, - &repair_info.repair_validators, - slot, - repair_stats, - outstanding_requests, - ) { - request_throttle.push(timestamp()); - repairable_dead_slot_pool.take(&slot).unwrap(); - } - } else { - break; - } - } - - repair_stats.report(); - } - - /// Find if any dead slots in `dead_slot_pool` have been frozen by sufficient - /// number of nodes in the cluster to justify adding to the `repairable_dead_slot_pool`. - fn find_epoch_slots_frozen_dead_slots( - cluster_slots: &ClusterSlots, - dead_slot_pool: &mut HashSet, - repairable_dead_slot_pool: &mut HashSet, - root_bank: &Bank, - ) { - dead_slot_pool.retain(|dead_slot| { - let epoch = root_bank.get_epoch_and_slot_index(*dead_slot).0; - if let Some(epoch_stakes) = root_bank.epoch_stakes(epoch) { - let status = cluster_slots.lookup(*dead_slot); - if let Some(completed_dead_slot_pubkeys) = status { - let total_stake = epoch_stakes.total_stake(); - let node_id_to_vote_accounts = epoch_stakes.node_id_to_vote_accounts(); - let total_completed_slot_stake: u64 = completed_dead_slot_pubkeys - .read() - .unwrap() - .iter() - .map(|(node_key, _)| { - node_id_to_vote_accounts - .get(node_key) - .map(|v| v.total_stake) - .unwrap_or(0) - }) - .sum(); - // If sufficient number of validators froze this slot, then there's a chance - // this dead slot was duplicate confirmed and will make it into in the main fork. - // This means it's worth asking the cluster to get the correct version. - if total_completed_slot_stake as f64 / total_stake as f64 > DUPLICATE_THRESHOLD - { - repairable_dead_slot_pool.insert(*dead_slot); - false - } else { - true - } - } else { - true - } - } else { - warn!( - "Dead slot {} is too far ahead of root bank {}", - dead_slot, - root_bank.slot() - ); - false - } - }) - } - - /// Returns true if a request was successfully made and the status - /// added to `ancestor_hashes_request_statuses` - fn initiate_ancestor_hashes_requests_for_duplicate_slot( - ancestor_hashes_request_statuses: &DashMap, - ancestor_hashes_request_socket: &UdpSocket, - cluster_slots: &ClusterSlots, - serve_repair: &ServeRepair, - repair_validators: &Option>, - duplicate_slot: Slot, - repair_stats: &mut AncestorRepairRequestsStats, - outstanding_requests: &RwLock, - ) -> bool { - let sampled_validators = serve_repair.repair_request_ancestor_hashes_sample_peers( - duplicate_slot, - cluster_slots, - repair_validators, - ); - - if let Ok(sampled_validators) = sampled_validators { - for (pubkey, socket_addr) in sampled_validators.iter() { - repair_stats - .ancestor_requests - .update(pubkey, duplicate_slot, 0); - let nonce = outstanding_requests - .write() - .unwrap() - .add_request(AncestorHashesRepairType(duplicate_slot), timestamp()); - let request_bytes = - serve_repair.ancestor_repair_request_bytes(duplicate_slot, nonce); - if let Ok(request_bytes) = request_bytes { - let _ = ancestor_hashes_request_socket.send_to(&request_bytes, socket_addr); - } - } - - let ancestor_request_status = DeadSlotAncestorRequestStatus::new( - sampled_validators - .into_iter() - .map(|(_pk, socket_addr)| socket_addr), - duplicate_slot, - ); - assert!(!ancestor_hashes_request_statuses.contains_key(&duplicate_slot)); - ancestor_hashes_request_statuses.insert(duplicate_slot, ancestor_request_status); - true - } else { - false - } - } -} - -#[cfg(test)] -mod test { - use { - super::*, - crate::{ - cluster_slot_state_verifier::DuplicateSlotsToRepair, - repair_service::DuplicateSlotsResetReceiver, - replay_stage::{ - tests::{replay_blockstore_components, ReplayBlockstoreComponents}, - ReplayStage, - }, - serve_repair::MAX_ANCESTOR_RESPONSES, - vote_simulator::VoteSimulator, - }, - solana_gossip::{ - cluster_info::{ClusterInfo, Node}, - contact_info::ContactInfo, - }, - solana_ledger::{blockstore::make_many_slot_entries, get_tmp_ledger_path}, - solana_runtime::{accounts_background_service::AbsRequestSender, bank_forks::BankForks}, - solana_sdk::{hash::Hash, signature::Keypair}, - solana_streamer::socket::SocketAddrSpace, - std::{collections::HashMap, sync::mpsc::channel}, - trees::tr, - }; - - #[test] - pub fn test_ancestor_hashes_service_process_replay_updates() { - let (ancestor_hashes_replay_update_sender, ancestor_hashes_replay_update_receiver) = - unbounded(); - let ancestor_hashes_request_statuses = DashMap::new(); - let mut dead_slot_pool = HashSet::new(); - let mut repairable_dead_slot_pool = HashSet::new(); - let slot = 10; - let mut root_slot = 0; - - // 1) Getting a dead signal should only add the slot to the dead pool - ancestor_hashes_replay_update_sender - .send(AncestorHashesReplayUpdate::Dead(slot)) - .unwrap(); - AncestorHashesService::process_replay_updates( - &ancestor_hashes_replay_update_receiver, - &ancestor_hashes_request_statuses, - &mut dead_slot_pool, - &mut repairable_dead_slot_pool, - root_slot, - ); - assert!(dead_slot_pool.contains(&slot)); - assert!(repairable_dead_slot_pool.is_empty()); - - // 2) Getting a duplicate confirmed dead slot should move the slot - // from the dead pool to the repairable pool - ancestor_hashes_replay_update_sender - .send(AncestorHashesReplayUpdate::DeadDuplicateConfirmed(slot)) - .unwrap(); - AncestorHashesService::process_replay_updates( - &ancestor_hashes_replay_update_receiver, - &ancestor_hashes_request_statuses, - &mut dead_slot_pool, - &mut repairable_dead_slot_pool, - root_slot, - ); - assert!(dead_slot_pool.is_empty()); - assert!(repairable_dead_slot_pool.contains(&slot)); - - // 3) Getting another dead signal should not add it back to the dead pool - ancestor_hashes_replay_update_sender - .send(AncestorHashesReplayUpdate::Dead(slot)) - .unwrap(); - AncestorHashesService::process_replay_updates( - &ancestor_hashes_replay_update_receiver, - &ancestor_hashes_request_statuses, - &mut dead_slot_pool, - &mut repairable_dead_slot_pool, - root_slot, - ); - assert!(dead_slot_pool.is_empty()); - assert!(repairable_dead_slot_pool.contains(&slot)); - - // 4) If an outstanding request for a slot already exists, should - // ignore any signals from replay stage - ancestor_hashes_request_statuses.insert(slot, DeadSlotAncestorRequestStatus::default()); - dead_slot_pool.clear(); - repairable_dead_slot_pool.clear(); - ancestor_hashes_replay_update_sender - .send(AncestorHashesReplayUpdate::Dead(slot)) - .unwrap(); - ancestor_hashes_replay_update_sender - .send(AncestorHashesReplayUpdate::DeadDuplicateConfirmed(slot)) - .unwrap(); - AncestorHashesService::process_replay_updates( - &ancestor_hashes_replay_update_receiver, - &ancestor_hashes_request_statuses, - &mut dead_slot_pool, - &mut repairable_dead_slot_pool, - root_slot, - ); - assert!(dead_slot_pool.is_empty()); - assert!(repairable_dead_slot_pool.is_empty()); - - // 5) If we get any signals for slots <= root_slot, they should be ignored - root_slot = 15; - ancestor_hashes_request_statuses.clear(); - dead_slot_pool.clear(); - repairable_dead_slot_pool.clear(); - ancestor_hashes_replay_update_sender - .send(AncestorHashesReplayUpdate::Dead(root_slot - 1)) - .unwrap(); - ancestor_hashes_replay_update_sender - .send(AncestorHashesReplayUpdate::DeadDuplicateConfirmed( - root_slot - 2, - )) - .unwrap(); - AncestorHashesService::process_replay_updates( - &ancestor_hashes_replay_update_receiver, - &ancestor_hashes_request_statuses, - &mut dead_slot_pool, - &mut repairable_dead_slot_pool, - root_slot, - ); - assert!(dead_slot_pool.is_empty()); - assert!(repairable_dead_slot_pool.is_empty()); - } - - #[test] - fn test_ancestor_hashes_service_find_epoch_slots_frozen_dead_slots() { - let vote_simulator = VoteSimulator::new(3); - let cluster_slots = ClusterSlots::default(); - let mut dead_slot_pool = HashSet::new(); - let mut repairable_dead_slot_pool = HashSet::new(); - let root_bank = vote_simulator.bank_forks.read().unwrap().root_bank(); - let dead_slot = 10; - dead_slot_pool.insert(dead_slot); - - // ClusterSlots doesn't have an entry for this slot yet, shouldn't move the slot - // from the dead slot pool. - AncestorHashesService::find_epoch_slots_frozen_dead_slots( - &cluster_slots, - &mut dead_slot_pool, - &mut repairable_dead_slot_pool, - &root_bank, - ); - assert_eq!(dead_slot_pool.len(), 1); - assert!(dead_slot_pool.contains(&dead_slot)); - assert!(repairable_dead_slot_pool.is_empty()); - - let max_epoch = root_bank.epoch_stakes_map().keys().max().unwrap(); - let slot_outside_known_epochs = root_bank - .epoch_schedule() - .get_last_slot_in_epoch(*max_epoch) - + 1; - dead_slot_pool.insert(slot_outside_known_epochs); - - // Should remove `slot_outside_known_epochs` - AncestorHashesService::find_epoch_slots_frozen_dead_slots( - &cluster_slots, - &mut dead_slot_pool, - &mut repairable_dead_slot_pool, - &root_bank, - ); - assert_eq!(dead_slot_pool.len(), 1); - assert!(dead_slot_pool.contains(&dead_slot)); - assert!(repairable_dead_slot_pool.is_empty()); - - // Slot hasn't reached the threshold - for (i, key) in (0..2).zip(vote_simulator.node_pubkeys.iter()) { - cluster_slots.insert_node_id(dead_slot, *key); - AncestorHashesService::find_epoch_slots_frozen_dead_slots( - &cluster_slots, - &mut dead_slot_pool, - &mut repairable_dead_slot_pool, - &root_bank, - ); - if i == 0 { - assert_eq!(dead_slot_pool.len(), 1); - assert!(dead_slot_pool.contains(&dead_slot)); - assert!(repairable_dead_slot_pool.is_empty()); - } else { - assert!(dead_slot_pool.is_empty()); - assert_eq!(repairable_dead_slot_pool.len(), 1); - assert!(repairable_dead_slot_pool.contains(&dead_slot)); - } - } - } - - struct ResponderThreads { - t_request_receiver: JoinHandle<()>, - t_listen: JoinHandle<()>, - exit: Arc, - responder_info: ContactInfo, - response_receiver: PacketBatchReceiver, - correct_bank_hashes: HashMap, - } - - impl ResponderThreads { - fn shutdown(self) { - self.exit.store(true, Ordering::Relaxed); - self.t_request_receiver.join().unwrap(); - self.t_listen.join().unwrap(); - } - - fn new(slot_to_query: Slot) -> Self { - assert!(slot_to_query >= MAX_ANCESTOR_RESPONSES as Slot); - let responder_node = Node::new_localhost(); - let cluster_info = ClusterInfo::new( - responder_node.info.clone(), - Arc::new(Keypair::new()), - SocketAddrSpace::Unspecified, - ); - let responder_serve_repair = - Arc::new(RwLock::new(ServeRepair::new(Arc::new(cluster_info)))); - - // Set up thread to give us responses - let ledger_path = get_tmp_ledger_path!(); - let exit = Arc::new(AtomicBool::new(false)); - let (requests_sender, requests_receiver) = channel(); - let (response_sender, response_receiver) = channel(); - - // Set up blockstore for responses - let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap()); - // Create slots [slot, slot + MAX_ANCESTOR_RESPONSES) with 5 shreds apiece - let (shreds, _) = - make_many_slot_entries(slot_to_query, MAX_ANCESTOR_RESPONSES as u64, 5); - blockstore - .insert_shreds(shreds, None, false) - .expect("Expect successful ledger write"); - let mut correct_bank_hashes = HashMap::new(); - for duplicate_confirmed_slot in - slot_to_query - MAX_ANCESTOR_RESPONSES as Slot + 1..=slot_to_query - { - let hash = Hash::new_unique(); - correct_bank_hashes.insert(duplicate_confirmed_slot, hash); - blockstore.insert_bank_hash(duplicate_confirmed_slot, hash, true); - } - - // Set up response threads - let t_request_receiver = streamer::receiver( - Arc::new(responder_node.sockets.serve_repair), - &exit, - requests_sender, - Recycler::default(), - "serve_repair_receiver", - 1, - false, - ); - let t_listen = ServeRepair::listen( - responder_serve_repair, - Some(blockstore), - requests_receiver, - response_sender, - &exit, - ); - - Self { - t_request_receiver, - t_listen, - exit, - responder_info: responder_node.info, - response_receiver, - correct_bank_hashes, - } - } - } - - struct ManageAncestorHashesState { - ancestor_hashes_request_statuses: Arc>, - ancestor_hashes_request_socket: Arc, - requester_serve_repair: ServeRepair, - repair_info: RepairInfo, - outstanding_requests: Arc>, - dead_slot_pool: HashSet, - repairable_dead_slot_pool: HashSet, - request_throttle: Vec, - repair_stats: AncestorRepairRequestsStats, - _duplicate_slots_reset_receiver: DuplicateSlotsResetReceiver, - retryable_slots_sender: RetryableSlotsSender, - retryable_slots_receiver: RetryableSlotsReceiver, - ancestor_hashes_replay_update_sender: AncestorHashesReplayUpdateSender, - ancestor_hashes_replay_update_receiver: AncestorHashesReplayUpdateReceiver, - } - - impl ManageAncestorHashesState { - fn new(bank_forks: Arc>) -> Self { - let ancestor_hashes_request_statuses = Arc::new(DashMap::new()); - let ancestor_hashes_request_socket = Arc::new(UdpSocket::bind("0.0.0.0:0").unwrap()); - let epoch_schedule = *bank_forks.read().unwrap().root_bank().epoch_schedule(); - let requester_cluster_info = Arc::new(ClusterInfo::new( - Node::new_localhost().info, - Arc::new(Keypair::new()), - SocketAddrSpace::Unspecified, - )); - let requester_serve_repair = ServeRepair::new(requester_cluster_info.clone()); - let (duplicate_slots_reset_sender, _duplicate_slots_reset_receiver) = unbounded(); - let repair_info = RepairInfo { - bank_forks, - cluster_info: requester_cluster_info, - cluster_slots: Arc::new(ClusterSlots::default()), - epoch_schedule, - duplicate_slots_reset_sender, - repair_validators: None, - }; - - let (ancestor_hashes_replay_update_sender, ancestor_hashes_replay_update_receiver) = - unbounded(); - let (retryable_slots_sender, retryable_slots_receiver) = unbounded(); - Self { - ancestor_hashes_request_statuses, - ancestor_hashes_request_socket, - requester_serve_repair, - repair_info, - outstanding_requests: Arc::new(RwLock::new( - OutstandingAncestorHashesRepairs::default(), - )), - dead_slot_pool: HashSet::new(), - repairable_dead_slot_pool: HashSet::new(), - request_throttle: vec![], - repair_stats: AncestorRepairRequestsStats::default(), - _duplicate_slots_reset_receiver, - ancestor_hashes_replay_update_sender, - ancestor_hashes_replay_update_receiver, - retryable_slots_sender, - retryable_slots_receiver, - } - } - } - - fn setup_dead_slot( - dead_slot: Slot, - correct_bank_hashes: &HashMap, - ) -> ReplayBlockstoreComponents { - assert!(dead_slot >= MAX_ANCESTOR_RESPONSES as Slot); - let mut forks = tr(0); - - // Create a bank_forks that includes everything but the dead slot - for slot in 1..dead_slot { - forks.push_front(tr(slot)); - } - let mut replay_blockstore_components = replay_blockstore_components(Some(forks), 1, None); - let ReplayBlockstoreComponents { - ref blockstore, - ref mut vote_simulator, - .. - } = replay_blockstore_components; - - // Create dead slot in bank_forks - let is_frozen = false; - vote_simulator.fill_bank_forks( - tr(dead_slot - 1) / tr(dead_slot), - &HashMap::new(), - is_frozen, - ); - - /*{ - let w_bank_forks = bank_forks.write().unwrap(); - assert!(w_bank_forks.get(dead_slot).is_none()); - let parent = w_bank_forks.get(dead_slot - 1).unwrap().clone(); - let dead_bank = Bank::new_from_parent(&parent, &Pubkey::default(), dead_slot); - bank_forks.insert(dead_bank); - - }*/ - - // Create slots [slot, slot + num_ancestors) with 5 shreds apiece - let (shreds, _) = make_many_slot_entries(dead_slot, dead_slot, 5); - blockstore - .insert_shreds(shreds, None, false) - .expect("Expect successful ledger write"); - for duplicate_confirmed_slot in 0..dead_slot { - let bank_hash = correct_bank_hashes - .get(&duplicate_confirmed_slot) - .cloned() - .unwrap_or_else(Hash::new_unique); - blockstore.insert_bank_hash(duplicate_confirmed_slot, bank_hash, true); - } - blockstore.set_dead_slot(dead_slot).unwrap(); - replay_blockstore_components - } - - #[test] - fn test_ancestor_hashes_service_initiate_ancestor_hashes_requests_for_duplicate_slot() { - let dead_slot = MAX_ANCESTOR_RESPONSES as Slot; - let responder_threads = ResponderThreads::new(dead_slot); - - let ResponderThreads { - ref responder_info, - ref response_receiver, - ref correct_bank_hashes, - .. - } = responder_threads; - - let ReplayBlockstoreComponents { - blockstore: requester_blockstore, - vote_simulator, - .. - } = setup_dead_slot(dead_slot, correct_bank_hashes); - - let ManageAncestorHashesState { - ancestor_hashes_request_statuses, - ancestor_hashes_request_socket, - repair_info, - outstanding_requests, - requester_serve_repair, - mut repair_stats, - .. - } = ManageAncestorHashesState::new(vote_simulator.bank_forks); - - let RepairInfo { - cluster_info: requester_cluster_info, - cluster_slots, - repair_validators, - .. - } = repair_info; - - AncestorHashesService::initiate_ancestor_hashes_requests_for_duplicate_slot( - &ancestor_hashes_request_statuses, - &ancestor_hashes_request_socket, - &cluster_slots, - &requester_serve_repair, - &repair_validators, - dead_slot, - &mut repair_stats, - &outstanding_requests, - ); - assert!(ancestor_hashes_request_statuses.is_empty()); - - // Add the responder to the eligible list for requests - let responder_id = responder_info.id; - cluster_slots.insert_node_id(dead_slot, responder_id); - requester_cluster_info.insert_info(responder_info.clone()); - // Now the request should actually be made - AncestorHashesService::initiate_ancestor_hashes_requests_for_duplicate_slot( - &ancestor_hashes_request_statuses, - &ancestor_hashes_request_socket, - &cluster_slots, - &requester_serve_repair, - &repair_validators, - dead_slot, - &mut repair_stats, - &outstanding_requests, - ); - - assert_eq!(ancestor_hashes_request_statuses.len(), 1); - assert!(ancestor_hashes_request_statuses.contains_key(&dead_slot)); - - // Should have received valid response - let mut response_packet = response_receiver - .recv_timeout(Duration::from_millis(10_000)) - .unwrap(); - let packet = &mut response_packet.packets[0]; - packet.meta.set_addr(&responder_info.serve_repair); - let decision = AncestorHashesService::verify_and_process_ancestor_response( - packet, - &ancestor_hashes_request_statuses, - &mut AncestorHashesResponsesStats::default(), - &outstanding_requests, - &requester_blockstore, - ) - .unwrap(); - - assert_matches!( - decision, - ( - _dead_slot, - DuplicateAncestorDecision::EarliestAncestorNotFrozen(_) - ) - ); - assert_eq!( - decision - .1 - .repair_status() - .unwrap() - .correct_ancestors_to_repair, - vec![(dead_slot, *correct_bank_hashes.get(&dead_slot).unwrap())] - ); - - // Should have removed the ancestor status on successful - // completion - assert!(ancestor_hashes_request_statuses.is_empty()); - responder_threads.shutdown(); - } - - #[test] - fn test_ancestor_hashes_service_manage_ancestor_requests() { - let vote_simulator = VoteSimulator::new(3); - let ManageAncestorHashesState { - ancestor_hashes_request_statuses, - ancestor_hashes_request_socket, - requester_serve_repair, - repair_info, - outstanding_requests, - mut dead_slot_pool, - mut repairable_dead_slot_pool, - mut request_throttle, - ancestor_hashes_replay_update_sender, - ancestor_hashes_replay_update_receiver, - retryable_slots_receiver, - .. - } = ManageAncestorHashesState::new(vote_simulator.bank_forks); - let responder_node = Node::new_localhost(); - let RepairInfo { - ref bank_forks, - ref cluster_info, - .. - } = repair_info; - cluster_info.insert_info(responder_node.info); - bank_forks.read().unwrap().root_bank().epoch_schedule(); - // 1) No signals from ReplayStage, no requests should be made - AncestorHashesService::manage_ancestor_requests( - &ancestor_hashes_request_statuses, - &ancestor_hashes_request_socket, - &repair_info, - &outstanding_requests, - &ancestor_hashes_replay_update_receiver, - &retryable_slots_receiver, - &requester_serve_repair, - &mut AncestorRepairRequestsStats::default(), - &mut dead_slot_pool, - &mut repairable_dead_slot_pool, - &mut request_throttle, - ); - - assert!(dead_slot_pool.is_empty()); - assert!(repairable_dead_slot_pool.is_empty()); - assert!(ancestor_hashes_request_statuses.is_empty()); - - // 2) Simulate signals from ReplayStage, should make a request - // for `dead_duplicate_confirmed_slot` - let dead_slot = 10; - let dead_duplicate_confirmed_slot = 14; - ancestor_hashes_replay_update_sender - .send(AncestorHashesReplayUpdate::Dead(dead_slot)) - .unwrap(); - ancestor_hashes_replay_update_sender - .send(AncestorHashesReplayUpdate::DeadDuplicateConfirmed( - dead_duplicate_confirmed_slot, - )) - .unwrap(); - ancestor_hashes_replay_update_sender - .send(AncestorHashesReplayUpdate::DeadDuplicateConfirmed( - dead_duplicate_confirmed_slot, - )) - .unwrap(); - AncestorHashesService::manage_ancestor_requests( - &ancestor_hashes_request_statuses, - &ancestor_hashes_request_socket, - &repair_info, - &outstanding_requests, - &ancestor_hashes_replay_update_receiver, - &retryable_slots_receiver, - &requester_serve_repair, - &mut AncestorRepairRequestsStats::default(), - &mut dead_slot_pool, - &mut repairable_dead_slot_pool, - &mut request_throttle, - ); - - assert_eq!(dead_slot_pool.len(), 1); - assert!(dead_slot_pool.contains(&dead_slot)); - assert!(repairable_dead_slot_pool.is_empty()); - assert_eq!(ancestor_hashes_request_statuses.len(), 1); - assert!(ancestor_hashes_request_statuses.contains_key(&dead_duplicate_confirmed_slot)); - - // 3) Simulate an outstanding request timing out - ancestor_hashes_request_statuses - .get_mut(&dead_duplicate_confirmed_slot) - .unwrap() - .value_mut() - .make_expired(); - - // If the request timed out, we should remove the slot from `ancestor_hashes_request_statuses`, - // and add it to `repairable_dead_slot_pool`. Because the request_throttle is at its limit, - // we should not immediately retry the timed request. - request_throttle.resize(MAX_ANCESTOR_HASHES_SLOT_REQUESTS_PER_SECOND, std::u64::MAX); - AncestorHashesService::manage_ancestor_requests( - &ancestor_hashes_request_statuses, - &ancestor_hashes_request_socket, - &repair_info, - &outstanding_requests, - &ancestor_hashes_replay_update_receiver, - &retryable_slots_receiver, - &requester_serve_repair, - &mut AncestorRepairRequestsStats::default(), - &mut dead_slot_pool, - &mut repairable_dead_slot_pool, - &mut request_throttle, - ); - - assert_eq!(dead_slot_pool.len(), 1); - assert!(dead_slot_pool.contains(&dead_slot)); - assert_eq!(repairable_dead_slot_pool.len(), 1); - assert!(repairable_dead_slot_pool.contains(&dead_duplicate_confirmed_slot)); - assert!(ancestor_hashes_request_statuses.is_empty()); - - // 4) If the throttle only has expired timestamps from more than a second ago, - // then on the next iteration, we should clear the entries in the throttle - // and retry a request for the timed out request - request_throttle.clear(); - request_throttle.resize( - MAX_ANCESTOR_HASHES_SLOT_REQUESTS_PER_SECOND, - timestamp() - 1001, - ); - AncestorHashesService::manage_ancestor_requests( - &ancestor_hashes_request_statuses, - &ancestor_hashes_request_socket, - &repair_info, - &outstanding_requests, - &ancestor_hashes_replay_update_receiver, - &retryable_slots_receiver, - &requester_serve_repair, - &mut AncestorRepairRequestsStats::default(), - &mut dead_slot_pool, - &mut repairable_dead_slot_pool, - &mut request_throttle, - ); - assert_eq!(dead_slot_pool.len(), 1); - assert!(dead_slot_pool.contains(&dead_slot)); - assert!(repairable_dead_slot_pool.is_empty()); - assert_eq!(ancestor_hashes_request_statuses.len(), 1); - assert!(ancestor_hashes_request_statuses.contains_key(&dead_duplicate_confirmed_slot)); - // Request throttle includes one item for the request we just made - assert_eq!( - request_throttle.len(), - ancestor_hashes_request_statuses.len() - ); - - // 5) If we've reached the throttle limit, no requests should be made, - // but should still read off the channel for replay updates - request_throttle.clear(); - request_throttle.resize(MAX_ANCESTOR_HASHES_SLOT_REQUESTS_PER_SECOND, std::u64::MAX); - let dead_duplicate_confirmed_slot_2 = 15; - ancestor_hashes_replay_update_sender - .send(AncestorHashesReplayUpdate::DeadDuplicateConfirmed( - dead_duplicate_confirmed_slot_2, - )) - .unwrap(); - AncestorHashesService::manage_ancestor_requests( - &ancestor_hashes_request_statuses, - &ancestor_hashes_request_socket, - &repair_info, - &outstanding_requests, - &ancestor_hashes_replay_update_receiver, - &retryable_slots_receiver, - &requester_serve_repair, - &mut AncestorRepairRequestsStats::default(), - &mut dead_slot_pool, - &mut repairable_dead_slot_pool, - &mut request_throttle, - ); - - assert_eq!(dead_slot_pool.len(), 1); - assert!(dead_slot_pool.contains(&dead_slot)); - assert_eq!(repairable_dead_slot_pool.len(), 1); - assert!(repairable_dead_slot_pool.contains(&dead_duplicate_confirmed_slot_2)); - assert_eq!(ancestor_hashes_request_statuses.len(), 1); - assert!(ancestor_hashes_request_statuses.contains_key(&dead_duplicate_confirmed_slot)); - - // 6) If root moves past slot, should remove it from all state - let bank_forks = &repair_info.bank_forks; - let root_bank = bank_forks.read().unwrap().root_bank(); - let new_root_slot = dead_duplicate_confirmed_slot_2 + 1; - let new_root_bank = Bank::new_from_parent(&root_bank, &Pubkey::default(), new_root_slot); - new_root_bank.freeze(); - { - let mut w_bank_forks = bank_forks.write().unwrap(); - w_bank_forks.insert(new_root_bank); - w_bank_forks.set_root(new_root_slot, &AbsRequestSender::default(), None); - } - assert!(!dead_slot_pool.is_empty()); - assert!(!repairable_dead_slot_pool.is_empty()); - assert!(!ancestor_hashes_request_statuses.is_empty()); - request_throttle.clear(); - AncestorHashesService::manage_ancestor_requests( - &ancestor_hashes_request_statuses, - &ancestor_hashes_request_socket, - &repair_info, - &outstanding_requests, - &ancestor_hashes_replay_update_receiver, - &retryable_slots_receiver, - &requester_serve_repair, - &mut AncestorRepairRequestsStats::default(), - &mut dead_slot_pool, - &mut repairable_dead_slot_pool, - &mut request_throttle, - ); - assert!(dead_slot_pool.is_empty()); - assert!(repairable_dead_slot_pool.is_empty()); - assert!(ancestor_hashes_request_statuses.is_empty()); - } - - #[test] - fn test_ancestor_hashes_service_manage_ancestor_hashes_after_replay_dump() { - let dead_slot = MAX_ANCESTOR_RESPONSES as Slot; - let responder_threads = ResponderThreads::new(dead_slot); - - let ResponderThreads { - ref responder_info, - ref response_receiver, - ref correct_bank_hashes, - .. - } = responder_threads; - - let ReplayBlockstoreComponents { - blockstore: requester_blockstore, - vote_simulator, - .. - } = setup_dead_slot(dead_slot, correct_bank_hashes); - - let VoteSimulator { - bank_forks, - mut progress, - .. - } = vote_simulator; - - let ManageAncestorHashesState { - ancestor_hashes_request_statuses, - ancestor_hashes_request_socket, - requester_serve_repair, - repair_info, - outstanding_requests, - mut dead_slot_pool, - mut repairable_dead_slot_pool, - mut request_throttle, - ancestor_hashes_replay_update_sender, - ancestor_hashes_replay_update_receiver, - retryable_slots_receiver, - .. - } = ManageAncestorHashesState::new(bank_forks.clone()); - - let RepairInfo { - cluster_info: ref requester_cluster_info, - ref cluster_slots, - .. - } = repair_info; - - // Add the responder to the eligible list for requests - let responder_id = responder_info.id; - cluster_slots.insert_node_id(dead_slot, responder_id); - requester_cluster_info.insert_info(responder_info.clone()); - - // Simulate getting duplicate confirmed dead slot - ancestor_hashes_replay_update_sender - .send(AncestorHashesReplayUpdate::DeadDuplicateConfirmed( - dead_slot, - )) - .unwrap(); - - // Simulate Replay dumping this slot - let mut duplicate_slots_to_repair = DuplicateSlotsToRepair::default(); - duplicate_slots_to_repair.insert(dead_slot, Hash::new_unique()); - ReplayStage::dump_then_repair_correct_slots( - &mut duplicate_slots_to_repair, - &mut bank_forks.read().unwrap().ancestors(), - &mut bank_forks.read().unwrap().descendants().clone(), - &mut progress, - &bank_forks, - &requester_blockstore, - None, - ); - - // Simulate making a request - AncestorHashesService::manage_ancestor_requests( - &ancestor_hashes_request_statuses, - &ancestor_hashes_request_socket, - &repair_info, - &outstanding_requests, - &ancestor_hashes_replay_update_receiver, - &retryable_slots_receiver, - &requester_serve_repair, - &mut AncestorRepairRequestsStats::default(), - &mut dead_slot_pool, - &mut repairable_dead_slot_pool, - &mut request_throttle, - ); - - assert_eq!(ancestor_hashes_request_statuses.len(), 1); - assert!(ancestor_hashes_request_statuses.contains_key(&dead_slot)); - - // Should have received valid response - let mut response_packet = response_receiver - .recv_timeout(Duration::from_millis(10_000)) - .unwrap(); - let packet = &mut response_packet.packets[0]; - packet.meta.set_addr(&responder_info.serve_repair); - let decision = AncestorHashesService::verify_and_process_ancestor_response( - packet, - &ancestor_hashes_request_statuses, - &mut AncestorHashesResponsesStats::default(), - &outstanding_requests, - &requester_blockstore, - ) - .unwrap(); - - assert_matches!( - decision, - ( - _dead_slot, - DuplicateAncestorDecision::EarliestAncestorNotFrozen(_) - ) - ); - assert_eq!( - decision - .1 - .repair_status() - .unwrap() - .correct_ancestors_to_repair, - vec![(dead_slot, *correct_bank_hashes.get(&dead_slot).unwrap())] - ); - - // Should have removed the ancestor status on successful - // completion - assert!(ancestor_hashes_request_statuses.is_empty()); - responder_threads.shutdown(); - } - - #[test] - fn test_ancestor_hashes_service_retryable_duplicate_ancestor_decision() { - let vote_simulator = VoteSimulator::new(1); - let ManageAncestorHashesState { - ancestor_hashes_request_statuses, - ancestor_hashes_request_socket, - requester_serve_repair, - repair_info, - outstanding_requests, - mut dead_slot_pool, - mut repairable_dead_slot_pool, - mut request_throttle, - ancestor_hashes_replay_update_receiver, - retryable_slots_receiver, - retryable_slots_sender, - .. - } = ManageAncestorHashesState::new(vote_simulator.bank_forks); - - let decision = DuplicateAncestorDecision::SampleNotDuplicateConfirmed; - assert!(decision.is_retryable()); - - // Simulate network response processing thread reaching a retryable - // decision - let request_slot = 10; - AncestorHashesService::handle_ancestor_request_decision( - request_slot, - decision, - &repair_info.duplicate_slots_reset_sender, - &retryable_slots_sender, - ); - - // Simulate ancestor request thread getting the retry signal - assert!(dead_slot_pool.is_empty()); - assert!(repairable_dead_slot_pool.is_empty()); - AncestorHashesService::manage_ancestor_requests( - &ancestor_hashes_request_statuses, - &ancestor_hashes_request_socket, - &repair_info, - &outstanding_requests, - &ancestor_hashes_replay_update_receiver, - &retryable_slots_receiver, - &requester_serve_repair, - &mut AncestorRepairRequestsStats::default(), - &mut dead_slot_pool, - &mut repairable_dead_slot_pool, - &mut request_throttle, - ); - - assert!(dead_slot_pool.is_empty()); - assert!(repairable_dead_slot_pool.contains(&request_slot)); - } -} diff --git a/core/src/banking_stage.rs b/core/src/banking_stage.rs index 32e4381e651..ceb7c1ffb76 100644 --- a/core/src/banking_stage.rs +++ b/core/src/banking_stage.rs @@ -690,14 +690,9 @@ impl BankingStage { } #[allow(clippy::too_many_arguments)] -<<<<<<< HEAD pub fn process_loop( my_pubkey: Pubkey, - verified_receiver: &CrossbeamReceiver>, -======= - fn process_loop( verified_receiver: &CrossbeamReceiver>, ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) poh_recorder: &Arc>, cluster_info: &ClusterInfo, recv_start: &mut Instant, @@ -715,12 +710,7 @@ impl BankingStage { let mut buffered_packet_batches = VecDeque::with_capacity(batch_limit); let banking_stage_stats = BankingStageStats::new(id); loop { -<<<<<<< HEAD - while !buffered_packets.is_empty() { -======= - let my_pubkey = cluster_info.id(); while !buffered_packet_batches.is_empty() { ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) let decision = Self::process_buffered_packets( &my_pubkey, &socket, @@ -1140,7 +1130,7 @@ impl BankingStage { .filter_map(|(tx, tx_index)| { // put transaction into retry queue if it wouldn't fit // into current bank - let is_vote = &msgs.packets[tx_index].meta.is_simple_vote_tx; + let is_vote = &packet_batch.packets[tx_index].meta.is_simple_vote_tx; // excluding vote TX from cost_model, for now if !is_vote @@ -1168,7 +1158,7 @@ impl BankingStage { filtered_transactions_with_packet_indexes .into_iter() .filter_map(|(tx, tx_index)| { - let p = &msgs.packets[tx_index]; + let p = &packet_batch.packets[tx_index]; let message_bytes = Self::packet_message(p)?; let message_hash = Message::hash_raw_message(message_bytes); Some(( @@ -1245,10 +1235,9 @@ impl BankingStage { cost_model: &Arc>, ) -> (usize, usize, Vec) { let mut packet_conversion_time = Measure::start("packet_conversion"); -<<<<<<< HEAD let (transactions, transaction_to_packet_indexes, retryable_packet_indexes) = Self::transactions_from_packets( - msgs, + packet_batch, &packet_indexes, &bank.feature_set, &bank.read_cost_tracker().unwrap(), @@ -1257,14 +1246,6 @@ impl BankingStage { bank.vote_only_bank(), cost_model, ); -======= - let (transactions, transaction_to_packet_indexes) = Self::transactions_from_packets( - packet_batch, - &packet_indexes, - &bank.feature_set, - bank.vote_only_bank(), - ); ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) packet_conversion_time.stop(); inc_new_counter_info!("banking_stage-packet_conversion", 1); @@ -1365,10 +1346,9 @@ impl BankingStage { let mut unprocessed_packet_conversion_time = Measure::start("unprocessed_packet_conversion"); -<<<<<<< HEAD let (transactions, transaction_to_packet_indexes, retry_packet_indexes) = Self::transactions_from_packets( - msgs, + packet_batch, &transaction_indexes, &bank.feature_set, &bank.read_cost_tracker().unwrap(), @@ -1377,14 +1357,6 @@ impl BankingStage { bank.vote_only_bank(), cost_model, ); -======= - let (transactions, transaction_to_packet_indexes) = Self::transactions_from_packets( - packet_batch, - transaction_indexes, - &bank.feature_set, - bank.vote_only_bank(), - ); ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) unprocessed_packet_conversion_time.stop(); let tx_count = transaction_to_packet_indexes.len(); @@ -1467,18 +1439,10 @@ impl BankingStage { let mut dropped_packets_count = 0; let mut dropped_packet_batches_count = 0; let mut newly_buffered_packets_count = 0; -<<<<<<< HEAD - while let Some(msgs) = mms_iter.next() { - let packet_indexes = Self::generate_packet_indexes(&msgs.packets); - let bank_start = poh.lock().unwrap().bank_start(); - if PohRecorder::get_bank_still_processing_txs(&bank_start).is_none() { -======= while let Some(packet_batch) = packet_batch_iter.next() { let packet_indexes = Self::generate_packet_indexes(&packet_batch.packets); - let poh_recorder_bank = poh.lock().unwrap().get_poh_recorder_bank(); - let working_bank_start = poh_recorder_bank.working_bank_start(); - if PohRecorder::get_working_bank_if_not_expired(&working_bank_start).is_none() { ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) + let bank_start = poh.lock().unwrap().bank_start(); + if PohRecorder::get_bank_still_processing_txs(&bank_start).is_none() { Self::push_unprocessed( buffered_packet_batches, packet_batch, @@ -1531,13 +1495,8 @@ impl BankingStage { while let Some(packet_batch) = packet_batch_iter.next() { let packet_indexes = Self::generate_packet_indexes(&packet_batch.packets); let unprocessed_indexes = Self::filter_unprocessed_packets( -<<<<<<< HEAD &bank, - &msgs, -======= - working_bank, &packet_batch, ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) &packet_indexes, my_pubkey, next_leader, @@ -3228,10 +3187,9 @@ mod tests { make_test_packets(vec![transfer_tx.clone(), transfer_tx.clone()], vote_indexes); let mut votes_only = false; -<<<<<<< HEAD let (txs, tx_packet_index, _retryable_packet_indexes) = BankingStage::transactions_from_packets( - &packets, + &packet_batch, &packet_indexes, &Arc::new(feature_set::FeatureSet::default()), &RwLock::new(CostTracker::default()).read().unwrap(), @@ -3240,22 +3198,13 @@ mod tests { votes_only, &Arc::new(RwLock::new(CostModel::default())), ); -======= - let (txs, tx_packet_index) = BankingStage::transactions_from_packets( - &packet_batch, - &packet_indexes, - &Arc::new(FeatureSet::default()), - votes_only, - ); ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) assert_eq!(2, txs.len()); assert_eq!(vec![0, 1], tx_packet_index); votes_only = true; -<<<<<<< HEAD let (txs, tx_packet_index, _retryable_packet_indexes) = BankingStage::transactions_from_packets( - &packets, + &packet_batch, &packet_indexes, &Arc::new(feature_set::FeatureSet::default()), &RwLock::new(CostTracker::default()).read().unwrap(), @@ -3264,14 +3213,6 @@ mod tests { votes_only, &Arc::new(RwLock::new(CostModel::default())), ); -======= - let (txs, tx_packet_index) = BankingStage::transactions_from_packets( - &packet_batch, - &packet_indexes, - &Arc::new(FeatureSet::default()), - votes_only, - ); ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) assert_eq!(0, txs.len()); assert_eq!(0, tx_packet_index.len()); } @@ -3285,10 +3226,9 @@ mod tests { ); let mut votes_only = false; -<<<<<<< HEAD let (txs, tx_packet_index, _retryable_packet_indexes) = BankingStage::transactions_from_packets( - &packets, + &packet_batch, &packet_indexes, &Arc::new(feature_set::FeatureSet::default()), &RwLock::new(CostTracker::default()).read().unwrap(), @@ -3297,22 +3237,13 @@ mod tests { votes_only, &Arc::new(RwLock::new(CostModel::default())), ); -======= - let (txs, tx_packet_index) = BankingStage::transactions_from_packets( - &packet_batch, - &packet_indexes, - &Arc::new(FeatureSet::default()), - votes_only, - ); ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) assert_eq!(3, txs.len()); assert_eq!(vec![0, 1, 2], tx_packet_index); votes_only = true; -<<<<<<< HEAD let (txs, tx_packet_index, _retryable_packet_indexes) = BankingStage::transactions_from_packets( - &packets, + &packet_batch, &packet_indexes, &Arc::new(feature_set::FeatureSet::default()), &RwLock::new(CostTracker::default()).read().unwrap(), @@ -3321,14 +3252,6 @@ mod tests { votes_only, &Arc::new(RwLock::new(CostModel::default())), ); -======= - let (txs, tx_packet_index) = BankingStage::transactions_from_packets( - &packet_batch, - &packet_indexes, - &Arc::new(FeatureSet::default()), - votes_only, - ); ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) assert_eq!(2, txs.len()); assert_eq!(vec![0, 2], tx_packet_index); } @@ -3342,10 +3265,9 @@ mod tests { ); let mut votes_only = false; -<<<<<<< HEAD let (txs, tx_packet_index, _retryable_packet_indexes) = BankingStage::transactions_from_packets( - &packets, + &packet_batch, &packet_indexes, &Arc::new(feature_set::FeatureSet::default()), &RwLock::new(CostTracker::default()).read().unwrap(), @@ -3354,22 +3276,13 @@ mod tests { votes_only, &Arc::new(RwLock::new(CostModel::default())), ); -======= - let (txs, tx_packet_index) = BankingStage::transactions_from_packets( - &packet_batch, - &packet_indexes, - &Arc::new(FeatureSet::default()), - votes_only, - ); ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) assert_eq!(3, txs.len()); assert_eq!(vec![0, 1, 2], tx_packet_index); votes_only = true; -<<<<<<< HEAD let (txs, tx_packet_index, _retryable_packet_indexes) = BankingStage::transactions_from_packets( - &packets, + &packet_batch, &packet_indexes, &Arc::new(feature_set::FeatureSet::default()), &RwLock::new(CostTracker::default()).read().unwrap(), @@ -3378,14 +3291,6 @@ mod tests { votes_only, &Arc::new(RwLock::new(CostModel::default())), ); -======= - let (txs, tx_packet_index) = BankingStage::transactions_from_packets( - &packet_batch, - &packet_indexes, - &Arc::new(FeatureSet::default()), - votes_only, - ); ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) assert_eq!(3, txs.len()); assert_eq!(vec![0, 1, 2], tx_packet_index); } diff --git a/core/src/cluster_info_vote_listener.rs b/core/src/cluster_info_vote_listener.rs index a68efa17717..d2ea764086d 100644 --- a/core/src/cluster_info_vote_listener.rs +++ b/core/src/cluster_info_vote_listener.rs @@ -56,8 +56,9 @@ use { // Map from a vote account to the authorized voter for an epoch pub type ThresholdConfirmedSlots = Vec<(Slot, Hash)>; pub type VotedHashUpdates = HashMap>; -pub type VerifiedLabelVotePacketsSender = CrossbeamSender>; -pub type VerifiedLabelVotePacketsReceiver = CrossbeamReceiver>; +pub type VerifiedLabelVotePacketsSender = CrossbeamSender>; +pub type VerifiedLabelVotePacketsReceiver = + CrossbeamReceiver>; pub type VerifiedVoteTransactionsSender = CrossbeamSender>; pub type VerifiedVoteTransactionsReceiver = CrossbeamReceiver>; pub type VerifiedVoteSender = CrossbeamSender<(Pubkey, Vec)>; @@ -349,64 +350,31 @@ impl ClusterInfoVoteListener { } #[allow(clippy::type_complexity)] -<<<<<<< HEAD fn verify_votes( votes: Vec, labels: Vec, - ) -> (Vec, Vec<(CrdsValueLabel, Slot, Packets)>) { - let mut msgs = packet::to_packets_chunked(&votes, 1); -======= - fn verify_votes(votes: Vec) -> (Vec, Vec) { + ) -> (Vec, Vec<(CrdsValueLabel, Slot, PacketBatch)>) { let mut packet_batches = packet::to_packet_batches(&votes, 1); ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) // Votes should already be filtered by this point. let reject_non_vote = false; sigverify::ed25519_verify_cpu(&mut packet_batches, reject_non_vote); -<<<<<<< HEAD - let (vote_txs, packets) = izip!(labels.into_iter(), votes.into_iter(), msgs,) - .filter_map(|(label, vote, packet)| { + let (vote_txs, packet_batch) = izip!(labels.into_iter(), votes.into_iter(), packet_batches) + .filter_map(|(label, vote, packet_batch)| { let slot = vote_transaction::parse_vote_transaction(&vote) .and_then(|(_, vote, _)| vote.slots.last().copied())?; - // to_packets_chunked() above split into 1 packet long chunks - assert_eq!(packet.packets.len(), 1); - if !packet.packets[0].meta.discard { - Some((vote, (label, slot, packet))) - } else { - None -======= - let (vote_txs, vote_metadata) = izip!(votes.into_iter(), packet_batches) - .filter_map(|(vote_tx, packet_batch)| { - let (vote, vote_account_key) = vote_transaction::parse_vote_transaction(&vote_tx) - .and_then(|(vote_account_key, vote, _)| { - if vote.slots().is_empty() { - None - } else { - Some((vote, vote_account_key)) - } - })?; - - // to_packet_batches() above splits into 1 packet long batches + // to_packet_batches() above split into 1 packet long chunks assert_eq!(packet_batch.packets.len(), 1); if !packet_batch.packets[0].meta.discard { - if let Some(signature) = vote_tx.signatures.first().cloned() { - return Some(( - vote_tx, - VerifiedVoteMetadata { - vote_account_key, - vote, - packet_batch, - signature, - }, - )); - } ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) + Some((vote, (label, slot, packet_batch))) + } else { + None } }) .unzip(); - (vote_txs, packets) + (vote_txs, packet_batch) } fn bank_send_loop( @@ -447,10 +415,11 @@ impl ClusterInfoVoteListener { let bank = poh_recorder.lock().unwrap().bank(); if let Some(bank) = bank { let last_version = bank.last_vote_sync.load(Ordering::Relaxed); - let (new_version, msgs) = verified_vote_packets.get_latest_votes(last_version); - inc_new_counter_info!("bank_send_loop_batch_size", msgs.packets.len()); + let (new_version, packet_batch) = + verified_vote_packets.get_latest_votes(last_version); + inc_new_counter_info!("bank_send_loop_batch_size", packet_batch.packets.len()); inc_new_counter_info!("bank_send_loop_num_batches", 1); - verified_packets_sender.send(vec![msgs])?; + verified_packets_sender.send(vec![packet_batch])?; #[allow(deprecated)] bank.last_vote_sync.compare_and_swap( last_version, @@ -463,61 +432,6 @@ impl ClusterInfoVoteListener { } } -<<<<<<< HEAD -======= - fn check_for_leader_bank_and_send_votes( - bank_vote_sender_state_option: &mut Option, - current_working_bank: Arc, - verified_packets_sender: &CrossbeamSender>, - verified_vote_packets: &VerifiedVotePackets, - ) -> Result<()> { - // We will take this lock at most once every `BANK_SEND_VOTES_LOOP_SLEEP_MS` - if let Some(bank_vote_sender_state) = bank_vote_sender_state_option { - if bank_vote_sender_state.bank.slot() != current_working_bank.slot() { - bank_vote_sender_state.report_metrics(); - *bank_vote_sender_state_option = - Some(BankVoteSenderState::new(current_working_bank)); - } - } else { - *bank_vote_sender_state_option = Some(BankVoteSenderState::new(current_working_bank)); - } - - let bank_vote_sender_state = bank_vote_sender_state_option.as_mut().unwrap(); - let BankVoteSenderState { - ref bank, - ref mut bank_send_votes_stats, - ref mut previously_sent_to_bank_votes, - } = bank_vote_sender_state; - - // This logic may run multiple times for the same leader bank, - // we just have to ensure that the same votes are not sent - // to the bank multiple times, which is guaranteed by - // `previously_sent_to_bank_votes` - let gossip_votes_iterator = ValidatorGossipVotesIterator::new( - bank.clone(), - verified_vote_packets, - previously_sent_to_bank_votes, - ); - - let mut filter_gossip_votes_timing = Measure::start("filter_gossip_votes"); - - // Send entire batch at a time so that there is no partial processing of - // a single validator's votes by two different banks. This might happen - // if we sent each vote individually, for instance if we creaed two different - // leader banks from the same common parent, one leader bank may process - // only the later votes and ignore the earlier votes. - for single_validator_votes in gossip_votes_iterator { - bank_send_votes_stats.num_votes_sent += single_validator_votes.len(); - bank_send_votes_stats.num_batches_sent += 1; - verified_packets_sender.send(single_validator_votes)?; - } - filter_gossip_votes_timing.stop(); - bank_send_votes_stats.total_elapsed += filter_gossip_votes_timing.as_us(); - - Ok(()) - } - ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) #[allow(clippy::too_many_arguments)] fn process_votes_loop( exit: Arc, @@ -1794,16 +1708,11 @@ mod tests { assert!(packets.is_empty()); } -<<<<<<< HEAD - fn verify_packets_len(packets: &[(CrdsValueLabel, Slot, Packets)], ref_value: usize) { - let num_packets: usize = packets.iter().map(|(_, _, p)| p.packets.len()).sum(); -======= - fn verify_packets_len(packets: &[VerifiedVoteMetadata], ref_value: usize) { + fn verify_packets_len(packets: &[(CrdsValueLabel, Slot, PacketBatch)], ref_value: usize) { let num_packets: usize = packets .iter() - .map(|vote_metadata| vote_metadata.packet_batch.packets.len()) + .map(|(_, _, batch)| batch.packets.len()) .sum(); ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) assert_eq!(num_packets, ref_value); } diff --git a/core/src/fetch_stage.rs b/core/src/fetch_stage.rs index 8cfa9745d0b..d890a9ed354 100644 --- a/core/src/fetch_stage.rs +++ b/core/src/fetch_stage.rs @@ -100,16 +100,9 @@ impl FetchStage { .unwrap() .would_be_leader(HOLD_TRANSACTIONS_SLOT_OFFSET.saturating_mul(DEFAULT_TICKS_PER_SLOT)) { -<<<<<<< HEAD - inc_new_counter_debug!("fetch_stage-honor_forwards", len); - for packets in batch { - if sendr.send(packets).is_err() { -======= inc_new_counter_debug!("fetch_stage-honor_forwards", num_packets); for packet_batch in packet_batches { - #[allow(clippy::question_mark)] if sendr.send(packet_batch).is_err() { ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) return Err(Error::Send); } } diff --git a/core/src/serve_repair.rs b/core/src/serve_repair.rs index 20a43d1df31..6ff37c0f925 100644 --- a/core/src/serve_repair.rs +++ b/core/src/serve_repair.rs @@ -635,43 +635,6 @@ impl ServeRepair { } Some(res) } -<<<<<<< HEAD -======= - - fn run_ancestor_hashes( - recycler: &PacketBatchRecycler, - from_addr: &SocketAddr, - blockstore: Option<&Arc>, - slot: Slot, - nonce: Nonce, - ) -> Option { - let blockstore = blockstore?; - let ancestor_slot_hashes = if blockstore.is_duplicate_confirmed(slot) { - let ancestor_iterator = - AncestorIteratorWithHash::from(AncestorIterator::new_inclusive(slot, blockstore)); - ancestor_iterator.take(MAX_ANCESTOR_RESPONSES).collect() - } else { - // If this slot is not duplicate confirmed, return nothing - vec![] - }; - let response = AncestorHashesResponseVersion::Current(ancestor_slot_hashes); - let serialized_response = serialize(&response).ok()?; - - // Could probably directly write response into packet via `serialize_into()` - // instead of incurring extra copy in `repair_response_packet_from_bytes`, but - // serialize_into doesn't return the written size... - let packet = repair_response::repair_response_packet_from_bytes( - serialized_response, - from_addr, - nonce, - )?; - Some(PacketBatch::new_unpinned_with_recycler_data( - recycler, - "run_ancestor_hashes", - vec![packet], - )) - } ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) } #[cfg(test)] @@ -1070,92 +1033,6 @@ mod tests { } #[test] -<<<<<<< HEAD -======= - fn test_run_ancestor_hashes() { - solana_logger::setup(); - let recycler = PacketBatchRecycler::default(); - let ledger_path = get_tmp_ledger_path!(); - { - let slot = 0; - let num_slots = MAX_ANCESTOR_RESPONSES as u64; - let nonce = 10; - - let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap()); - - // Create slots [slot, slot + num_slots) with 5 shreds apiece - let (shreds, _) = make_many_slot_entries(slot, num_slots, 5); - - blockstore - .insert_shreds(shreds, None, false) - .expect("Expect successful ledger write"); - - // We don't have slot `slot + num_slots`, so we return empty - let rv = ServeRepair::run_ancestor_hashes( - &recycler, - &socketaddr_any!(), - Some(&blockstore), - slot + num_slots, - nonce, - ) - .expect("run_ancestor_hashes packets") - .packets; - assert_eq!(rv.len(), 1); - let packet = &rv[0]; - let ancestor_hashes_response: AncestorHashesResponseVersion = - limited_deserialize(&packet.data[..packet.meta.size - SIZE_OF_NONCE]).unwrap(); - assert!(ancestor_hashes_response.into_slot_hashes().is_empty()); - - // `slot + num_slots - 1` is not marked duplicate confirmed so nothing should return - // empty - let rv = ServeRepair::run_ancestor_hashes( - &recycler, - &socketaddr_any!(), - Some(&blockstore), - slot + num_slots - 1, - nonce, - ) - .expect("run_ancestor_hashes packets") - .packets; - assert_eq!(rv.len(), 1); - let packet = &rv[0]; - let ancestor_hashes_response: AncestorHashesResponseVersion = - limited_deserialize(&packet.data[..packet.meta.size - SIZE_OF_NONCE]).unwrap(); - assert!(ancestor_hashes_response.into_slot_hashes().is_empty()); - - // Set duplicate confirmed - let mut expected_ancestors = Vec::with_capacity(num_slots as usize); - expected_ancestors.resize(num_slots as usize, (0, Hash::default())); - for (i, duplicate_confirmed_slot) in (slot..slot + num_slots).enumerate() { - let frozen_hash = Hash::new_unique(); - expected_ancestors[num_slots as usize - i - 1] = - (duplicate_confirmed_slot, frozen_hash); - blockstore.insert_bank_hash(duplicate_confirmed_slot, frozen_hash, true); - } - let rv = ServeRepair::run_ancestor_hashes( - &recycler, - &socketaddr_any!(), - Some(&blockstore), - slot + num_slots - 1, - nonce, - ) - .expect("run_ancestor_hashes packets") - .packets; - assert_eq!(rv.len(), 1); - let packet = &rv[0]; - let ancestor_hashes_response: AncestorHashesResponseVersion = - limited_deserialize(&packet.data[..packet.meta.size - SIZE_OF_NONCE]).unwrap(); - assert_eq!( - ancestor_hashes_response.into_slot_hashes(), - expected_ancestors - ); - } - - Blockstore::destroy(&ledger_path).expect("Expected successful database destruction"); - } - - #[test] ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) fn test_repair_with_repair_validators() { let cluster_slots = ClusterSlots::default(); let me = ContactInfo::new_localhost(&solana_sdk::pubkey::new_rand(), timestamp()); diff --git a/core/src/shred_fetch_stage.rs b/core/src/shred_fetch_stage.rs index 4beaf3cc1b8..ae93816d72f 100644 --- a/core/src/shred_fetch_stage.rs +++ b/core/src/shred_fetch_stage.rs @@ -97,13 +97,8 @@ impl ShredFetchStage { slots_per_epoch = root_bank.get_slots_in_epoch(root_bank.epoch()); } } -<<<<<<< HEAD - stats.shred_count += p.packets.len(); - p.packets.iter_mut().for_each(|mut packet| { -======= stats.shred_count += packet_batch.packets.len(); - packet_batch.packets.iter_mut().for_each(|packet| { ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) + packet_batch.packets.iter_mut().for_each(|mut packet| { Self::process_packet( &mut packet, &mut shreds_received, diff --git a/core/src/sigverify_stage.rs b/core/src/sigverify_stage.rs index 3b6f88e2a70..5e09cc62205 100644 --- a/core/src/sigverify_stage.rs +++ b/core/src/sigverify_stage.rs @@ -174,7 +174,6 @@ impl SigVerifyStage { verifier: &T, stats: &mut SigVerifierStats, ) -> Result<()> { -<<<<<<< HEAD let (mut batches, len, recv_time) = streamer::recv_batch(recvr)?; let mut verify_batch_time = Measure::start("sigverify_batch_time"); @@ -183,23 +182,7 @@ impl SigVerifyStage { if len > MAX_SIGVERIFY_BATCH { Self::discard_excess_packets(&mut batches, MAX_SIGVERIFY_BATCH); } - sendr.send(verifier.verify_batch(batches))?; -======= - let (mut batches, num_packets, recv_duration) = streamer::recv_packet_batches(recvr)?; - - let batches_len = batches.len(); - debug!( - "@{:?} verifier: verifying: {}", - timing::timestamp(), - num_packets, - ); - if num_packets > MAX_SIGVERIFY_BATCH { - Self::discard_excess_packets(&mut batches, MAX_SIGVERIFY_BATCH); - } - - let mut verify_batch_time = Measure::start("sigverify_batch_time"); sendr.send(verifier.verify_batches(batches))?; ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) verify_batch_time.stop(); debug!( diff --git a/core/src/verified_vote_packets.rs b/core/src/verified_vote_packets.rs index 0f8efed6b9a..c2fd3ff31a1 100644 --- a/core/src/verified_vote_packets.rs +++ b/core/src/verified_vote_packets.rs @@ -1,147 +1,16 @@ use { crate::{cluster_info_vote_listener::VerifiedLabelVotePacketsReceiver, result::Result}, -<<<<<<< HEAD solana_gossip::crds_value::CrdsValueLabel, - solana_perf::packet::Packets, - solana_sdk::clock::Slot, -======= - crossbeam_channel::Select, solana_perf::packet::PacketBatch, - solana_runtime::bank::Bank, - solana_sdk::{ - account::from_account, clock::Slot, hash::Hash, pubkey::Pubkey, signature::Signature, - slot_hashes::SlotHashes, sysvar, - }, - solana_vote_program::vote_state::VoteTransaction, ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) + solana_sdk::clock::Slot, std::{ collections::{hash_map::Entry, HashMap}, time::Duration, }, }; -<<<<<<< HEAD -======= -const MAX_VOTES_PER_VALIDATOR: usize = 1000; - -pub struct VerifiedVoteMetadata { - pub vote_account_key: Pubkey, - pub vote: Box, - pub packet_batch: PacketBatch, - pub signature: Signature, -} - -pub struct ValidatorGossipVotesIterator<'a> { - my_leader_bank: Arc, - slot_hashes: SlotHashes, - verified_vote_packets: &'a VerifiedVotePackets, - vote_account_keys: Vec, - previously_sent_to_bank_votes: &'a mut HashSet, -} - -impl<'a> ValidatorGossipVotesIterator<'a> { - pub fn new( - my_leader_bank: Arc, - verified_vote_packets: &'a VerifiedVotePackets, - previously_sent_to_bank_votes: &'a mut HashSet, - ) -> Self { - let slot_hashes_account = my_leader_bank.get_account(&sysvar::slot_hashes::id()); - - if slot_hashes_account.is_none() { - warn!( - "Slot hashes sysvar doesn't exist on bank {}", - my_leader_bank.slot() - ); - } - - let slot_hashes_account = slot_hashes_account.unwrap_or_default(); - let slot_hashes = from_account::(&slot_hashes_account).unwrap_or_default(); - - // TODO: my_leader_bank.vote_accounts() may not contain zero-staked validators - // in this epoch, but those validators may have stake warming up in the next epoch - let vote_account_keys: Vec = - my_leader_bank.vote_accounts().keys().copied().collect(); - - Self { - my_leader_bank, - slot_hashes, - verified_vote_packets, - vote_account_keys, - previously_sent_to_bank_votes, - } - } -} - -/// Each iteration returns all of the missing votes for a single validator, the votes -/// ordered from smallest to largest. -/// -/// Iterator is done after iterating through all vote accounts -impl<'a> Iterator for ValidatorGossipVotesIterator<'a> { - type Item = Vec; - - fn next(&mut self) -> Option { - // TODO: Maybe prioritize by stake weight - while !self.vote_account_keys.is_empty() { - let vote_account_key = self.vote_account_keys.pop().unwrap(); - // Get all the gossip votes we've queued up for this validator - // that are: - // 1) missing from the current leader bank - // 2) on the same fork - let validator_votes = self - .verified_vote_packets - .0 - .get(&vote_account_key) - .and_then(|validator_gossip_votes| { - // Fetch the validator's vote state from the bank - self.my_leader_bank - .vote_accounts() - .get(&vote_account_key) - .and_then(|(_stake, vote_account)| { - vote_account.vote_state().as_ref().ok().map(|vote_state| { - let start_vote_slot = - vote_state.last_voted_slot().map(|x| x + 1).unwrap_or(0); - // Filter out the votes that are outdated - validator_gossip_votes - .range((start_vote_slot, Hash::default())..) - .filter_map(|((slot, hash), (packet, tx_signature))| { - if self.previously_sent_to_bank_votes.contains(tx_signature) - { - return None; - } - // Don't send the same vote to the same bank multiple times - self.previously_sent_to_bank_votes.insert(*tx_signature); - // Filter out votes on the wrong fork (or too old to be) - // on this fork - if self - .slot_hashes - .get(slot) - .map(|found_hash| found_hash == hash) - .unwrap_or(false) - { - Some(packet.clone()) - } else { - None - } - }) - .collect::>() - }) - }) - }); - if let Some(validator_votes) = validator_votes { - if !validator_votes.is_empty() { - return Some(validator_votes); - } - } - } - None - } -} - -pub type SingleValidatorVotes = BTreeMap<(Slot, Hash), (PacketBatch, Signature)>; - ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) #[derive(Default)] -pub struct VerifiedVotePackets(HashMap); +pub struct VerifiedVotePackets(HashMap); impl VerifiedVotePackets { pub fn receive_and_process_vote_packets( @@ -163,32 +32,8 @@ impl VerifiedVotePackets { } while let Ok(vote_packets) = vote_packets_receiver.try_recv() { if would_be_leader { -<<<<<<< HEAD for (label, slot, packet) in vote_packets { self.0.insert(label, (*last_update_version, slot, packet)); -======= - for verfied_vote_metadata in gossip_votes { - let VerifiedVoteMetadata { - vote_account_key, - vote, - packet_batch, - signature, - } = verfied_vote_metadata; - if vote.is_empty() { - error!("Empty votes should have been filtered out earlier in the pipeline"); - continue; - } - let slot = vote.last_voted_slot().unwrap(); - let hash = vote.hash(); - - let validator_votes = self.0.entry(vote_account_key).or_default(); - validator_votes.insert((slot, hash), (packet_batch, signature)); - - if validator_votes.len() > MAX_VOTES_PER_VALIDATOR { - let smallest_key = validator_votes.keys().next().cloned().unwrap(); - validator_votes.remove(&smallest_key).unwrap(); - } ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) } } } @@ -196,11 +41,11 @@ impl VerifiedVotePackets { } #[cfg(test)] - fn get_vote_packets(&self, key: &CrdsValueLabel) -> Option<&(u64, Slot, Packets)> { + fn get_vote_packets(&self, key: &CrdsValueLabel) -> Option<&(u64, Slot, PacketBatch)> { self.0.get(key) } - pub fn get_latest_votes(&self, last_update_version: u64) -> (u64, Packets) { + pub fn get_latest_votes(&self, last_update_version: u64) -> (u64, PacketBatch) { let mut new_update_version = last_update_version; let mut votes = HashMap::new(); for (label, (version, slot, packets)) in &self.0 { @@ -225,7 +70,7 @@ impl VerifiedVotePackets { .flat_map(|(_, (_, packets))| &packets.packets) .cloned() .collect(); - (new_update_version, Packets::new(packets)) + (new_update_version, PacketBatch::new(packets)) } } @@ -245,7 +90,6 @@ mod tests { let label2 = CrdsValueLabel::Vote(1, pubkey); let mut verified_vote_packets = VerifiedVotePackets(HashMap::new()); -<<<<<<< HEAD let data = Packet { meta: Meta { repair: true, @@ -254,290 +98,14 @@ mod tests { ..Packet::default() }; - let none_empty_packets = Packets::new(vec![data, Packet::default()]); + let none_empty_packets = PacketBatch::new(vec![data, Packet::default()]); verified_vote_packets .0 .insert(label1, (2, 42, none_empty_packets)); -======= - // Send a vote from `vote_account_key`, check that it was inserted - let vote_slot = 0; - let vote_hash = Hash::new_unique(); - let vote = Vote::new(vec![vote_slot], vote_hash); - s.send(vec![VerifiedVoteMetadata { - vote_account_key, - vote: Box::new(vote.clone()), - packet_batch: PacketBatch::default(), - signature: Signature::new(&[1u8; 64]), - }]) - .unwrap(); - verified_vote_packets - .receive_and_process_vote_packets(&r, true) - .unwrap(); - assert_eq!( - verified_vote_packets - .0 - .get(&vote_account_key) - .unwrap() - .len(), - 1 - ); - - // Same slot, same hash, should not be inserted - s.send(vec![VerifiedVoteMetadata { - vote_account_key, - vote: Box::new(vote), - packet_batch: PacketBatch::default(), - signature: Signature::new(&[1u8; 64]), - }]) - .unwrap(); - verified_vote_packets - .receive_and_process_vote_packets(&r, true) - .unwrap(); - assert_eq!( - verified_vote_packets - .0 - .get(&vote_account_key) - .unwrap() - .len(), - 1 - ); - - // Same slot, different hash, should still be inserted - let new_vote_hash = Hash::new_unique(); - let vote = Vote::new(vec![vote_slot], new_vote_hash); - s.send(vec![VerifiedVoteMetadata { - vote_account_key, - vote: Box::new(vote), - packet_batch: PacketBatch::default(), - signature: Signature::new(&[1u8; 64]), - }]) - .unwrap(); - verified_vote_packets - .receive_and_process_vote_packets(&r, true) - .unwrap(); - assert_eq!( - verified_vote_packets - .0 - .get(&vote_account_key) - .unwrap() - .len(), - 2 - ); - - // Different vote slot, should be inserted - let vote_slot = 1; - let vote_hash = Hash::new_unique(); - let vote = Vote::new(vec![vote_slot], vote_hash); - s.send(vec![VerifiedVoteMetadata { - vote_account_key, - vote: Box::new(vote), - packet_batch: PacketBatch::default(), - signature: Signature::new(&[2u8; 64]), - }]) - .unwrap(); - verified_vote_packets - .receive_and_process_vote_packets(&r, true) - .unwrap(); - assert_eq!( - verified_vote_packets - .0 - .get(&vote_account_key) - .unwrap() - .len(), - 3 - ); - - // No new messages, should time out - assert_matches!( - verified_vote_packets.receive_and_process_vote_packets(&r, true), - Err(Error::ReadyTimeout) - ); - } - - #[test] - fn test_verified_vote_packets_receive_and_process_vote_packets_max_len() { - let (s, r) = unbounded(); - let vote_account_key = solana_sdk::pubkey::new_rand(); - - // Construct the buffer - let mut verified_vote_packets = VerifiedVotePackets(HashMap::new()); - - // Send many more votes than the upper limit per validator - for _ in 0..2 * MAX_VOTES_PER_VALIDATOR { - let vote_slot = 0; - let vote_hash = Hash::new_unique(); - let vote = Vote::new(vec![vote_slot], vote_hash); - s.send(vec![VerifiedVoteMetadata { - vote_account_key, - vote: Box::new(vote), - packet_batch: PacketBatch::default(), - signature: Signature::new(&[1u8; 64]), - }]) - .unwrap(); - } - - // At most `MAX_VOTES_PER_VALIDATOR` should be stored per validator - verified_vote_packets - .receive_and_process_vote_packets(&r, true) - .unwrap(); - assert_eq!( - verified_vote_packets - .0 - .get(&vote_account_key) - .unwrap() - .len(), - MAX_VOTES_PER_VALIDATOR - ); - } - - #[test] - fn test_verified_vote_packets_validator_gossip_votes_iterator_wrong_fork() { - let (s, r) = unbounded(); - let vote_simulator = VoteSimulator::new(1); - let my_leader_bank = vote_simulator.bank_forks.read().unwrap().root_bank(); - let vote_account_key = vote_simulator.vote_pubkeys[0]; - - // Create a bunch of votes with random vote hashes, which should all be ignored - // since they are not on the same fork as `my_leader_bank`, i.e. their hashes do - // not exist in the SlotHashes sysvar for `my_leader_bank` - for _ in 0..MAX_VOTES_PER_VALIDATOR { - let vote_slot = 0; - let vote_hash = Hash::new_unique(); - let vote = Vote::new(vec![vote_slot], vote_hash); - s.send(vec![VerifiedVoteMetadata { - vote_account_key, - vote: Box::new(vote), - packet_batch: PacketBatch::default(), - signature: Signature::new_unique(), - }]) - .unwrap(); - } - - // Ingest the votes into the buffer - let mut verified_vote_packets = VerifiedVotePackets(HashMap::new()); - verified_vote_packets - .receive_and_process_vote_packets(&r, true) - .unwrap(); - - // Create tracker for previously sent bank votes - let mut previously_sent_to_bank_votes = HashSet::new(); - let mut gossip_votes_iterator = ValidatorGossipVotesIterator::new( - my_leader_bank, - &verified_vote_packets, - &mut previously_sent_to_bank_votes, - ); - - // Wrong fork, we should get no hashes - assert!(gossip_votes_iterator.next().is_none()); - } - - #[test] - fn test_verified_vote_packets_validator_gossip_votes_iterator_correct_fork() { - let (s, r) = unbounded(); - let num_validators = 2; - let vote_simulator = VoteSimulator::new(2); - let mut my_leader_bank = vote_simulator.bank_forks.read().unwrap().root_bank(); - - // Create a set of valid ancestor hashes for this fork - for _ in 0..MAX_ENTRIES { - my_leader_bank = Arc::new(Bank::new_from_parent( - &my_leader_bank, - &Pubkey::default(), - my_leader_bank.slot() + 1, - )); - } - let slot_hashes_account = my_leader_bank - .get_account(&sysvar::slot_hashes::id()) - .expect("Slot hashes sysvar must exist"); - let slot_hashes = from_account::(&slot_hashes_account).unwrap(); - - // Create valid votes - for i in 0..num_validators { - let vote_account_key = vote_simulator.vote_pubkeys[i]; - // Used to uniquely identify the packets for each validator - let num_packets = i + 1; - for (vote_slot, vote_hash) in slot_hashes.slot_hashes().iter() { - let vote = Vote::new(vec![*vote_slot], *vote_hash); - s.send(vec![VerifiedVoteMetadata { - vote_account_key, - vote: Box::new(vote), - packet_batch: PacketBatch::new(vec![Packet::default(); num_packets]), - signature: Signature::new_unique(), - }]) - .unwrap(); - } - } - - // Ingest the votes into the buffer - let mut verified_vote_packets = VerifiedVotePackets(HashMap::new()); - verified_vote_packets - .receive_and_process_vote_packets(&r, true) - .unwrap(); - - // Check we get two batches, one for each validator. Each batch - // should only contain a packets structure with the specific number - // of packets associated with that batch - assert_eq!(verified_vote_packets.0.len(), 2); - // Every validator should have `slot_hashes.slot_hashes().len()` votes - assert!(verified_vote_packets - .0 - .values() - .all(|validator_votes| validator_votes.len() == slot_hashes.slot_hashes().len())); - - let mut previously_sent_to_bank_votes = HashSet::new(); - let mut gossip_votes_iterator = ValidatorGossipVotesIterator::new( - my_leader_bank.clone(), - &verified_vote_packets, - &mut previously_sent_to_bank_votes, - ); - - // Get and verify batches - let num_expected_batches = 2; - for _ in 0..num_expected_batches { - let validator_batch: Vec = gossip_votes_iterator.next().unwrap(); - assert_eq!(validator_batch.len(), slot_hashes.slot_hashes().len()); - let expected_len = validator_batch[0].packets.len(); - assert!(validator_batch - .iter() - .all(|batch| batch.packets.len() == expected_len)); - } - - // Should be empty now - assert!(gossip_votes_iterator.next().is_none()); - - // If we construct another iterator, should return nothing because `previously_sent_to_bank_votes` - // should filter out everything - let mut gossip_votes_iterator = ValidatorGossipVotesIterator::new( - my_leader_bank.clone(), - &verified_vote_packets, - &mut previously_sent_to_bank_votes, - ); - assert!(gossip_votes_iterator.next().is_none()); - - // If we add a new vote, we should return it - my_leader_bank.freeze(); - let vote_slot = my_leader_bank.slot(); - let vote_hash = my_leader_bank.hash(); - let my_leader_bank = Arc::new(Bank::new_from_parent( - &my_leader_bank, - &Pubkey::default(), - my_leader_bank.slot() + 1, - )); - let vote_account_key = vote_simulator.vote_pubkeys[1]; - let vote = Box::new(Vote::new(vec![vote_slot], vote_hash)); - s.send(vec![VerifiedVoteMetadata { - vote_account_key, - vote, - packet_batch: PacketBatch::default(), - signature: Signature::new_unique(), - }]) - .unwrap(); - // Ingest the votes into the buffer ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) verified_vote_packets .0 - .insert(label2, (1, 23, Packets::default())); + .insert(label2, (1, 23, PacketBatch::default())); // Both updates have timestamps greater than 0, so both should be returned let (new_update_version, updates) = verified_vote_packets.get_latest_votes(0); @@ -564,9 +132,9 @@ mod tests { let label1 = CrdsValueLabel::Vote(0, pubkey); let label2 = CrdsValueLabel::Vote(1, pubkey); let mut update_version = 0; - s.send(vec![(label1.clone(), 17, Packets::default())]) + s.send(vec![(label1.clone(), 17, PacketBatch::default())]) .unwrap(); - s.send(vec![(label2.clone(), 23, Packets::default())]) + s.send(vec![(label2.clone(), 23, PacketBatch::default())]) .unwrap(); let data = Packet { @@ -577,7 +145,7 @@ mod tests { ..Packet::default() }; - let later_packets = Packets::new(vec![data, Packet::default()]); + let later_packets = PacketBatch::new(vec![data, Packet::default()]); s.send(vec![(label1.clone(), 42, later_packets)]).unwrap(); let mut verified_vote_packets = VerifiedVotePackets(HashMap::new()); verified_vote_packets @@ -612,7 +180,7 @@ mod tests { ); // Test timestamp for next batch overwrites the original - s.send(vec![(label2.clone(), 51, Packets::default())]) + s.send(vec![(label2.clone(), 51, PacketBatch::default())]) .unwrap(); verified_vote_packets .receive_and_process_vote_packets(&r, &mut update_version, true) diff --git a/core/src/window_service.rs b/core/src/window_service.rs index ef4f3460a88..cb7f35dd793 100644 --- a/core/src/window_service.rs +++ b/core/src/window_service.rs @@ -453,12 +453,8 @@ impl WindowService { #[allow(clippy::too_many_arguments)] pub(crate) fn new( blockstore: Arc, -<<<<<<< HEAD cluster_info: Arc, - verified_receiver: CrossbeamReceiver>, -======= verified_receiver: CrossbeamReceiver>, ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) retransmit_sender: Sender>, repair_socket: Arc, exit: Arc, diff --git a/gossip/src/cluster_info.rs b/gossip/src/cluster_info.rs index ab9991492ad..f6c377d536f 100644 --- a/gossip/src/cluster_info.rs +++ b/gossip/src/cluster_info.rs @@ -1817,12 +1817,8 @@ impl ClusterInfo { thread_pool: &ThreadPool, recycler: &PacketBatchRecycler, stakes: &HashMap, -<<<<<<< HEAD - response_sender: &PacketSender, - require_stake_for_gossip: bool, -======= response_sender: &PacketBatchSender, ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) + require_stake_for_gossip: bool, ) { let _st = ScopedTimer::from(&self.stats.handle_batch_pull_requests_time); if requests.is_empty() { @@ -1933,12 +1929,8 @@ impl ClusterInfo { recycler: &PacketBatchRecycler, requests: Vec, stakes: &HashMap, -<<<<<<< HEAD require_stake_for_gossip: bool, - ) -> Packets { -======= ) -> PacketBatch { ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) const DEFAULT_EPOCH_DURATION_MS: u64 = DEFAULT_SLOTS_PER_EPOCH * DEFAULT_MS_PER_SLOT; let mut time = Measure::start("handle_pull_requests"); let callers = crds_value::filter_current(requests.iter().map(|r| &r.caller)); @@ -2216,12 +2208,8 @@ impl ClusterInfo { thread_pool: &ThreadPool, recycler: &PacketBatchRecycler, stakes: &HashMap, -<<<<<<< HEAD - response_sender: &PacketSender, - require_stake_for_gossip: bool, -======= response_sender: &PacketBatchSender, ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) + require_stake_for_gossip: bool, ) { let _st = ScopedTimer::from(&self.stats.handle_batch_push_messages_time); if messages.is_empty() { @@ -2296,13 +2284,8 @@ impl ClusterInfo { let num_prune_packets = packet_batch.packets.len(); self.stats .push_response_count -<<<<<<< HEAD - .add_relaxed(packets.packets.len() as u64); - let new_push_requests = self.new_push_requests(stakes, require_stake_for_gossip); -======= .add_relaxed(packet_batch.packets.len() as u64); - let new_push_requests = self.new_push_requests(stakes); ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) + let new_push_requests = self.new_push_requests(stakes, require_stake_for_gossip); inc_new_counter_debug!("cluster_info-push_message-pushes", new_push_requests.len()); for (address, request) in new_push_requests { if ContactInfo::is_valid_address(&address, &self.socket_addr_space) { diff --git a/ledger/src/entry.rs b/ledger/src/entry.rs index feb2f564486..272262f571b 100644 --- a/ledger/src/entry.rs +++ b/ledger/src/entry.rs @@ -13,17 +13,7 @@ use { solana_measure::measure::Measure, solana_merkle_tree::MerkleTree, solana_metrics::*, -<<<<<<< HEAD:ledger/src/entry.rs solana_perf::{cuda_runtime::PinnedVec, perf_libs, recycler::Recycler}, -======= - solana_perf::{ - cuda_runtime::PinnedVec, - packet::{Packet, PacketBatch, PacketBatchRecycler, PACKETS_PER_BATCH}, - perf_libs, - recycler::Recycler, - sigverify, - }, ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)):entry/src/entry.rs solana_rayon_threadlimit::get_thread_count, solana_runtime::hashed_transaction::HashedTransaction, solana_sdk::{ @@ -277,12 +267,6 @@ pub struct EntryVerificationState { pub struct VerifyRecyclers { hash_recycler: Recycler>, tick_count_recycler: Recycler>, -<<<<<<< HEAD:ledger/src/entry.rs -======= - packet_recycler: PacketBatchRecycler, - out_recycler: Recycler>, - tx_offset_recycler: Recycler, ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)):entry/src/entry.rs } #[derive(PartialEq, Clone, Copy, Debug)] @@ -348,199 +332,6 @@ impl EntryVerificationState { } } -<<<<<<< HEAD:ledger/src/entry.rs -======= -pub fn verify_transactions( - entries: Vec, - verify: Arc Result + Send + Sync>, -) -> Result> { - PAR_THREAD_POOL.with(|thread_pool| { - thread_pool.borrow().install(|| { - entries - .into_par_iter() - .map(|entry| { - if entry.transactions.is_empty() { - Ok(EntryType::Tick(entry.hash)) - } else { - Ok(EntryType::Transactions( - entry - .transactions - .into_par_iter() - .map(verify.as_ref()) - .collect::>>()?, - )) - } - }) - .collect() - }) - }) -} - -pub fn start_verify_transactions( - entries: Vec, - skip_verification: bool, - verify_recyclers: VerifyRecyclers, - verify: Arc< - dyn Fn(VersionedTransaction, TransactionVerificationMode) -> Result - + Send - + Sync, - >, -) -> Result { - let api = perf_libs::api(); - - // Use the CPU if we have too few transactions for GPU signature verification to be worth it. - // We will also use the CPU if no acceleration API is used or if we're skipping - // the signature verification as we'd have nothing to do on the GPU in that case. - // TODO: make the CPU-to GPU crossover point dynamic, perhaps based on similar future - // heuristics to what might be used in sigverify::ed25519_verify when a dynamic crossover - // is introduced for that function (see TODO in sigverify::ed25519_verify) - let use_cpu = skip_verification - || api.is_none() - || entries - .iter() - .try_fold(0, |accum: usize, entry: &Entry| -> Option { - if accum.saturating_add(entry.transactions.len()) < 512 { - Some(accum.saturating_add(entry.transactions.len())) - } else { - None - } - }) - .is_some(); - - if use_cpu { - let verify_func = { - let verification_mode = if skip_verification { - TransactionVerificationMode::HashOnly - } else { - TransactionVerificationMode::FullVerification - }; - move |versioned_tx: VersionedTransaction| -> Result { - verify(versioned_tx, verification_mode) - } - }; - - let entries = verify_transactions(entries, Arc::new(verify_func)); - - match entries { - Ok(entries_val) => { - return Ok(EntrySigVerificationState { - verification_status: EntryVerificationStatus::Success, - entries: Some(entries_val), - device_verification_data: DeviceSigVerificationData::Cpu(), - gpu_verify_duration_us: 0, - }); - } - Err(err) => { - return Err(err); - } - } - } - - let verify_func = { - move |versioned_tx: VersionedTransaction| -> Result { - verify( - versioned_tx, - TransactionVerificationMode::HashAndVerifyPrecompiles, - ) - } - }; - let entries = verify_transactions(entries, Arc::new(verify_func)); - match entries { - Ok(entries) => { - let num_transactions: usize = entries - .iter() - .map(|entry: &EntryType| -> usize { - match entry { - EntryType::Transactions(transactions) => transactions.len(), - EntryType::Tick(_) => 0, - } - }) - .sum(); - - if num_transactions == 0 { - return Ok(EntrySigVerificationState { - verification_status: EntryVerificationStatus::Success, - entries: Some(entries), - device_verification_data: DeviceSigVerificationData::Cpu(), - gpu_verify_duration_us: 0, - }); - } - let entry_txs: Vec<&SanitizedTransaction> = entries - .iter() - .filter_map(|entry_type| match entry_type { - EntryType::Tick(_) => None, - EntryType::Transactions(transactions) => Some(transactions), - }) - .flatten() - .collect::>(); - let mut packet_batches = entry_txs - .par_iter() - .chunks(PACKETS_PER_BATCH) - .map(|slice| { - let vec_size = slice.len(); - let mut packet_batch = PacketBatch::new_with_recycler( - verify_recyclers.packet_recycler.clone(), - vec_size, - "entry-sig-verify", - ); - // We use set_len here instead of resize(num_transactions, Packet::default()), to save - // memory bandwidth and avoid writing a large amount of data that will be overwritten - // soon afterwards. As well, Packet::default() actually leaves the packet data - // uninitialized anyway, so the initilization would simply write junk into - // the vector anyway. - unsafe { - packet_batch.packets.set_len(vec_size); - } - let entry_tx_iter = slice - .into_par_iter() - .map(|tx| tx.to_versioned_transaction()); - - let res = packet_batch - .packets - .par_iter_mut() - .zip(entry_tx_iter) - .all(|pair| { - pair.0.meta = Meta::default(); - Packet::populate_packet(pair.0, None, &pair.1).is_ok() - }); - if res { - Ok(packet_batch) - } else { - Err(TransactionError::SanitizeFailure) - } - }) - .collect::>>()?; - - let tx_offset_recycler = verify_recyclers.tx_offset_recycler; - let out_recycler = verify_recyclers.out_recycler; - let gpu_verify_thread = thread::spawn(move || { - let mut verify_time = Measure::start("sigverify"); - sigverify::ed25519_verify( - &mut packet_batches, - &tx_offset_recycler, - &out_recycler, - false, - ); - let verified = packet_batches - .iter() - .all(|batch| batch.packets.iter().all(|p| !p.meta.discard)); - verify_time.stop(); - (verified, verify_time.as_us()) - }); - Ok(EntrySigVerificationState { - verification_status: EntryVerificationStatus::Pending, - entries: Some(entries), - device_verification_data: DeviceSigVerificationData::Gpu(GpuSigVerificationData { - thread_h: Some(gpu_verify_thread), - }), - gpu_verify_duration_us: 0, - }) - } - Err(err) => Err(err), - } -} - ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)):entry/src/entry.rs fn compare_hashes(computed_hash: Hash, ref_entry: &Entry) -> bool { let actual = if !ref_entry.transactions.is_empty() { let tx_hash = hash_transactions(&ref_entry.transactions); diff --git a/streamer/src/streamer.rs b/streamer/src/streamer.rs index 0e659dbe970..4197784d871 100644 --- a/streamer/src/streamer.rs +++ b/streamer/src/streamer.rs @@ -132,13 +132,7 @@ fn recv_send( Ok(()) } -<<<<<<< HEAD -pub fn recv_batch(recvr: &PacketReceiver) -> Result<(Vec, usize, u64)> { -======= -pub fn recv_packet_batches( - recvr: &PacketBatchReceiver, -) -> Result<(Vec, usize, Duration)> { ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) +pub fn recv_batch(recvr: &PacketBatchReceiver) -> Result<(Vec, usize, u64)> { let timer = Duration::new(1, 0); let packet_batch = recvr.recv_timeout(timer)?; let recv_start = Instant::now(); @@ -151,21 +145,16 @@ pub fn recv_packet_batches( packet_batches.push(packet_batch); } let recv_duration = recv_start.elapsed(); -<<<<<<< HEAD - trace!("batch len {}", batch.len()); - Ok(( - batch, - len, - solana_sdk::timing::duration_as_ms(&recv_duration), - )) -======= trace!( "packet batches len: {}, num packets: {}", packet_batches.len(), num_packets ); - Ok((packet_batches, num_packets, recv_duration)) ->>>>>>> 254ef3e7b (Rename Packets to PacketBatch (#21794)) + Ok(( + packet_batches, + num_packets, + solana_sdk::timing::duration_as_ms(&recv_duration), + )) } pub fn responder(