diff --git a/Cargo.lock b/Cargo.lock index b95af21ad5b6f1..9530d34d3313f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1582,6 +1582,15 @@ dependencies = [ "slab", ] +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + [[package]] name = "gag" version = "1.0.0" @@ -3268,6 +3277,60 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" +[[package]] +name = "quinn" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61a84d97630b137463c8e6802adc1dfe9de81457b41bb1ac59189e6761ab9255" +dependencies = [ + "bytes 1.1.0", + "futures-channel", + "futures-util", + "fxhash", + "quinn-proto", + "quinn-udp", + "rustls 0.20.2", + "thiserror", + "tokio", + "tracing", + "webpki 0.22.0", +] + +[[package]] +name = "quinn-proto" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "063dedf7983c8d57db474218f258daa85b627de6f2dbc458b690a93b1de790e8" +dependencies = [ + "bytes 1.1.0", + "fxhash", + "rand 0.8.4", + "ring", + "rustls 0.20.2", + "rustls-native-certs", + "rustls-pemfile", + "slab", + "thiserror", + "tinyvec", + "tracing", + "webpki 0.22.0", +] + +[[package]] +name = "quinn-udp" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f7996776e9ee3fc0e5c14476c1a640a17e993c847ae9c81191c2c102fbef903" +dependencies = [ + "futures-util", + "libc", + "mio 0.7.14", + "quinn-proto", + "socket2", + "tokio", + "tracing", +] + [[package]] name = "quote" version = "0.6.13" @@ -3771,9 +3834,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.20.0" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b5ac6078ca424dc1d3ae2328526a76787fecc7f8011f520e3276730e711fc95" +checksum = "d37e5e2290f3e040b594b1a9e04377c2c671f1a1cfd9bfdef82106ac1c113f84" dependencies = [ "log 0.4.14", "ring", @@ -3781,6 +3844,27 @@ dependencies = [ "webpki 0.22.0", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca9ebdfa27d3fc180e42879037b5338ab1c040c06affd00d8338598e7800943" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eebeaeb360c87bfb72e84abdb3447159c0eaececf1bef2aecd65a8be949d1c9" +dependencies = [ + "base64 0.13.0", +] + [[package]] name = "rustversion" version = "1.0.6" @@ -4441,6 +4525,7 @@ dependencies = [ name = "solana-bench-tps" version = "1.10.0" dependencies = [ + "bincode", "clap 2.33.3", "log 0.4.14", "rayon", @@ -5812,16 +5897,21 @@ dependencies = [ name = "solana-streamer" version = "1.10.0" dependencies = [ + "anyhow", + "bytes 1.1.0", "histogram", "itertools 0.10.3", "libc", "log 0.4.14", "nix", + "quinn", + "rustls 0.20.2", "solana-logger 1.10.0", "solana-metrics", "solana-perf", "solana-sdk", "thiserror", + "tokio", ] [[package]] @@ -6955,7 +7045,7 @@ dependencies = [ "httparse", "log 0.4.14", "rand 0.8.4", - "rustls 0.20.0", + "rustls 0.20.2", "sha-1", "thiserror", "url 2.2.2", diff --git a/accountsdb-plugin-postgres/src/postgres_client/postgres_client_transaction.rs b/accountsdb-plugin-postgres/src/postgres_client/postgres_client_transaction.rs index b2019f70f50211..0423b7561acd5e 100644 --- a/accountsdb-plugin-postgres/src/postgres_client/postgres_client_transaction.rs +++ b/accountsdb-plugin-postgres/src/postgres_client/postgres_client_transaction.rs @@ -1294,7 +1294,7 @@ pub(crate) mod tests { let transaction = VersionedTransaction::from(transaction); let transaction = - SanitizedTransaction::try_create(transaction, message_hash, Some(true), |_| { + SanitizedTransaction::try_create(transaction, message_hash, Some(true), None, |_| { Err(TransactionError::UnsupportedVersion) }) .unwrap(); @@ -1333,7 +1333,7 @@ pub(crate) mod tests { transaction.sanitize().unwrap(); let transaction = - SanitizedTransaction::try_create(transaction, message_hash, Some(true), |_message| { + SanitizedTransaction::try_create(transaction, message_hash, Some(true), None, |_message| { Ok(LoadedAddresses { writable: vec![Pubkey::new_unique(), Pubkey::new_unique()], readonly: vec![Pubkey::new_unique(), Pubkey::new_unique()], diff --git a/banking-bench/src/main.rs b/banking-bench/src/main.rs index 874b2e2c903f39..72513959cf9eab 100644 --- a/banking-bench/src/main.rs +++ b/banking-bench/src/main.rs @@ -13,7 +13,7 @@ use { get_tmp_ledger_path, }, solana_measure::measure::Measure, - solana_perf::packet::to_packet_batches, + solana_perf::packet::{to_packet_batches, PacketBatch}, solana_poh::poh_recorder::{create_test_recorder, PohRecorder, WorkingBankEntry}, solana_runtime::{ accounts_background_service::AbsRequestSender, bank::Bank, bank_forks::BankForks, @@ -21,6 +21,7 @@ use { }, solana_sdk::{ hash::Hash, + packet::Packet, signature::{Keypair, Signature}, system_transaction, timing::{duration_as_us, timestamp}, @@ -167,6 +168,8 @@ fn main() { } = create_genesis_config(mint_total); let (verified_sender, verified_receiver) = unbounded(); + // TODO: Actually pump some stuff into _verified_extended_sender like verified_sender ? + let (verified_extended_sender, verified_extended_receiver) = unbounded(); let (vote_sender, vote_receiver) = unbounded(); let (tpu_vote_sender, tpu_vote_receiver) = unbounded(); let (replay_vote_sender, _replay_vote_receiver) = unbounded(); @@ -212,7 +215,8 @@ fn main() { bank.clear_signatures(); } - let mut verified: Vec<_> = to_packet_batches(&transactions, packets_per_chunk); + let mut verified: Vec> = + to_packet_batches(&transactions, packets_per_chunk); let ledger_path = get_tmp_ledger_path!(); { let blockstore = Arc::new( @@ -231,6 +235,7 @@ fn main() { &cluster_info, &poh_recorder, verified_receiver, + verified_extended_receiver, tpu_vote_receiver, vote_receiver, None, @@ -389,6 +394,7 @@ fn main() { ); drop(verified_sender); + drop(verified_extended_sender); drop(tpu_vote_sender); drop(vote_sender); exit.store(true, Ordering::Relaxed); diff --git a/banks-server/src/banks_server.rs b/banks-server/src/banks_server.rs index 6e94d244016b48..78f845612fa4f7 100644 --- a/banks-server/src/banks_server.rs +++ b/banks-server/src/banks_server.rs @@ -373,6 +373,7 @@ pub async fn start_local_server( pub async fn start_tcp_server( listen_addr: SocketAddr, tpu_addr: SocketAddr, + tpu_extended_addr: SocketAddr, bank_forks: Arc>, block_commitment_cache: Arc>, ) -> io::Result<()> { @@ -396,6 +397,7 @@ pub async fn start_tcp_server( SendTransactionService::new::( tpu_addr, + tpu_extended_addr, &bank_forks, None, receiver, diff --git a/banks-server/src/rpc_banks_service.rs b/banks-server/src/rpc_banks_service.rs index bc328e18a74fe8..d2d7239c557b57 100644 --- a/banks-server/src/rpc_banks_service.rs +++ b/banks-server/src/rpc_banks_service.rs @@ -27,6 +27,7 @@ pub struct RpcBanksService { async fn start_abortable_tcp_server( listen_addr: SocketAddr, tpu_addr: SocketAddr, + tpu_extended_addr: SocketAddr, bank_forks: Arc>, block_commitment_cache: Arc>, exit: Arc, @@ -34,6 +35,7 @@ async fn start_abortable_tcp_server( let server = start_tcp_server( listen_addr, tpu_addr, + tpu_extended_addr, bank_forks.clone(), block_commitment_cache.clone(), ) @@ -56,6 +58,7 @@ impl RpcBanksService { fn run( listen_addr: SocketAddr, tpu_addr: SocketAddr, + tpu_extended_addr: SocketAddr, bank_forks: Arc>, block_commitment_cache: Arc>, exit: Arc, @@ -63,6 +66,7 @@ impl RpcBanksService { let server = start_abortable_tcp_server( listen_addr, tpu_addr, + tpu_extended_addr, bank_forks, block_commitment_cache, exit, @@ -73,6 +77,7 @@ impl RpcBanksService { pub fn new( listen_addr: SocketAddr, tpu_addr: SocketAddr, + tpu_extended_addr: SocketAddr, bank_forks: &Arc>, block_commitment_cache: &Arc>, exit: &Arc, @@ -86,6 +91,7 @@ impl RpcBanksService { Self::run( listen_addr, tpu_addr, + tpu_extended_addr, bank_forks, block_commitment_cache, exit, diff --git a/bench-streamer/src/main.rs b/bench-streamer/src/main.rs index 46eeeb761380e2..9292133130d385 100644 --- a/bench-streamer/src/main.rs +++ b/bench-streamer/src/main.rs @@ -3,7 +3,7 @@ use { clap::{crate_description, crate_name, App, Arg}, solana_streamer::{ packet::{Packet, PacketBatch, PacketBatchRecycler, PACKET_DATA_SIZE}, - streamer::{receiver, PacketBatchReceiver}, + streamer::{receiver, StandardPacketReceiver}, }, std::{ cmp::max, @@ -42,7 +42,8 @@ fn producer(addr: &SocketAddr, exit: Arc) -> JoinHandle<()> { }) } -fn sink(exit: Arc, rvs: Arc, r: PacketBatchReceiver) -> JoinHandle<()> { +// TODO: Revisit whether this should be generic +fn sink(exit: Arc, rvs: Arc, r: StandardPacketReceiver) -> JoinHandle<()> { spawn(move || loop { if exit.load(Ordering::Relaxed) { return; diff --git a/bench-tps/Cargo.toml b/bench-tps/Cargo.toml index 505e5e855ac0f5..846e4ea21f9ddf 100644 --- a/bench-tps/Cargo.toml +++ b/bench-tps/Cargo.toml @@ -27,7 +27,7 @@ solana-runtime = { path = "../runtime", version = "=1.10.0" } solana-sdk = { path = "../sdk", version = "=1.10.0" } solana-streamer = { path = "../streamer", version = "=1.10.0" } solana-version = { path = "../version", version = "=1.10.0" } - +bincode = "1.3.3" [dev-dependencies] serial_test = "0.5.1" solana-local-cluster = { path = "../local-cluster", version = "=1.10.0" } diff --git a/bench-tps/src/bench.rs b/bench-tps/src/bench.rs index fe7c4f79157ae0..6b3ebb253ec349 100644 --- a/bench-tps/src/bench.rs +++ b/bench-tps/src/bench.rs @@ -18,7 +18,7 @@ use { signature::{Keypair, Signer}, system_instruction, system_transaction, timing::{duration_as_ms, duration_as_s, duration_as_us, timestamp}, - transaction::Transaction, + transaction::{Transaction, TransactionError}, }, std::{ collections::{HashSet, VecDeque}, @@ -324,6 +324,29 @@ fn metrics_submit_lamport_balance(lamport_balance: u64) { ); } +/// Create and sign new large transaction of many system_instruction::Transfer +pub fn big_transfer( + from_keypair: &Keypair, + to: &Pubkey, + lamports: u64, + recent_blockhash: Hash, +) -> Transaction { + let from_pubkey = from_keypair.pubkey(); + + let instructions = (0..128).map(|_| + system_instruction::transfer(&from_pubkey, &to, lamports) + ).collect::>(); + + let tx = Transaction::new(&[from_keypair], + Message::new( + &instructions, + Some(&from_pubkey), + ), + recent_blockhash); + info!("transaction size: {}", bincode::serialized_size(&tx).map_err(|_| 0 as u64).unwrap()); + tx +} + fn generate_system_txs( source: &[&Keypair], dest: &VecDeque<&Keypair>, @@ -340,7 +363,7 @@ fn generate_system_txs( .par_iter() .map(|(from, to)| { ( - system_transaction::transfer(from, &to.pubkey(), 1, *blockhash), + big_transfer(from, &to.pubkey(), 1, *blockhash), timestamp(), ) }) diff --git a/bench-tps/src/main.rs b/bench-tps/src/main.rs index d1a2e379d2832b..d3b03a14a8255d 100644 --- a/bench-tps/src/main.rs +++ b/bench-tps/src/main.rs @@ -48,6 +48,7 @@ fn main() { info!("Generating {} keypairs", keypair_count); let (keypairs, _) = generate_keypairs(id, keypair_count as u64); let num_accounts = keypairs.len() as u64; + //let num_lamports_per_account = num_lamports_per_account*200; let max_fee = FeeRateGovernor::new(*target_lamports_per_signature, 0).max_lamports_per_signature; let num_lamports_per_account = (num_accounts - 1 + NUM_SIGNATURES_FOR_TXS * max_fee) diff --git a/client/src/thin_client.rs b/client/src/thin_client.rs index 12b50eb60647ca..2e72d19b9eccfe 100644 --- a/client/src/thin_client.rs +++ b/client/src/thin_client.rs @@ -17,7 +17,7 @@ use { hash::Hash, instruction::Instruction, message::Message, - packet::PACKET_DATA_SIZE, + packet::{PACKET_DATA_SIZE, MAX_TRANSACTION_SIZE}, pubkey::Pubkey, signature::{Keypair, Signature, Signer}, signers::Signers, @@ -120,6 +120,7 @@ impl ClientOptimizer { pub struct ThinClient { transactions_socket: UdpSocket, tpu_addrs: Vec, + extended_tpu_addrs: Vec, rpc_clients: Vec, optimizer: ClientOptimizer, } @@ -127,9 +128,10 @@ pub struct ThinClient { impl ThinClient { /// Create a new ThinClient that will interface with the Rpc at `rpc_addr` using TCP /// and the Tpu at `tpu_addr` over `transactions_socket` using UDP. - pub fn new(rpc_addr: SocketAddr, tpu_addr: SocketAddr, transactions_socket: UdpSocket) -> Self { + pub fn new(rpc_addr: SocketAddr, tpu_addr: SocketAddr, extended_tpu_addr: SocketAddr, transactions_socket: UdpSocket) -> Self { Self::new_from_client( tpu_addr, + extended_tpu_addr, transactions_socket, RpcClient::new_socket(rpc_addr), ) @@ -138,21 +140,24 @@ impl ThinClient { pub fn new_socket_with_timeout( rpc_addr: SocketAddr, tpu_addr: SocketAddr, + extended_tpu_addr: SocketAddr, transactions_socket: UdpSocket, timeout: Duration, ) -> Self { let rpc_client = RpcClient::new_socket_with_timeout(rpc_addr, timeout); - Self::new_from_client(tpu_addr, transactions_socket, rpc_client) + Self::new_from_client(tpu_addr, extended_tpu_addr, transactions_socket, rpc_client) } fn new_from_client( tpu_addr: SocketAddr, + extended_tpu_addr: SocketAddr, transactions_socket: UdpSocket, rpc_client: RpcClient, ) -> Self { Self { transactions_socket, tpu_addrs: vec![tpu_addr], + extended_tpu_addrs: vec![extended_tpu_addr], rpc_clients: vec![rpc_client], optimizer: ClientOptimizer::new(0), } @@ -161,6 +166,7 @@ impl ThinClient { pub fn new_from_addrs( rpc_addrs: Vec, tpu_addrs: Vec, + extended_tpu_addrs: Vec, transactions_socket: UdpSocket, ) -> Self { assert!(!rpc_addrs.is_empty()); @@ -171,6 +177,7 @@ impl ThinClient { Self { transactions_socket, tpu_addrs, + extended_tpu_addrs, rpc_clients, optimizer, } @@ -180,6 +187,10 @@ impl ThinClient { &self.tpu_addrs[self.optimizer.best()] } + fn tpu_extended_addr(&self) -> &SocketAddr { + &self.extended_tpu_addrs[self.optimizer.best()] + } + fn rpc_client(&self) -> &RpcClient { &self.rpc_clients[self.optimizer.best()] } @@ -613,9 +624,15 @@ impl AsyncClient for ThinClient { let mut wr = std::io::Cursor::new(&mut buf[..]); serialize_into(&mut wr, &transaction) .expect("serialize Transaction in pub fn transfer_signed"); - assert!(buf.len() < PACKET_DATA_SIZE); + assert!(buf.len() < MAX_TRANSACTION_SIZE); + if buf.len() < PACKET_DATA_SIZE { self.transactions_socket .send_to(&buf[..], &self.tpu_addr())?; + } + else { + self.transactions_socket + .send_to(&buf[..], &self.tpu_extended_addr())?; + } Ok(transaction.signatures[0]) } fn async_send_message( @@ -649,20 +666,20 @@ impl AsyncClient for ThinClient { } } -pub fn create_client((rpc, tpu): (SocketAddr, SocketAddr), range: (u16, u16)) -> ThinClient { +pub fn create_client((rpc, tpu, tpu_extended): (SocketAddr, SocketAddr, SocketAddr), range: (u16, u16)) -> ThinClient { let (_, transactions_socket) = solana_net_utils::bind_in_range(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), range).unwrap(); - ThinClient::new(rpc, tpu, transactions_socket) + ThinClient::new(rpc, tpu, tpu_extended, transactions_socket) } pub fn create_client_with_timeout( - (rpc, tpu): (SocketAddr, SocketAddr), + (rpc, tpu, tpu_extended): (SocketAddr, SocketAddr, SocketAddr), range: (u16, u16), timeout: Duration, ) -> ThinClient { let (_, transactions_socket) = solana_net_utils::bind_in_range(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), range).unwrap(); - ThinClient::new_socket_with_timeout(rpc, tpu, transactions_socket, timeout) + ThinClient::new_socket_with_timeout(rpc, tpu, tpu_extended, transactions_socket, timeout) } #[cfg(test)] diff --git a/core/src/ancestor_hashes_service.rs b/core/src/ancestor_hashes_service.rs index 9ce9a5efb2d78e..aa3e651ecfebe6 100644 --- a/core/src/ancestor_hashes_service.rs +++ b/core/src/ancestor_hashes_service.rs @@ -14,7 +14,7 @@ use { solana_ledger::{blockstore::Blockstore, shred::SIZE_OF_NONCE}, solana_measure::measure::Measure, solana_perf::{ - packet::{limited_deserialize, Packet, PacketBatch}, + packet::{limited_deserialize, Packet, StandardPacketBatch}, recycler::Recycler, }, solana_runtime::bank::Bank, @@ -23,7 +23,7 @@ use { pubkey::Pubkey, timing::timestamp, }, - solana_streamer::streamer::{self, PacketBatchReceiver}, + solana_streamer::streamer::{self, StandardPacketReceiver}, std::{ collections::HashSet, net::UdpSocket, @@ -197,7 +197,7 @@ impl AncestorHashesService { /// Listen for responses to our ancestors hashes repair requests fn run_responses_listener( ancestor_hashes_request_statuses: Arc>, - response_receiver: PacketBatchReceiver, + response_receiver: StandardPacketReceiver, blockstore: Arc, outstanding_requests: Arc>, exit: Arc, @@ -240,7 +240,7 @@ impl AncestorHashesService { /// Process messages from the network fn process_new_packets_from_channel( ancestor_hashes_request_statuses: &DashMap, - response_receiver: &PacketBatchReceiver, + response_receiver: &StandardPacketReceiver, blockstore: &Blockstore, outstanding_requests: &RwLock, stats: &mut AncestorHashesResponsesStats, @@ -291,7 +291,7 @@ impl AncestorHashesService { fn process_packet_batch( ancestor_hashes_request_statuses: &DashMap, - packet_batch: PacketBatch, + packet_batch: StandardPacketBatch, stats: &mut AncestorHashesResponsesStats, outstanding_requests: &RwLock, blockstore: &Blockstore, diff --git a/core/src/banking_stage.rs b/core/src/banking_stage.rs index 343b13354bea18..a7bad65696f4b9 100644 --- a/core/src/banking_stage.rs +++ b/core/src/banking_stage.rs @@ -14,7 +14,10 @@ use { solana_perf::{ cuda_runtime::PinnedVec, data_budget::DataBudget, - packet::{limited_deserialize, Packet, PacketBatch, PACKETS_PER_BATCH}, + packet::{ + limited_deserialize, ExtendedPacketBatch, PacketBatch, StandardPacketBatch, + PACKETS_PER_BATCH, + }, perf_libs, }, solana_poh::poh_recorder::{BankStart, PohRecorder, PohRecorderError, TransactionRecorder}, @@ -36,13 +39,14 @@ use { }, feature_set, message::Message, + packet::PacketInterface, pubkey::Pubkey, short_vec::decode_shortu16_len, signature::Signature, timing::{duration_as_ms, timestamp, AtomicInterval}, transaction::{self, SanitizedTransaction, TransactionError, VersionedTransaction}, }, - solana_streamer::sendmmsg::{batch_send, SendPktsError}, + solana_streamer::sendmmsg::{batch_send, SendPktsError, batch_send_quic}, solana_transaction_status::token_balances::{ collect_token_balances, TransactionTokenBalancesSet, }, @@ -62,37 +66,49 @@ use { }; /// (packets, valid_indexes, forwarded) -/// Batch of packets with a list of which are valid and if this batch has been forwarded. -type PacketBatchAndOffsets = (PacketBatch, Vec, bool); +/// Set of packets with a list of which are valid and if this batch has been forwarded. +type PacketBatchAndOffsets

= (PacketBatch

, Vec, bool); -pub type UnprocessedPacketBatches = VecDeque; +pub type UnprocessedPacketBatches

= VecDeque>; /// Transaction forwarding pub const FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET: u64 = 2; pub const HOLD_TRANSACTIONS_SLOT_OFFSET: u64 = 20; // Fixed thread size seems to be fastest on GCP setup -pub const NUM_THREADS: u32 = 4; +pub const NUM_THREADS: u32 = 5; const TOTAL_BUFFERED_PACKETS: usize = 500_000; const MAX_NUM_TRANSACTIONS_PER_BATCH: usize = 128; const NUM_VOTE_PROCESSING_THREADS: u32 = 2; -const MIN_THREADS_BANKING: u32 = 1; +const MIN_THREADS_BANKING: u32 = 2; + +// TODO: Revisit values of NUM_THREADS and MIN_THREADS_BANKING; this changed bump each by 1 +// Once values picked, make sure they map correctly to cases in new_num_threads() #[derive(Debug, Default)] pub struct BankingStageStats { last_report: AtomicInterval, id: u32, receive_and_buffer_packets_count: AtomicUsize, + receive_and_buffer_extended_packets_count: AtomicUsize, dropped_packet_batches_count: AtomicUsize, + dropped_extended_packet_batches_count: AtomicUsize, + dropped_extended_packets_count: AtomicUsize, dropped_packets_count: AtomicUsize, pub(crate) dropped_duplicated_packets_count: AtomicUsize, + pub(crate) dropped_duplicated_extended_packets_count: AtomicUsize, newly_buffered_packets_count: AtomicUsize, + newly_buffered_extended_packets_count: AtomicUsize, current_buffered_packets_count: AtomicUsize, + current_buffered_extended_packets_count: AtomicUsize, + current_buffered_extended_packet_batches_count: AtomicUsize, current_buffered_packet_batches_count: AtomicUsize, + rebuffered_extended_packets_count: AtomicUsize, rebuffered_packets_count: AtomicUsize, + consumed_buffered_extended_packets_count: AtomicUsize, consumed_buffered_packets_count: AtomicUsize, // Timing @@ -123,6 +139,9 @@ impl BankingStageStats { + self .dropped_duplicated_packets_count .load(Ordering::Relaxed) as u64 + + self + .dropped_duplicated_extended_packets_count + .load(Ordering::Relaxed) as u64 + self.newly_buffered_packets_count.load(Ordering::Relaxed) as u64 + self.current_buffered_packets_count.load(Ordering::Relaxed) as u64 + self @@ -168,45 +187,93 @@ impl BankingStageStats { self.dropped_packet_batches_count.swap(0, Ordering::Relaxed) as i64, i64 ), + ( + "dropped_extended_packet_batches_count", + self.dropped_extended_packet_batches_count + .swap(0, Ordering::Relaxed) as i64, + i64 + ), ( "dropped_packets_count", self.dropped_packets_count.swap(0, Ordering::Relaxed) as i64, i64 ), + ( + "dropped_extended_packets_count", + self.dropped_extended_packets_count + .swap(0, Ordering::Relaxed) as i64, + i64 + ), ( "dropped_duplicated_packets_count", self.dropped_duplicated_packets_count .swap(0, Ordering::Relaxed) as i64, i64 ), + ( + "dropped_duplicated_extended_packets_count", + self.dropped_duplicated_extended_packets_count + .swap(0, Ordering::Relaxed) as i64, + i64 + ), ( "newly_buffered_packets_count", self.newly_buffered_packets_count.swap(0, Ordering::Relaxed) as i64, i64 ), + ( + "newly_buffered_extended_packets_count", + self.newly_buffered_extended_packets_count + .swap(0, Ordering::Relaxed) as i64, + i64 + ), ( "current_buffered_packet_batches_count", self.current_buffered_packet_batches_count .swap(0, Ordering::Relaxed) as i64, i64 ), + ( + "current_buffered_extended_packet_batches_count", + self.current_buffered_extended_packet_batches_count + .swap(0, Ordering::Relaxed) as i64, + i64 + ), ( "current_buffered_packets_count", self.current_buffered_packets_count .swap(0, Ordering::Relaxed) as i64, i64 ), + ( + "current_buffered_extended_packets_count", + self.current_buffered_extended_packets_count + .swap(0, Ordering::Relaxed) as i64, + i64 + ), ( "rebuffered_packets_count", self.rebuffered_packets_count.swap(0, Ordering::Relaxed) as i64, i64 ), + ( + "rebuffered_extended_packets_count", + self.rebuffered_extended_packets_count + .swap(0, Ordering::Relaxed) as i64, + i64 + ), ( "consumed_buffered_packets_count", self.consumed_buffered_packets_count .swap(0, Ordering::Relaxed) as i64, i64 ), + ( + "consumed_buffered_extended_packets_count", + self.consumed_buffered_extended_packets_count + .swap(0, Ordering::Relaxed) as i64, + i64 + ), ( "consume_buffered_packets_elapsed", self.consume_buffered_packets_elapsed @@ -285,9 +352,10 @@ impl BankingStage { pub fn new( cluster_info: &Arc, poh_recorder: &Arc>, - verified_receiver: CrossbeamReceiver>, - tpu_verified_vote_receiver: CrossbeamReceiver>, - verified_vote_receiver: CrossbeamReceiver>, + tpu_verified_receiver: CrossbeamReceiver>, + tpu_verified_extended_receiver: CrossbeamReceiver>, + tpu_verified_vote_receiver: CrossbeamReceiver>, + verified_vote_receiver: CrossbeamReceiver>, transaction_status_sender: Option, gossip_vote_sender: ReplayVoteSender, cost_model: Arc>, @@ -296,7 +364,8 @@ impl BankingStage { Self::new_num_threads( cluster_info, poh_recorder, - verified_receiver, + tpu_verified_receiver, + tpu_verified_extended_receiver, tpu_verified_vote_receiver, verified_vote_receiver, Self::num_threads(), @@ -311,9 +380,10 @@ impl BankingStage { fn new_num_threads( cluster_info: &Arc, poh_recorder: &Arc>, - verified_receiver: CrossbeamReceiver>, - tpu_verified_vote_receiver: CrossbeamReceiver>, - verified_vote_receiver: CrossbeamReceiver>, + tpu_verified_receiver: CrossbeamReceiver>, + tpu_verified_extended_receiver: CrossbeamReceiver>, + tpu_verified_vote_receiver: CrossbeamReceiver>, + verified_vote_receiver: CrossbeamReceiver>, num_threads: u32, transaction_status_sender: Option, gossip_vote_sender: ReplayVoteSender, @@ -329,19 +399,6 @@ impl BankingStage { assert!(num_threads >= NUM_VOTE_PROCESSING_THREADS + MIN_THREADS_BANKING); let bank_thread_hdls: Vec> = (0..num_threads) .map(|i| { - let (verified_receiver, forward_option) = match i { - 0 => { - // Disable forwarding of vote transactions - // from gossip. Note - votes can also arrive from tpu - (verified_vote_receiver.clone(), ForwardOption::NotForward) - } - 1 => ( - tpu_verified_vote_receiver.clone(), - ForwardOption::ForwardTpuVote, - ), - _ => (verified_receiver.clone(), ForwardOption::ForwardTransaction), - }; - let poh_recorder = poh_recorder.clone(); let cluster_info = cluster_info.clone(); let mut recv_start = Instant::now(); @@ -350,33 +407,107 @@ impl BankingStage { let packet_deduper = packet_deduper.clone(); let data_budget = data_budget.clone(); let cost_model = cost_model.clone(); - Builder::new() - .name("solana-banking-stage-tx".to_string()) - .spawn(move || { - Self::process_loop( - &verified_receiver, - &poh_recorder, - &cluster_info, - &mut recv_start, - forward_option, - i, - batch_limit, - transaction_status_sender, - gossip_vote_sender, - &packet_deduper, - &data_budget, - cost_model, - ); - }) - .unwrap() + + match i { + 0 => { + let verified_vote_receiver = verified_vote_receiver.clone(); + // Disable forwarding of vote transactions + // from gossip. Note - votes can also arrive from tpu + Builder::new() + .name("solana-banking-stage-tx".to_string()) + .spawn(move || { + Self::process_loop( + &verified_vote_receiver, + &poh_recorder, + &cluster_info, + &mut recv_start, + ForwardOption::NotForward, + i, + batch_limit, + transaction_status_sender, + gossip_vote_sender, + &packet_deduper, + &data_budget, + cost_model, + ); + }) + .unwrap() + } + 1 => { + let tpu_verified_vote_receiver = tpu_verified_vote_receiver.clone(); + Builder::new() + .name("solana-banking-stage-tx".to_string()) + .spawn(move || { + Self::process_loop( + &tpu_verified_vote_receiver, + &poh_recorder, + &cluster_info, + &mut recv_start, + ForwardOption::ForwardTpuVote, + i, + batch_limit, + transaction_status_sender, + gossip_vote_sender, + &packet_deduper, + &data_budget, + cost_model, + ); + }) + .unwrap() + } + 2 => { + let tpu_verified_extended_receiver = tpu_verified_extended_receiver.clone(); + Builder::new() + .name("solana-banking-stage-tx".to_string()) + .spawn(move || { + Self::process_loop( + &tpu_verified_extended_receiver, + &poh_recorder, + &cluster_info, + &mut recv_start, + ForwardOption::ForwardTransaction, + i, + batch_limit, + transaction_status_sender, + gossip_vote_sender, + &packet_deduper, + &data_budget, + cost_model, + ); + }) + .unwrap() + } + _ => { + let tpu_verified_receiver = tpu_verified_receiver.clone(); + Builder::new() + .name("solana-banking-stage-tx".to_string()) + .spawn(move || { + Self::process_loop( + &tpu_verified_receiver, + &poh_recorder, + &cluster_info, + &mut recv_start, + ForwardOption::ForwardTransaction, + i, + batch_limit, + transaction_status_sender, + gossip_vote_sender, + &packet_deduper, + &data_budget, + cost_model, + ); + }) + .unwrap() + } + } }) .collect(); Self { bank_thread_hdls } } - fn filter_valid_packets_for_forwarding<'a>( - packet_batches: impl Iterator, - ) -> Vec<&'a Packet> { + fn filter_valid_packets_for_forwarding<'a, PacketType: 'static + PacketInterface>( + packet_batches: impl Iterator>, + ) -> Vec<&'a PacketType> { packet_batches .filter(|(_batch, _indexes, forwarded)| !forwarded) .flat_map(|(batch, valid_indexes, _forwarded)| { @@ -385,10 +516,10 @@ impl BankingStage { .collect() } - fn forward_buffered_packets( + fn forward_buffered_packets( socket: &std::net::UdpSocket, tpu_forwards: &std::net::SocketAddr, - buffered_packet_batches: &UnprocessedPacketBatches, + buffered_packet_batches: &UnprocessedPacketBatches, data_budget: &DataBudget, ) -> std::io::Result<()> { let packets = Self::filter_valid_packets_for_forwarding(buffered_packet_batches.iter()); @@ -403,11 +534,29 @@ impl BankingStage { ) }); + if PacketType::is_extended() { + let packet_vec: Vec<&PacketType> = packets + .into_iter() + .filter(|p| { + !p.get_meta().forwarded && data_budget.take(p.get_meta().size) + }) + .collect(); + if !packet_vec.is_empty() { + inc_new_counter_info!("banking_stage-forwarded_packets", packet_vec.len()); + + // todo: handle this properly + match batch_send_quic(socket, packet_vec, tpu_forwards) { + Ok(()) => {} + _ => return Err(std::io::Error::new(std::io::ErrorKind::NotFound, "Send packet error")) + } + } + } + else { let packet_vec: Vec<_> = packets .iter() .filter_map(|p| { - if !p.meta.forwarded && data_budget.take(p.meta.size) { - Some((&p.data[..p.meta.size], tpu_forwards)) + if !p.get_meta().forwarded && data_budget.take(p.get_meta().size) { + Some((&p.get_data()[..p.get_meta().size], tpu_forwards)) } else { None } @@ -421,7 +570,7 @@ impl BankingStage { return Err(ioerr); } } - + } Ok(()) } @@ -440,11 +589,11 @@ impl BankingStage { } #[allow(clippy::too_many_arguments)] - pub fn consume_buffered_packets( + pub fn consume_buffered_packets( my_pubkey: &Pubkey, max_tx_ingestion_ns: u128, poh_recorder: &Arc>, - buffered_packet_batches: &mut UnprocessedPacketBatches, + buffered_packet_batches: &mut UnprocessedPacketBatches, transaction_status_sender: Option, gossip_vote_sender: &ReplayVoteSender, test_fn: Option, @@ -458,6 +607,11 @@ impl BankingStage { let mut proc_start = Measure::start("consume_buffered_process"); let mut reached_end_of_slot = None; + warn!( + "consume_buffered_packets::<{}>", + PacketType::get_packet_type_name() + ); + buffered_packet_batches.retain_mut(|buffered_packet_batch_and_offsets| { let (packet_batch, ref mut original_unprocessed_indexes, _forwarded) = buffered_packet_batch_and_offsets; @@ -543,15 +697,30 @@ impl BankingStage { (new_tx_count as f32) / (proc_start.as_s()) ); + //todo: it doesn't seem necessary to separate the timings + // for extended and normal packets, so we'll leave them mixed for now. + // in the future, possible add separate timings, and think about how meaningful + // a single "elapsed" packet processing time is, when standard and extended packets + // may be processed in parallel banking_stage_stats .consume_buffered_packets_elapsed .fetch_add(proc_start.as_us(), Ordering::Relaxed); - banking_stage_stats - .rebuffered_packets_count - .fetch_add(rebuffered_packet_count, Ordering::Relaxed); - banking_stage_stats - .consumed_buffered_packets_count - .fetch_add(new_tx_count, Ordering::Relaxed); + + if PacketType::is_extended() { + banking_stage_stats + .rebuffered_extended_packets_count + .fetch_add(rebuffered_packet_count, Ordering::Relaxed); + banking_stage_stats + .consumed_buffered_extended_packets_count + .fetch_add(new_tx_count, Ordering::Relaxed); + } else { + banking_stage_stats + .rebuffered_packets_count + .fetch_add(rebuffered_packet_count, Ordering::Relaxed); + banking_stage_stats + .consumed_buffered_packets_count + .fetch_add(new_tx_count, Ordering::Relaxed); + } } fn consume_or_forward_packets( @@ -588,12 +757,12 @@ impl BankingStage { } #[allow(clippy::too_many_arguments)] - fn process_buffered_packets( + fn process_buffered_packets( my_pubkey: &Pubkey, socket: &std::net::UdpSocket, poh_recorder: &Arc>, cluster_info: &ClusterInfo, - buffered_packet_batches: &mut UnprocessedPacketBatches, + buffered_packet_batches: &mut UnprocessedPacketBatches, forward_option: &ForwardOption, transaction_status_sender: Option, gossip_vote_sender: &ReplayVoteSender, @@ -671,10 +840,10 @@ impl BankingStage { decision } - fn handle_forwarding( + fn handle_forwarding( forward_option: &ForwardOption, cluster_info: &ClusterInfo, - buffered_packet_batches: &mut UnprocessedPacketBatches, + buffered_packet_batches: &mut UnprocessedPacketBatches, poh_recorder: &Arc>, socket: &UdpSocket, hold: bool, @@ -688,12 +857,23 @@ impl BankingStage { return; } ForwardOption::ForwardTransaction => { - next_leader_tpu_forwards(cluster_info, poh_recorder) + if PacketType::is_extended() { + next_leader_tpu_extended_forwards(cluster_info, poh_recorder) + } else { + next_leader_tpu_forwards(cluster_info, poh_recorder) + } } ForwardOption::ForwardTpuVote => next_leader_tpu_vote(cluster_info, poh_recorder), }; let addr = match addr { - Some(addr) => addr, + Some(addr) => { + warn!( + "Forwarding {} to {}", + PacketType::get_packet_type_name(), + addr + ); + addr + } None => return, }; let _ = Self::forward_buffered_packets(socket, &addr, buffered_packet_batches, data_budget); @@ -708,8 +888,8 @@ impl BankingStage { } #[allow(clippy::too_many_arguments)] - fn process_loop( - verified_receiver: &CrossbeamReceiver>, + fn process_loop( + verified_receiver: &CrossbeamReceiver>>, poh_recorder: &Arc>, cluster_info: &ClusterInfo, recv_start: &mut Instant, @@ -723,8 +903,10 @@ impl BankingStage { cost_model: Arc>, ) { let recorder = poh_recorder.lock().unwrap().recorder(); + // TODO: Double check that 0.0.0.0 will be fine here with mixing packet sizes let socket = UdpSocket::bind("0.0.0.0:0").unwrap(); - let mut buffered_packet_batches = VecDeque::with_capacity(batch_limit); + let mut buffered_packet_batches: UnprocessedPacketBatches = + VecDeque::with_capacity(batch_limit); let banking_stage_stats = BankingStageStats::new(id); let qos_service = QosService::new(cost_model, id); loop { @@ -1102,47 +1284,54 @@ impl BankingStage { } /// Read the transaction message from packet data - fn packet_message(packet: &Packet) -> Option<&[u8]> { - let (sig_len, sig_size) = decode_shortu16_len(&packet.data).ok()?; + fn packet_message(packet: &PacketType) -> Option<&[u8]> { + let (sig_len, sig_size) = decode_shortu16_len(packet.get_data()).ok()?; let msg_start = sig_len .checked_mul(size_of::()) .and_then(|v| v.checked_add(sig_size))?; - let msg_end = packet.meta.size; - Some(&packet.data[msg_start..msg_end]) + let msg_end = packet.get_meta().size; + Some(&packet.get_data()[msg_start..msg_end]) } // This function deserializes packets into transactions, computes the blake3 hash of transaction // messages, and verifies secp256k1 instructions. A list of sanitized transactions are returned // with their packet indexes. #[allow(clippy::needless_collect)] - fn transactions_from_packets( - packet_batch: &PacketBatch, + fn transactions_from_packets( + packet_batch: &PacketBatch, transaction_indexes: &[usize], feature_set: &Arc, votes_only: bool, ) -> (Vec, Vec) { - transaction_indexes + let ret = transaction_indexes .iter() .filter_map(|tx_index| { let p = &packet_batch.packets[*tx_index]; - if votes_only && !p.meta.is_simple_vote_tx { + if votes_only && !p.get_meta().is_simple_vote_tx { return None; } - let tx: VersionedTransaction = limited_deserialize(&p.data[0..p.meta.size]).ok()?; + let tx: VersionedTransaction = + limited_deserialize(&p.get_data()[0..p.get_meta().size]).ok()?; let message_bytes = Self::packet_message(p)?; let message_hash = Message::hash_raw_message(message_bytes); let tx = SanitizedTransaction::try_create( tx, message_hash, - Some(p.meta.is_simple_vote_tx), + Some(p.get_meta().is_simple_vote_tx), + None, |_| Err(TransactionError::UnsupportedVersion), ) .ok()?; tx.verify_precompiles(feature_set).ok()?; Some((tx, *tx_index)) }) - .unzip() + .unzip(); + warn!( + "transactions_from_packets::<{}> succeeded", + PacketType::get_packet_type_name() + ); + ret } /// This function filters pending packets that are still valid @@ -1185,11 +1374,11 @@ impl BankingStage { } #[allow(clippy::too_many_arguments)] - fn process_packets_transactions( + fn process_packets_transactions( bank: &Arc, bank_creation_time: &Instant, poh: &TransactionRecorder, - packet_batch: &PacketBatch, + packet_batch: &PacketBatch, packet_indexes: Vec, transaction_status_sender: Option, gossip_vote_sender: &ReplayVoteSender, @@ -1208,6 +1397,11 @@ impl BankingStage { let tx_len = transactions.len(); + warn!( + "process_packets_transactions::<{}> num transactions in batch: {}", + PacketType::get_packet_type_name(), + tx_len + ); let mut process_tx_time = Measure::start("process_tx_time"); let (processed, unprocessed_tx_indexes) = Self::process_transactions( bank, @@ -1252,9 +1446,9 @@ impl BankingStage { (processed, tx_len, filtered_unprocessed_packet_indexes) } - fn filter_unprocessed_packets( + fn filter_unprocessed_packets( bank: &Arc, - packet_batch: &PacketBatch, + packet_batch: &PacketBatch, transaction_indexes: &[usize], my_pubkey: &Pubkey, next_leader: Option, @@ -1303,30 +1497,30 @@ impl BankingStage { filtered_unprocessed_packet_indexes } - fn generate_packet_indexes(vers: &PinnedVec) -> Vec { + fn generate_packet_indexes( + vers: &PinnedVec, + ) -> Vec { vers.iter() .enumerate() - .filter_map( - |(index, ver)| { - if !ver.meta.discard { - Some(index) - } else { - None - } - }, - ) + .filter_map(|(index, ver)| { + if !ver.get_meta().discard { + Some(index) + } else { + None + } + }) .collect() } #[allow(clippy::too_many_arguments)] - /// Receive incoming packets, push into unprocessed buffer with packet indexes - fn receive_and_buffer_packets( - verified_receiver: &CrossbeamReceiver>, + /// Process the incoming packets + fn receive_and_buffer_packets( + verified_receiver: &CrossbeamReceiver>>, recv_start: &mut Instant, recv_timeout: Duration, id: u32, batch_limit: usize, - buffered_packet_batches: &mut UnprocessedPacketBatches, + buffered_packet_batches: &mut UnprocessedPacketBatches, banking_stage_stats: &BankingStageStats, packet_deduper: &PacketDeduper, ) -> Result<(), RecvTimeoutError> { @@ -1373,7 +1567,36 @@ impl BankingStage { packet_count, id, ); - banking_stage_stats + if PacketType::is_extended() { + banking_stage_stats + .receive_and_buffer_packets_elapsed + .fetch_add(proc_start.as_us(), Ordering::Relaxed); + banking_stage_stats + .receive_and_buffer_extended_packets_count + .fetch_add(packet_count, Ordering::Relaxed); + banking_stage_stats + .dropped_extended_packet_batches_count + .fetch_add(dropped_packet_batches_count, Ordering::Relaxed); + banking_stage_stats + .dropped_extended_packets_count + .fetch_add(dropped_packets_count, Ordering::Relaxed); + banking_stage_stats + .newly_buffered_extended_packets_count + .fetch_add(newly_buffered_packets_count, Ordering::Relaxed); + banking_stage_stats + .current_buffered_extended_packet_batches_count + .swap(buffered_packet_batches.len(), Ordering::Relaxed); + banking_stage_stats + .current_buffered_extended_packets_count + .swap( + buffered_packet_batches + .iter() + .map(|packets| packets.1.len()) + .sum(), + Ordering::Relaxed, + ); + } else { + banking_stage_stats .receive_and_buffer_packets_elapsed .fetch_add(proc_start.as_us(), Ordering::Relaxed); banking_stage_stats @@ -1398,13 +1621,14 @@ impl BankingStage { .sum(), Ordering::Relaxed, ); + } *recv_start = Instant::now(); Ok(()) } - fn push_unprocessed( - unprocessed_packet_batches: &mut UnprocessedPacketBatches, - packet_batch: PacketBatch, + fn push_unprocessed( + unprocessed_packet_batches: &mut UnprocessedPacketBatches, + packet_batch: PacketBatch, mut packet_indexes: Vec, dropped_packet_batches_count: &mut usize, dropped_packets_count: &mut usize, @@ -1452,6 +1676,15 @@ fn next_leader_tpu_forwards( next_leader_x(cluster_info, poh_recorder, |leader| leader.tpu_forwards) } +fn next_leader_tpu_extended_forwards( + cluster_info: &ClusterInfo, + poh_recorder: &Mutex, +) -> Option { + next_leader_x(cluster_info, poh_recorder, |leader| { + leader.tpu_extended_forwards + }) +} + pub(crate) fn next_leader_tpu_vote( cluster_info: &ClusterInfo, poh_recorder: &Mutex, diff --git a/core/src/cluster_info_vote_listener.rs b/core/src/cluster_info_vote_listener.rs index 9691895f382d19..f5cef82f107f65 100644 --- a/core/src/cluster_info_vote_listener.rs +++ b/core/src/cluster_info_vote_listener.rs @@ -21,7 +21,7 @@ use { solana_ledger::blockstore::Blockstore, solana_measure::measure::Measure, solana_metrics::inc_new_counter_debug, - solana_perf::packet::{self, PacketBatch}, + solana_perf::packet::{self, StandardPacketBatch, Packet}, solana_poh::poh_recorder::PohRecorder, solana_rpc::{ optimistically_confirmed_bank_tracker::{BankNotification, BankNotificationSender}, @@ -298,7 +298,7 @@ impl ClusterInfoVoteListener { pub fn new( exit: Arc, cluster_info: Arc, - verified_packets_sender: CrossbeamSender>, + verified_packets_sender: CrossbeamSender>, poh_recorder: Arc>, vote_tracker: Arc, bank_forks: Arc>, @@ -396,7 +396,7 @@ impl ClusterInfoVoteListener { votes: Vec, bank_forks: &RwLock, ) -> (Vec, Vec) { - let mut packet_batches = packet::to_packet_batches(&votes, 1); + let mut packet_batches = packet::to_packet_batches::<_, Packet>(&votes, 1); // Votes should already be filtered by this point. sigverify::ed25519_verify_cpu(&mut packet_batches, /*reject_non_vote=*/ false); @@ -437,7 +437,7 @@ impl ClusterInfoVoteListener { exit: Arc, verified_vote_label_packets_receiver: VerifiedLabelVotePacketsReceiver, poh_recorder: Arc>, - verified_packets_sender: &CrossbeamSender>, + verified_packets_sender: &CrossbeamSender>, ) -> Result<()> { let mut verified_vote_packets = VerifiedVotePackets::default(); let mut time_since_lock = Instant::now(); @@ -485,7 +485,7 @@ impl ClusterInfoVoteListener { fn check_for_leader_bank_and_send_votes( bank_vote_sender_state_option: &mut Option, current_working_bank: Arc, - verified_packets_sender: &CrossbeamSender>, + verified_packets_sender: &CrossbeamSender>, verified_vote_packets: &VerifiedVotePackets, ) -> Result<()> { // We will take this lock at most once every `BANK_SEND_VOTES_LOOP_SLEEP_MS` diff --git a/core/src/fetch_stage.rs b/core/src/fetch_stage.rs index 1e78a0ddc447b8..5258b86a609d87 100644 --- a/core/src/fetch_stage.rs +++ b/core/src/fetch_stage.rs @@ -8,7 +8,10 @@ use { solana_metrics::{inc_new_counter_debug, inc_new_counter_info}, solana_perf::{packet::PacketBatchRecycler, recycler::Recycler}, solana_poh::poh_recorder::PohRecorder, - solana_sdk::{clock::DEFAULT_TICKS_PER_SLOT, packet::Packet}, + solana_sdk::{ + clock::DEFAULT_TICKS_PER_SLOT, + packet::{ExtendedPacket, Packet, PacketInterface}, + }, solana_streamer::streamer::{self, PacketBatchReceiver, PacketBatchSender}, std::{ net::UdpSocket, @@ -25,73 +28,120 @@ pub struct FetchStage { thread_hdls: Vec>, } + impl FetchStage { #[allow(clippy::new_ret_no_self)] pub fn new( - sockets: Vec, - tpu_forwards_sockets: Vec, + tpu_tx_sockets: Vec, + tpu_tx_forwards_sockets: Vec, + tpu_tx_extended_sockets: Vec, + tpu_tx_extended_forwards_sockets: Vec, tpu_vote_sockets: Vec, exit: &Arc, poh_recorder: &Arc>, coalesce_ms: u64, - ) -> (Self, PacketBatchReceiver, PacketBatchReceiver) { - let (sender, receiver) = channel(); + ) -> ( + Self, + PacketBatchReceiver, + PacketBatchReceiver, + PacketBatchReceiver, + ) { + let (tx_sender, tx_receiver) = channel(); + let (extended_sender, extended_receiver) = channel(); let (vote_sender, vote_receiver) = channel(); ( Self::new_with_sender( - sockets, - tpu_forwards_sockets, + tpu_tx_sockets, + tpu_tx_forwards_sockets, + tpu_tx_extended_sockets, + tpu_tx_extended_forwards_sockets, tpu_vote_sockets, exit, - &sender, + &tx_sender, + &extended_sender, &vote_sender, poh_recorder, coalesce_ms, ), - receiver, + tx_receiver, + extended_receiver, vote_receiver, ) } + #[allow(clippy::too_many_arguments)] pub fn new_with_sender( - sockets: Vec, - tpu_forwards_sockets: Vec, + tpu_tx_sockets: Vec, + tpu_tx_forwards_sockets: Vec, + tpu_tx_extended_sockets: Vec, + tpu_tx_extended_forwards_sockets: Vec, tpu_vote_sockets: Vec, exit: &Arc, - sender: &PacketBatchSender, - vote_sender: &PacketBatchSender, + tx_sender: &PacketBatchSender, + tx_extended_sender: &PacketBatchSender, + vote_sender: &PacketBatchSender, poh_recorder: &Arc>, coalesce_ms: u64, ) -> Self { - let tx_sockets = sockets.into_iter().map(Arc::new).collect(); - let tpu_forwards_sockets = tpu_forwards_sockets.into_iter().map(Arc::new).collect(); + let tpu_tx_sockets = tpu_tx_sockets.into_iter().map(Arc::new).collect(); + let tpu_tx_forwards_sockets = tpu_tx_forwards_sockets.into_iter().map(Arc::new).collect(); + let tpu_tx_extended_sockets = tpu_tx_extended_sockets.into_iter().map(Arc::new).collect(); + let tpu_tx_extended_forwards_sockets = tpu_tx_extended_forwards_sockets + .into_iter() + .map(Arc::new) + .collect(); let tpu_vote_sockets = tpu_vote_sockets.into_iter().map(Arc::new).collect(); + Self::new_multi_socket( - tx_sockets, - tpu_forwards_sockets, + tpu_tx_sockets, + tpu_tx_forwards_sockets, + tpu_tx_extended_sockets, + tpu_tx_extended_forwards_sockets, tpu_vote_sockets, exit, - sender, + tx_sender, + tx_extended_sender, vote_sender, poh_recorder, coalesce_ms, ) } - fn handle_forwarded_packets( - recvr: &PacketBatchReceiver, - sendr: &PacketBatchSender, + fn new_handle_forwarded_packets_thread( + receiver: PacketBatchReceiver

, + sender: PacketBatchSender

, + poh_recorder: Arc>, + ) -> JoinHandle<()> { + Builder::new() + .name("solana-fetch-stage-fwd-rcvr".to_string()) + .spawn(move || loop { + if let Err(e) = Self::handle_forwarded_packets(&receiver, &sender, &poh_recorder) { + match e { + Error::RecvTimeout(RecvTimeoutError::Disconnected) => break, + Error::RecvTimeout(RecvTimeoutError::Timeout) => (), + Error::Recv(_) => break, + Error::Send => break, + _ => error!("{:?}", e), + } + } + }) + .unwrap() + } + + fn handle_forwarded_packets( + receiver: &PacketBatchReceiver

, + sender: &PacketBatchSender

, poh_recorder: &Arc>, ) -> Result<()> { - let mark_forwarded = |packet: &mut Packet| { - packet.meta.forwarded = true; + let mark_forwarded = |packet: &mut P| { + packet.get_meta_mut().forwarded = true; }; - let mut packet_batch = recvr.recv()?; + let mut packet_batch = receiver.recv()?; let mut num_packets = packet_batch.packets.len(); packet_batch.packets.iter_mut().for_each(mark_forwarded); let mut packet_batches = vec![packet_batch]; - while let Ok(mut packet_batch) = recvr.try_recv() { + while let Ok(mut packet_batch) = receiver.try_recv() { packet_batch.packets.iter_mut().for_each(mark_forwarded); num_packets += packet_batch.packets.len(); packet_batches.push(packet_batch); @@ -109,7 +159,7 @@ impl FetchStage { inc_new_counter_debug!("fetch_stage-honor_forwards", num_packets); for packet_batch in packet_batches { #[allow(clippy::question_mark)] - if sendr.send(packet_batch).is_err() { + if sender.send(packet_batch).is_err() { return Err(Error::Send); } } @@ -120,23 +170,28 @@ impl FetchStage { Ok(()) } + #[allow(clippy::too_many_arguments)] fn new_multi_socket( tpu_sockets: Vec>, tpu_forwards_sockets: Vec>, + tpu_tx_extended_sockets: Vec>, + tpu_tx_extended_forwards_sockets: Vec>, tpu_vote_sockets: Vec>, exit: &Arc, - sender: &PacketBatchSender, - vote_sender: &PacketBatchSender, + tx_sender: &PacketBatchSender, + tx_extended_sender: &PacketBatchSender, + vote_sender: &PacketBatchSender, poh_recorder: &Arc>, coalesce_ms: u64, ) -> Self { - let recycler: PacketBatchRecycler = Recycler::warmed(1000, 1024); + let recycler: PacketBatchRecycler = Recycler::warmed(1000, 1024); + let extended_recycler: PacketBatchRecycler = Recycler::warmed(1000, 1024); let tpu_threads = tpu_sockets.into_iter().map(|socket| { streamer::receiver( socket, exit, - sender.clone(), + tx_sender.clone(), recycler.clone(), "fetch_stage", coalesce_ms, @@ -157,6 +212,32 @@ impl FetchStage { ) }); + let tpu_extended_threads = tpu_tx_extended_sockets.into_iter().map(|socket| { + streamer::receiver( + socket, + exit, + tx_extended_sender.clone(), + extended_recycler.clone(), + "fetch_extended_stage", + coalesce_ms, + true, + ) + }); + + let (extended_forward_sender, extended_forward_receiver) = channel(); + let tpu_extended_forwards_threads = + tpu_tx_extended_forwards_sockets.into_iter().map(|socket| { + streamer::receiver( + socket, + exit, + extended_forward_sender.clone(), + extended_recycler.clone(), + "fetch_forward_extended_stage", + coalesce_ms, + true, + ) + }); + let tpu_vote_threads = tpu_vote_sockets.into_iter().map(|socket| { streamer::receiver( socket, @@ -169,31 +250,27 @@ impl FetchStage { ) }); - let sender = sender.clone(); - let poh_recorder = poh_recorder.clone(); + let forward_thread_hdl = Self::new_handle_forwarded_packets_thread( + forward_receiver, + tx_sender.clone(), + poh_recorder.clone(), + ); - let fwd_thread_hdl = Builder::new() - .name("solana-fetch-stage-fwd-rcvr".to_string()) - .spawn(move || loop { - if let Err(e) = - Self::handle_forwarded_packets(&forward_receiver, &sender, &poh_recorder) - { - match e { - Error::RecvTimeout(RecvTimeoutError::Disconnected) => break, - Error::RecvTimeout(RecvTimeoutError::Timeout) => (), - Error::Recv(_) => break, - Error::Send => break, - _ => error!("{:?}", e), - } - } - }) - .unwrap(); + let forward_extended_thread_hdl = Self::new_handle_forwarded_packets_thread( + extended_forward_receiver, + tx_extended_sender.clone(), + poh_recorder.clone(), + ); let mut thread_hdls: Vec<_> = tpu_threads .chain(tpu_forwards_threads) .chain(tpu_vote_threads) + .chain(tpu_extended_threads) + .chain(tpu_extended_forwards_threads) .collect(); - thread_hdls.push(fwd_thread_hdl); + thread_hdls.push(forward_thread_hdl); + thread_hdls.push(forward_extended_thread_hdl); + Self { thread_hdls } } diff --git a/core/src/packet_deduper.rs b/core/src/packet_deduper.rs index 24c3aea10e5f23..c78239276b130c 100644 --- a/core/src/packet_deduper.rs +++ b/core/src/packet_deduper.rs @@ -7,6 +7,7 @@ use { ops::DerefMut, sync::{atomic::Ordering, Arc, Mutex}, }, + solana_sdk::packet::PacketInterface, }; const DEFAULT_LRU_SIZE: usize = 200_000; @@ -24,9 +25,9 @@ impl Default for PacketDeduper { } impl PacketDeduper { - pub fn dedupe_packets( + pub fn dedupe_packets( &self, - packet_batch: &PacketBatch, + packet_batch: &PacketBatch, packet_indexes: &mut Vec, banking_stage_stats: &BankingStageStats, ) { @@ -48,12 +49,22 @@ impl PacketDeduper { banking_stage_stats .packet_duplicate_check_elapsed .fetch_add(packet_duplicate_check_time.as_us(), Ordering::Relaxed); + if PacketType::is_extended() { + banking_stage_stats + .dropped_duplicated_extended_packets_count + .fetch_add( + original_packets_count.saturating_sub(packet_indexes.len()), + Ordering::Relaxed, + ); + } + else { banking_stage_stats .dropped_duplicated_packets_count .fetch_add( original_packets_count.saturating_sub(packet_indexes.len()), Ordering::Relaxed, ); + } } pub fn reset(&self) { diff --git a/core/src/packet_hasher.rs b/core/src/packet_hasher.rs index 575c9733fdb8f5..29a217866e7f3a 100644 --- a/core/src/packet_hasher.rs +++ b/core/src/packet_hasher.rs @@ -5,7 +5,7 @@ use { ahash::AHasher, rand::{thread_rng, Rng}, solana_ledger::shred::Shred, - solana_perf::packet::Packet, + solana_perf::packet::PacketInterface, std::hash::Hasher, }; @@ -25,9 +25,9 @@ impl Default for PacketHasher { } impl PacketHasher { - pub(crate) fn hash_packet(&self, packet: &Packet) -> u64 { - let size = packet.data.len().min(packet.meta.size); - self.hash_data(&packet.data[..size]) + pub(crate) fn hash_packet(&self, packet: &P) -> u64 { + let size = packet.get_data().len().min(packet.get_meta().size); + self.hash_data(&packet.get_data()[..size]) } pub(crate) fn hash_shred(&self, shred: &Shred) -> u64 { diff --git a/core/src/retransmit_stage.rs b/core/src/retransmit_stage.rs index 3b5e03045b2525..96243f36a7aeb6 100644 --- a/core/src/retransmit_stage.rs +++ b/core/src/retransmit_stage.rs @@ -27,7 +27,7 @@ use { shred::{Shred, ShredId}, }, solana_measure::measure::Measure, - solana_perf::packet::PacketBatch, + solana_perf::packet::StandardPacketBatch, solana_rayon_threadlimit::get_thread_count, solana_rpc::{max_slots::MaxSlots, rpc_subscriptions::RpcSubscriptions}, solana_runtime::{bank::Bank, bank_forks::BankForks}, @@ -433,8 +433,8 @@ impl RetransmitStage { cluster_info: Arc, retransmit_sockets: Arc>, repair_socket: Arc, - ancestor_hashes_socket: Arc, - verified_receiver: Receiver>, + ancestor_hashes_socket: Arc, + verified_receiver: Receiver>, exit: Arc, cluster_slots_update_receiver: ClusterSlotsUpdateReceiver, epoch_schedule: EpochSchedule, diff --git a/core/src/serve_repair.rs b/core/src/serve_repair.rs index f643302737448a..6b23b2c43f9206 100644 --- a/core/src/serve_repair.rs +++ b/core/src/serve_repair.rs @@ -25,11 +25,11 @@ use { }, solana_measure::measure::Measure, solana_metrics::inc_new_counter_debug, - solana_perf::packet::{limited_deserialize, PacketBatch, PacketBatchRecycler}, + solana_perf::packet::{limited_deserialize, StandardPacketBatch, StandardPacketBatchRecycler}, solana_sdk::{ clock::Slot, hash::Hash, packet::PACKET_DATA_SIZE, pubkey::Pubkey, timing::duration_as_ms, }, - solana_streamer::streamer::{PacketBatchReceiver, PacketBatchSender}, + solana_streamer::streamer::{StandardPacketReceiver, StandardPacketSender}, std::{ collections::HashSet, net::SocketAddr, @@ -229,12 +229,12 @@ impl ServeRepair { fn handle_repair( me: &Arc>, - recycler: &PacketBatchRecycler, + recycler: &StandardPacketBatchRecycler, from_addr: &SocketAddr, blockstore: Option<&Arc>, request: RepairProtocol, stats: &mut ServeRepairStats, - ) -> Option { + ) -> Option { let now = Instant::now(); let my_id = me.read().unwrap().my_id(); @@ -317,10 +317,10 @@ impl ServeRepair { /// Process messages from the network fn run_listen( obj: &Arc>, - recycler: &PacketBatchRecycler, + recycler: &StandardPacketBatchRecycler, blockstore: Option<&Arc>, - requests_receiver: &PacketBatchReceiver, - response_sender: &PacketBatchSender, + requests_receiver: &StandardPacketReceiver, + response_sender: &StandardPacketSender, stats: &mut ServeRepairStats, max_packets: &mut usize, ) -> Result<()> { @@ -392,12 +392,12 @@ impl ServeRepair { pub fn listen( me: Arc>, blockstore: Option>, - requests_receiver: PacketBatchReceiver, - response_sender: PacketBatchSender, + requests_receiver: StandardPacketReceiver, + response_sender: StandardPacketSender, exit: &Arc, ) -> JoinHandle<()> { let exit = exit.clone(); - let recycler = PacketBatchRecycler::default(); + let recycler = StandardPacketBatchRecycler::default(); Builder::new() .name("solana-repair-listen".to_string()) .spawn(move || { @@ -432,10 +432,10 @@ impl ServeRepair { fn handle_packets( me: &Arc>, - recycler: &PacketBatchRecycler, + recycler: &StandardPacketBatchRecycler, blockstore: Option<&Arc>, - packet_batch: PacketBatch, - response_sender: &PacketBatchSender, + packet_batch: StandardPacketBatch, + response_sender: &StandardPacketSender, stats: &mut ServeRepairStats, ) { // iter over the packets @@ -609,7 +609,7 @@ impl ServeRepair { } fn run_window_request( - recycler: &PacketBatchRecycler, + recycler: &StandardPacketBatchRecycler, from: &ContactInfo, from_addr: &SocketAddr, blockstore: Option<&Arc>, @@ -617,7 +617,7 @@ impl ServeRepair { slot: Slot, shred_index: u64, nonce: Nonce, - ) -> Option { + ) -> Option { if let Some(blockstore) = blockstore { // Try to find the requested index in one of the slots let packet = repair_response::repair_response_packet( @@ -630,7 +630,7 @@ impl ServeRepair { if let Some(packet) = packet { inc_new_counter_debug!("serve_repair-window-request-ledger", 1); - return Some(PacketBatch::new_unpinned_with_recycler_data( + return Some(StandardPacketBatch::new_unpinned_with_recycler_data( recycler, "run_window_request", vec![packet], @@ -651,13 +651,13 @@ impl ServeRepair { } fn run_highest_window_request( - recycler: &PacketBatchRecycler, + recycler: &StandardPacketBatchRecycler, from_addr: &SocketAddr, blockstore: Option<&Arc>, slot: Slot, highest_index: u64, nonce: Nonce, - ) -> Option { + ) -> Option { let blockstore = blockstore?; // Try to find the requested index in one of the slots let meta = blockstore.meta(slot).ok()??; @@ -670,7 +670,7 @@ impl ServeRepair { from_addr, nonce, )?; - return Some(PacketBatch::new_unpinned_with_recycler_data( + return Some(StandardPacketBatch::new_unpinned_with_recycler_data( recycler, "run_highest_window_request", vec![packet], @@ -680,14 +680,15 @@ impl ServeRepair { } fn run_orphan( - recycler: &PacketBatchRecycler, + recycler: &StandardPacketBatchRecycler, from_addr: &SocketAddr, blockstore: Option<&Arc>, mut slot: Slot, max_responses: usize, nonce: Nonce, - ) -> Option { - let mut res = PacketBatch::new_unpinned_with_recycler(recycler.clone(), 64, "run_orphan"); + ) -> Option { + let mut res = + StandardPacketBatch::new_unpinned_with_recycler(recycler.clone(), 64, "run_orphan"); if let Some(blockstore) = blockstore { // Try to find the next "n" parent slots of the input slot while let Ok(Some(meta)) = blockstore.meta(slot) { @@ -720,12 +721,12 @@ impl ServeRepair { } fn run_ancestor_hashes( - recycler: &PacketBatchRecycler, + recycler: &StandardPacketBatchRecycler, from_addr: &SocketAddr, blockstore: Option<&Arc>, slot: Slot, nonce: Nonce, - ) -> Option { + ) -> Option { let blockstore = blockstore?; let ancestor_slot_hashes = if blockstore.is_duplicate_confirmed(slot) { let ancestor_iterator = @@ -746,7 +747,7 @@ impl ServeRepair { from_addr, nonce, )?; - Some(PacketBatch::new_unpinned_with_recycler_data( + Some(StandardPacketBatch::new_unpinned_with_recycler_data( recycler, "run_ancestor_hashes", vec![packet], diff --git a/core/src/shred_fetch_stage.rs b/core/src/shred_fetch_stage.rs index 787ae5c07cf18f..90fe0be5d25582 100644 --- a/core/src/shred_fetch_stage.rs +++ b/core/src/shred_fetch_stage.rs @@ -11,7 +11,7 @@ use { }, solana_runtime::bank_forks::BankForks, solana_sdk::clock::{Slot, DEFAULT_MS_PER_SLOT}, - solana_streamer::streamer::{self, PacketBatchReceiver, PacketBatchSender}, + solana_streamer::streamer::{self, StandardPacketReceiver, StandardPacketSender}, std::{ net::UdpSocket, sync::{atomic::AtomicBool, mpsc::channel, Arc, RwLock}, @@ -63,8 +63,8 @@ impl ShredFetchStage { // updates packets received on a channel and sends them on another channel fn modify_packets( - recvr: PacketBatchReceiver, - sendr: PacketBatchSender, + recvr: StandardPacketReceiver, + sendr: StandardPacketSender, bank_forks: Option>>, name: &'static str, modify: F, @@ -133,7 +133,7 @@ impl ShredFetchStage { fn packet_modifier( sockets: Vec>, exit: &Arc, - sender: PacketBatchSender, + sender: StandardPacketSender, recycler: Recycler>, bank_forks: Option>>, name: &'static str, @@ -169,11 +169,11 @@ impl ShredFetchStage { sockets: Vec>, forward_sockets: Vec>, repair_socket: Arc, - sender: &PacketBatchSender, + sender: &StandardPacketSender, bank_forks: Option>>, exit: &Arc, ) -> Self { - let recycler: PacketBatchRecycler = Recycler::warmed(100, 1024); + let recycler: PacketBatchRecycler = Recycler::warmed(100, 1024); let (mut tvu_threads, tvu_filter) = Self::packet_modifier( sockets, diff --git a/core/src/sigverify.rs b/core/src/sigverify.rs index 74dbf5bdfc80ff..4b6175518bc7d1 100644 --- a/core/src/sigverify.rs +++ b/core/src/sigverify.rs @@ -10,16 +10,19 @@ pub use solana_perf::sigverify::{ use { crate::sigverify_stage::SigVerifier, solana_perf::{cuda_runtime::PinnedVec, packet::PacketBatch, recycler::Recycler, sigverify}, + solana_sdk::packet::PacketInterface, + std::marker::PhantomData, }; #[derive(Clone)] -pub struct TransactionSigVerifier { +pub struct TransactionSigVerifier { recycler: Recycler, recycler_out: Recycler>, reject_non_vote: bool, + _phantom: PhantomData

, } -impl TransactionSigVerifier { +impl TransactionSigVerifier

{ pub fn new_reject_non_vote() -> Self { TransactionSigVerifier { reject_non_vote: true, @@ -28,19 +31,20 @@ impl TransactionSigVerifier { } } -impl Default for TransactionSigVerifier { +impl Default for TransactionSigVerifier

{ fn default() -> Self { init(); Self { recycler: Recycler::warmed(50, 4096), recycler_out: Recycler::warmed(50, 4096), reject_non_vote: false, + _phantom: PhantomData::default(), } } } -impl SigVerifier for TransactionSigVerifier { - fn verify_batches(&self, mut batches: Vec) -> Vec { +impl SigVerifier

for TransactionSigVerifier

{ + fn verify_batches(&self, mut batches: Vec>) -> Vec> { sigverify::ed25519_verify( &mut batches, &self.recycler, diff --git a/core/src/sigverify_shreds.rs b/core/src/sigverify_shreds.rs index 85078f510f3c6a..c4795e1ba23754 100644 --- a/core/src/sigverify_shreds.rs +++ b/core/src/sigverify_shreds.rs @@ -5,8 +5,9 @@ use { leader_schedule_cache::LeaderScheduleCache, shred::Shred, sigverify_shreds::verify_shreds_gpu, }, - solana_perf::{self, packet::PacketBatch, recycler_cache::RecyclerCache}, + solana_perf::{self, packet::StandardPacketBatch, recycler_cache::RecyclerCache}, solana_runtime::bank_forks::BankForks, + solana_sdk::packet::Packet, std::{ collections::{HashMap, HashSet}, sync::{Arc, RwLock}, @@ -32,7 +33,8 @@ impl ShredSigVerifier { recycler_cache: RecyclerCache::warmed(), } } - fn read_slots(batches: &[PacketBatch]) -> HashSet { + + fn read_slots(batches: &[StandardPacketBatch]) -> HashSet { batches .iter() .flat_map(|batch| batch.packets.iter().filter_map(Shred::get_slot_from_packet)) @@ -40,8 +42,8 @@ impl ShredSigVerifier { } } -impl SigVerifier for ShredSigVerifier { - fn verify_batches(&self, mut batches: Vec) -> Vec { +impl SigVerifier for ShredSigVerifier { + fn verify_batches(&self, mut batches: Vec) -> Vec { let r_bank = self.bank_forks.read().unwrap().working_bank(); let slots: HashSet = Self::read_slots(&batches); let mut leader_slots: HashMap = slots diff --git a/core/src/sigverify_stage.rs b/core/src/sigverify_stage.rs index 9b63ba2b83c7b4..9bf8a7024441d2 100644 --- a/core/src/sigverify_stage.rs +++ b/core/src/sigverify_stage.rs @@ -9,12 +9,13 @@ use { crate::sigverify, crossbeam_channel::{SendError, Sender as CrossbeamSender}, solana_measure::measure::Measure, - solana_perf::packet::PacketBatch, + solana_perf::packet::{PacketBatch, StandardPacketBatch}, + solana_sdk::packet::{Packet, PacketInterface}, solana_sdk::timing, solana_streamer::streamer::{self, PacketBatchReceiver, StreamerError}, std::{ collections::HashMap, - sync::mpsc::{Receiver, RecvTimeoutError}, + sync::mpsc::RecvTimeoutError, thread::{self, Builder, JoinHandle}, time::Instant, }, @@ -24,22 +25,22 @@ use { const MAX_SIGVERIFY_BATCH: usize = 10_000; #[derive(Error, Debug)] -pub enum SigVerifyServiceError { +pub enum SigVerifyServiceError { #[error("send packets batch error")] - Send(#[from] SendError>), + Send(#[from] SendError>>), #[error("streamer error")] - Streamer(#[from] StreamerError), + Streamer(#[from] StreamerError

), } -type Result = std::result::Result; +type Result = std::result::Result>; pub struct SigVerifyStage { thread_hdl: JoinHandle<()>, } -pub trait SigVerifier { - fn verify_batches(&self, batches: Vec) -> Vec; +pub trait SigVerifier { + fn verify_batches(&self, batches: Vec>) -> Vec>; } #[derive(Default, Clone)] @@ -121,8 +122,8 @@ impl SigVerifierStats { } } -impl SigVerifier for DisabledSigVerifier { - fn verify_batches(&self, mut batches: Vec) -> Vec { +impl SigVerifier for DisabledSigVerifier { + fn verify_batches(&self, mut batches: Vec) -> Vec { sigverify::ed25519_verify_disabled(&mut batches); batches } @@ -130,21 +131,24 @@ impl SigVerifier for DisabledSigVerifier { impl SigVerifyStage { #[allow(clippy::new_ret_no_self)] - pub fn new( - packet_receiver: Receiver, - verified_sender: CrossbeamSender>, + pub fn new + 'static + Send + Clone>( + packet_receiver: PacketBatchReceiver

, + verified_sender: CrossbeamSender>>, verifier: T, ) -> Self { let thread_hdl = Self::verifier_services(packet_receiver, verified_sender, verifier); Self { thread_hdl } } - pub fn discard_excess_packets(batches: &mut Vec, max_packets: usize) { + pub fn discard_excess_packets( + batches: &mut Vec>, + max_packets: usize, + ) { let mut received_ips = HashMap::new(); for (batch_index, batch) in batches.iter().enumerate() { for (packet_index, packets) in batch.packets.iter().enumerate() { let e = received_ips - .entry(packets.meta.addr().ip()) + .entry(packets.get_meta().addr().ip()) .or_insert_with(Vec::new); e.push((batch_index, packet_index)); } @@ -163,17 +167,19 @@ impl SigVerifyStage { } for (_addr, indexes) in received_ips { for (batch_index, packet_index) in indexes { - batches[batch_index].packets[packet_index].meta.discard = true; + batches[batch_index].packets[packet_index] + .get_meta_mut() + .discard = true; } } } - fn verifier( - recvr: &PacketBatchReceiver, - sendr: &CrossbeamSender>, + fn verifier + 'static>( + recvr: &PacketBatchReceiver

, + sendr: &CrossbeamSender>>, verifier: &T, stats: &mut SigVerifierStats, - ) -> Result<()> { + ) -> Result<(), P> { let (mut batches, num_packets, recv_duration) = streamer::recv_packet_batches(recvr)?; let batches_len = batches.len(); @@ -187,7 +193,29 @@ impl SigVerifyStage { } let mut verify_batch_time = Measure::start("sigverify_batch_time"); - sendr.send(verifier.verify_batches(batches))?; + + let verified_packets = verifier.verify_batches(batches); + warn!( + "verifier::<{}>: num packets that failed sigverify: {}, num packets: {}", + P::get_packet_type_name(), + verified_packets + .iter() + .map(|packets| -> usize { + packets + .packets + .iter() + .map(|packet| if packet.get_meta().discard { 1 } else { 0 }) + .sum() + }) + .sum::(), + verified_packets + .iter() + .map(|packets| -> usize { + packets.packets.len() + }) + .sum::() + ); + sendr.send(verified_packets)?; verify_batch_time.stop(); debug!( @@ -215,9 +243,12 @@ impl SigVerifyStage { Ok(()) } - fn verifier_service( - packet_receiver: PacketBatchReceiver, - verified_sender: CrossbeamSender>, + fn verifier_service< + P: PacketInterface + 'static, + T: SigVerifier

+ 'static + Send + Clone, + >( + packet_receiver: PacketBatchReceiver

, + verified_sender: CrossbeamSender>>, verifier: &T, ) -> JoinHandle<()> { let verifier = verifier.clone(); @@ -251,9 +282,12 @@ impl SigVerifyStage { .unwrap() } - fn verifier_services( - packet_receiver: PacketBatchReceiver, - verified_sender: CrossbeamSender>, + fn verifier_services< + P: PacketInterface + 'static, + T: SigVerifier

+ 'static + Send + Clone, + >( + packet_receiver: PacketBatchReceiver

, + verified_sender: CrossbeamSender>>, verifier: T, ) -> JoinHandle<()> { Self::verifier_service(packet_receiver, verified_sender, &verifier) diff --git a/core/src/tpu.rs b/core/src/tpu.rs index f34c4e6cf3cd2a..3889e73d33a521 100644 --- a/core/src/tpu.rs +++ b/core/src/tpu.rs @@ -17,6 +17,7 @@ use { crossbeam_channel::unbounded, solana_gossip::cluster_info::ClusterInfo, solana_ledger::{blockstore::Blockstore, blockstore_processor::TransactionStatusSender}, + solana_perf::packet::{ExtendedPacketBatch, StandardPacketBatch}, solana_poh::poh_recorder::{PohRecorder, WorkingBankEntry}, solana_rpc::{ optimistically_confirmed_bank_tracker::BankNotificationSender, @@ -27,6 +28,7 @@ use { cost_model::CostModel, vote_sender_types::{ReplayVoteReceiver, ReplayVoteSender}, }, + solana_sdk::packet::{ExtendedPacket, Packet}, std::{ net::UdpSocket, sync::{ @@ -42,7 +44,8 @@ pub const DEFAULT_TPU_COALESCE_MS: u64 = 5; pub struct Tpu { fetch_stage: FetchStage, - sigverify_stage: SigVerifyStage, + tx_sigverify_stage: SigVerifyStage, + tx_extended_sigverify_stage: SigVerifyStage, vote_sigverify_stage: SigVerifyStage, banking_stage: BankingStage, cluster_info_vote_listener: ClusterInfoVoteListener, @@ -56,8 +59,10 @@ impl Tpu { poh_recorder: &Arc>, entry_receiver: Receiver, retransmit_slots_receiver: RetransmitSlotsReceiver, - transactions_sockets: Vec, - tpu_forwards_sockets: Vec, + tpu_tx_sockets: Vec, + tpu_tx_forwards_sockets: Vec, + tpu_tx_extended_sockets: Vec, + tpu_tx_extended_forwards_sockets: Vec, tpu_vote_sockets: Vec, broadcast_sockets: Vec, subscriptions: &Arc, @@ -78,28 +83,40 @@ impl Tpu { cost_model: &Arc>, ) -> Self { let (packet_sender, packet_receiver) = channel(); + let (extended_sender, extended_receiver) = channel(); let (vote_packet_sender, vote_packet_receiver) = channel(); - let fetch_stage = FetchStage::new_with_sender( - transactions_sockets, - tpu_forwards_sockets, + + let fetch_stage: FetchStage = FetchStage::new_with_sender( + tpu_tx_sockets, + tpu_tx_forwards_sockets, + tpu_tx_extended_sockets, + tpu_tx_extended_forwards_sockets, tpu_vote_sockets, exit, &packet_sender, + &extended_sender, &vote_packet_sender, poh_recorder, tpu_coalesce_ms, ); - let (verified_sender, verified_receiver) = unbounded(); - let sigverify_stage = { - let verifier = TransactionSigVerifier::default(); + let (verified_sender, verified_receiver) = unbounded::>(); + let tx_sigverify_stage = { + let verifier = TransactionSigVerifier::::default(); SigVerifyStage::new(packet_receiver, verified_sender, verifier) }; - let (verified_tpu_vote_packets_sender, verified_tpu_vote_packets_receiver) = unbounded(); + let (verified_extended_sender, verified_extended_receiver) = + unbounded::>(); + let tx_extended_sigverify_stage = { + let verifier = TransactionSigVerifier::::default(); + SigVerifyStage::new(extended_receiver, verified_extended_sender, verifier) + }; + let (verified_tpu_vote_packets_sender, verified_tpu_vote_packets_receiver) = + unbounded::>(); let vote_sigverify_stage = { - let verifier = TransactionSigVerifier::new_reject_non_vote(); + let verifier = TransactionSigVerifier::::new_reject_non_vote(); SigVerifyStage::new( vote_packet_receiver, verified_tpu_vote_packets_sender, @@ -108,7 +125,7 @@ impl Tpu { }; let (verified_gossip_vote_packets_sender, verified_gossip_vote_packets_receiver) = - unbounded(); + unbounded::>(); let cluster_info_vote_listener = ClusterInfoVoteListener::new( exit.clone(), cluster_info.clone(), @@ -125,10 +142,11 @@ impl Tpu { cluster_confirmed_slot_sender, ); - let banking_stage = BankingStage::new( + let banking_stage: BankingStage = BankingStage::new( cluster_info, poh_recorder, verified_receiver, + verified_extended_receiver, verified_tpu_vote_packets_receiver, verified_gossip_vote_packets_receiver, transaction_status_sender, @@ -150,7 +168,8 @@ impl Tpu { Self { fetch_stage, - sigverify_stage, + tx_sigverify_stage, + tx_extended_sigverify_stage, vote_sigverify_stage, banking_stage, cluster_info_vote_listener, @@ -161,7 +180,8 @@ impl Tpu { pub fn join(self) -> thread::Result<()> { let results = vec![ self.fetch_stage.join(), - self.sigverify_stage.join(), + self.tx_sigverify_stage.join(), + self.tx_extended_sigverify_stage.join(), self.vote_sigverify_stage.join(), self.cluster_info_vote_listener.join(), self.banking_stage.join(), diff --git a/core/src/validator.rs b/core/src/validator.rs index 447ca9f12f3580..8549a3d99b3f5f 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -901,6 +901,8 @@ impl Validator { retransmit_slots_receiver, node.sockets.tpu, node.sockets.tpu_forwards, + node.sockets.tpu_extended, + node.sockets.tpu_extended_forwards, node.sockets.tpu_vote, node.sockets.broadcast, &rpc_subscriptions, diff --git a/core/src/verified_vote_packets.rs b/core/src/verified_vote_packets.rs index 191d23039eb675..ea6d423359f76c 100644 --- a/core/src/verified_vote_packets.rs +++ b/core/src/verified_vote_packets.rs @@ -1,6 +1,7 @@ use { crate::{cluster_info_vote_listener::VerifiedLabelVotePacketsReceiver, result::Result}, - solana_perf::packet::PacketBatch, + crossbeam_channel::Select, + solana_perf::packet::StandardPacketBatch, solana_runtime::bank::Bank, solana_sdk::{ account::from_account, clock::Slot, hash::Hash, pubkey::Pubkey, signature::Signature, @@ -19,7 +20,7 @@ const MAX_VOTES_PER_VALIDATOR: usize = 1000; pub struct VerifiedVoteMetadata { pub vote_account_key: Pubkey, pub vote: VoteTransaction, - pub packet_batch: PacketBatch, + pub packet_batch: StandardPacketBatch, pub signature: Signature, } @@ -69,7 +70,7 @@ impl<'a> ValidatorGossipVotesIterator<'a> { /// /// Iterator is done after iterating through all vote accounts impl<'a> Iterator for ValidatorGossipVotesIterator<'a> { - type Item = Vec; + type Item = Vec; fn next(&mut self) -> Option { // TODO: Maybe prioritize by stake weight @@ -115,7 +116,7 @@ impl<'a> Iterator for ValidatorGossipVotesIterator<'a> { None } }) - .collect::>() + .collect::>() }) }) }); @@ -129,7 +130,7 @@ impl<'a> Iterator for ValidatorGossipVotesIterator<'a> { } } -pub type SingleValidatorVotes = BTreeMap<(Slot, Hash), (PacketBatch, Signature)>; +pub type SingleValidatorVotes = BTreeMap<(Slot, Hash), (StandardPacketBatch, Signature)>; #[derive(Default)] pub struct VerifiedVotePackets(HashMap); diff --git a/core/src/window_service.rs b/core/src/window_service.rs index e77ab6f0b6e0af..2bd0072fa15f18 100644 --- a/core/src/window_service.rs +++ b/core/src/window_service.rs @@ -22,7 +22,7 @@ use { }, solana_measure::measure::Measure, solana_metrics::{inc_new_counter_debug, inc_new_counter_error}, - solana_perf::packet::{Packet, PacketBatch}, + solana_perf::packet::{Packet, StandardPacketBatch}, solana_rayon_threadlimit::get_thread_count, solana_runtime::{bank::Bank, bank_forks::BankForks}, solana_sdk::{clock::Slot, packet::PACKET_DATA_SIZE, pubkey::Pubkey}, @@ -346,7 +346,7 @@ fn recv_window( blockstore: &Blockstore, bank_forks: &RwLock, insert_shred_sender: &CrossbeamSender<(Vec, Vec>)>, - verified_receiver: &CrossbeamReceiver>, + verified_receiver: &CrossbeamReceiver>, retransmit_sender: &Sender>, shred_filter: F, thread_pool: &ThreadPool, @@ -451,7 +451,7 @@ impl WindowService { #[allow(clippy::too_many_arguments)] pub(crate) fn new( blockstore: Arc, - verified_receiver: CrossbeamReceiver>, + verified_receiver: CrossbeamReceiver>, retransmit_sender: Sender>, repair_socket: Arc, ancestor_hashes_socket: Arc, @@ -624,7 +624,7 @@ impl WindowService { exit: Arc, blockstore: Arc, insert_sender: CrossbeamSender<(Vec, Vec>)>, - verified_receiver: CrossbeamReceiver>, + verified_receiver: CrossbeamReceiver>, shred_filter: F, bank_forks: Arc>, retransmit_sender: Sender>, diff --git a/dos/src/main.rs b/dos/src/main.rs index 7ec6274edb0846..5a63866c1e49f5 100644 --- a/dos/src/main.rs +++ b/dos/src/main.rs @@ -48,6 +48,7 @@ fn run_dos( "tvu_forwards" => Some(node.tvu_forwards), "tpu" => Some(node.tpu), "tpu_forwards" => Some(node.tpu_forwards), + "tpu_extended_forwards" => Some(node.tpu_extended_forwards), "repair" => Some(node.repair), "serve_repair" => Some(node.serve_repair), "rpc" => { diff --git a/entry/benches/entry_sigverify.rs b/entry/benches/entry_sigverify.rs index d2c80a69a7a863..ffee7a6b28c06f 100644 --- a/entry/benches/entry_sigverify.rs +++ b/entry/benches/entry_sigverify.rs @@ -35,7 +35,7 @@ fn bench_gpusigverify(bencher: &mut Bencher) { versioned_tx.message.hash() }; - SanitizedTransaction::try_create(versioned_tx, message_hash, None, |_| { + SanitizedTransaction::try_create(versioned_tx, message_hash, None, None, |_| { Err(TransactionError::UnsupportedVersion) }) }?; @@ -74,7 +74,7 @@ fn bench_cpusigverify(bencher: &mut Bencher) { let sanitized_tx = { let message_hash = versioned_tx.verify_and_hash_message()?; - SanitizedTransaction::try_create(versioned_tx, message_hash, None, |_| { + SanitizedTransaction::try_create(versioned_tx, message_hash, None, None, |_| { Err(TransactionError::UnsupportedVersion) }) }?; diff --git a/entry/src/entry.rs b/entry/src/entry.rs index a0d5337b76365a..53270691f6e125 100644 --- a/entry/src/entry.rs +++ b/entry/src/entry.rs @@ -15,7 +15,10 @@ use { solana_metrics::*, solana_perf::{ cuda_runtime::PinnedVec, - packet::{Packet, PacketBatch, PacketBatchRecycler, PACKETS_PER_BATCH}, + packet::{ + ExtendedPacket, Packet, PacketBatch, PacketBatchRecycler, PacketInterface, + PACKETS_PER_BATCH, + }, perf_libs, recycler::Recycler, sigverify, @@ -260,6 +263,7 @@ pub struct EntryVerificationState { pub struct GpuSigVerificationData { thread_h: Option>, + thread_h_extended: Option>, } pub enum DeviceSigVerificationData { @@ -283,13 +287,22 @@ impl<'a> EntrySigVerificationState { DeviceSigVerificationData::Gpu(verification_state) => { let (verified, gpu_time_us) = verification_state.thread_h.take().unwrap().join().unwrap(); - self.gpu_verify_duration_us = gpu_time_us; - self.verification_status = if verified { + let (verified_extended, gpu_time_us_extended) = verification_state + .thread_h_extended + .take() + .unwrap() + .join() + .unwrap(); + // todo: improve this metric, perhaps by simply recording the start and end timestamps + // for both verifications and taking the difference of the most extreme of + // the start and end times for both + self.gpu_verify_duration_us = cmp::max(gpu_time_us, gpu_time_us_extended); + self.verification_status = if verified && verified_extended { EntryVerificationStatus::Success } else { EntryVerificationStatus::Failure }; - verified + verified && verified_extended } DeviceSigVerificationData::Cpu() => { self.verification_status == EntryVerificationStatus::Success @@ -308,7 +321,8 @@ impl<'a> EntrySigVerificationState { pub struct VerifyRecyclers { hash_recycler: Recycler>, tick_count_recycler: Recycler>, - packet_recycler: PacketBatchRecycler, + packet_recycler: PacketBatchRecycler, + extended_packet_recycler: PacketBatchRecycler, out_recycler: Recycler>, tx_offset_recycler: Recycler, } @@ -473,17 +487,19 @@ pub fn start_verify_transactions( let entries = verify_transactions(entries, Arc::new(verify_func)); match entries { Ok(entries) => { - let num_transactions: usize = entries + let (entry_txs, large_entry_txs): ( + Vec<&SanitizedTransaction>, + Vec<&SanitizedTransaction>, + ) = entries .iter() - .map(|entry: &EntryType| -> usize { - match entry { - EntryType::Transactions(transactions) => transactions.len(), - EntryType::Tick(_) => 0, - } + .filter_map(|entry_type| match entry_type { + EntryType::Tick(_) => None, + EntryType::Transactions(transactions) => Some(transactions.iter()), }) - .sum(); + .flatten() + .partition(|transaction| transaction.is_large_transaction()); - if num_transactions == 0 { + if entry_txs.is_empty() && large_entry_txs.is_empty() { return Ok(EntrySigVerificationState { verification_status: EntryVerificationStatus::Success, entries: Some(entries), @@ -491,28 +507,22 @@ pub fn start_verify_transactions( gpu_verify_duration_us: 0, }); } - let entry_txs: Vec<&SanitizedTransaction> = entries - .iter() - .filter_map(|entry_type| match entry_type { - EntryType::Tick(_) => None, - EntryType::Transactions(transactions) => Some(transactions), - }) - .flatten() - .collect::>(); + + // TODO: Still lots of duplicate code below let mut packet_batches = entry_txs .par_iter() .chunks(PACKETS_PER_BATCH) .map(|slice| { let vec_size = slice.len(); - let mut packet_batch = PacketBatch::new_with_recycler( + let mut packet_batch = PacketBatch::::new_with_recycler( verify_recyclers.packet_recycler.clone(), vec_size, "entry-sig-verify", ); - // We use set_len here instead of resize(num_transactions, Packet::default()), to save + // We use set_len here instead of resize(vec_size, Packet::default()), to save // memory bandwidth and avoid writing a large amount of data that will be overwritten // soon afterwards. As well, Packet::default() actually leaves the packet data - // uninitialized anyway, so the initilization would simply write junk into + // uninitialized, so the initialization would simply write junk into // the vector anyway. unsafe { packet_batch.packets.set_len(vec_size); @@ -537,19 +547,85 @@ pub fn start_verify_transactions( }) .collect::>>()?; + let mut extended_packet_batches = large_entry_txs + .par_iter() + .chunks(PACKETS_PER_BATCH) + .map(|slice| { + let vec_size = slice.len(); + let mut packet_batch = PacketBatch::::new_with_recycler( + verify_recyclers.extended_packet_recycler.clone(), + vec_size, + "entry-sig-verify", + ); + // We use set_len here instead of resize(vec_size, Packet::default()), to save + // memory bandwidth and avoid writing a large amount of data that will be overwritten + // soon afterwards. As well, Packet::default() actually leaves the packet data + // uninitialized, so the initialization would simply write junk into + // the vector anyway. + unsafe { + packet_batch.packets.set_len(vec_size); + } + let entry_tx_iter = slice + .into_par_iter() + .map(|tx| tx.to_versioned_transaction()); + + let res = packet_batch + .packets + .par_iter_mut() + .zip(entry_tx_iter) + .all(|pair| { + pair.0.meta = Meta::default(); + ExtendedPacket::populate_packet(pair.0, None, &pair.1).is_ok() + }); + if res { + Ok(packet_batch) + } else { + Err(TransactionError::SanitizeFailure) + } + }) + .collect::>>()?; + let tx_offset_recycler = verify_recyclers.tx_offset_recycler; let out_recycler = verify_recyclers.out_recycler; - let gpu_verify_thread = thread::spawn(move || { + let gpu_verify_thread = { + let tx_offset_recycler = tx_offset_recycler.clone(); + let out_recycler = out_recycler.clone(); + thread::spawn(move || { + let mut verify_time = Measure::start("sigverify"); + let verified = if !packet_batches.is_empty() { + sigverify::ed25519_verify( + &mut packet_batches, + &tx_offset_recycler, + &out_recycler, + false, + ); + packet_batches + .iter() + .all(|batch| batch.packets.iter().all(|p| !p.meta.discard)) + } else { + true + }; + verify_time.stop(); + (verified, verify_time.as_us()) + }) + }; + + let gpu_extended_verify_thread = thread::spawn(move || { let mut verify_time = Measure::start("sigverify"); - sigverify::ed25519_verify( - &mut packet_batches, - &tx_offset_recycler, - &out_recycler, - false, - ); - let verified = packet_batches - .iter() - .all(|batch| batch.packets.iter().all(|p| !p.meta.discard)); + let verified = if !extended_packet_batches.is_empty() { + //todo: verify that sigverify::ed25519_verify is thread-safe + sigverify::ed25519_verify( + &mut extended_packet_batches, + &tx_offset_recycler, + &out_recycler, + false, + ); + extended_packet_batches + .iter() + .all(|batch| batch.packets.iter().all(|p| !p.meta.discard)) + } else { + true + }; verify_time.stop(); (verified, verify_time.as_us()) }); @@ -558,6 +634,7 @@ pub fn start_verify_transactions( entries: Some(entries), device_verification_data: DeviceSigVerificationData::Gpu(GpuSigVerificationData { thread_h: Some(gpu_verify_thread), + thread_h_extended: Some(gpu_extended_verify_thread), }), gpu_verify_duration_us: 0, }) @@ -1003,7 +1080,7 @@ mod tests { versioned_tx.message.hash() }; - SanitizedTransaction::try_create(versioned_tx, message_hash, None, |_| { + SanitizedTransaction::try_create(versioned_tx, message_hash, None, None, |_| { Err(TransactionError::UnsupportedVersion) }) }?; diff --git a/gossip/src/cluster_info.rs b/gossip/src/cluster_info.rs index 860e553f53fee9..0ec1059e5f7710 100644 --- a/gossip/src/cluster_info.rs +++ b/gossip/src/cluster_info.rs @@ -47,8 +47,8 @@ use { solana_perf::{ data_budget::DataBudget, packet::{ - limited_deserialize, to_packet_batch_with_destination, Packet, PacketBatch, - PacketBatchRecycler, PACKET_DATA_SIZE, + limited_deserialize, to_packet_batch_with_destination, Packet, PacketInterface, + StandardPacketBatch, StandardPacketBatchRecycler, PACKET_DATA_SIZE, }, }, solana_rayon_threadlimit::get_thread_count, @@ -67,7 +67,7 @@ use { packet, sendmmsg::{multi_target_send, SendPktsError}, socket::SocketAddrSpace, - streamer::{PacketBatchReceiver, PacketBatchSender}, + streamer::{StandardPacketReceiver, StandardPacketSender}, }, solana_vote_program::{ vote_state::MAX_LOCKOUT_HISTORY, vote_transaction::parse_vote_transaction, @@ -773,6 +773,7 @@ impl ClusterInfo { } }; let ip_addr = node.gossip.ip(); + // TODO: Add Extended ports to this printout Some(format!( "{:15} {:2}| {:5} | {:44} |{:^9}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {}\n", if ContactInfo::is_valid_address(&node.gossip, &self.socket_addr_space) { @@ -802,6 +803,7 @@ impl ClusterInfo { }) .collect(); + // TODO: Add Extended ports to this printout format!( "IP Address |Age(ms)| Node identifier \ | Version |Gossip|TPUvote| TPU |TPUfwd| TVU |TVUfwd|Repair|ServeR|ShredVer\n\ @@ -1588,9 +1590,9 @@ impl ClusterInfo { &self, thread_pool: &ThreadPool, gossip_validators: Option<&HashSet>, - recycler: &PacketBatchRecycler, + recycler: &StandardPacketBatchRecycler, stakes: &HashMap, - sender: &PacketBatchSender, + sender: &StandardPacketSender, generate_pull_requests: bool, ) -> Result<(), GossipError> { let reqs = self.generate_new_gossip_requests( @@ -1699,7 +1701,7 @@ impl ClusterInfo { pub fn gossip( self: Arc, bank_forks: Option>>, - sender: PacketBatchSender, + sender: StandardPacketSender, gossip_validators: Option>, exit: Arc, ) -> JoinHandle<()> { @@ -1715,7 +1717,7 @@ impl ClusterInfo { let mut last_contact_info_trace = timestamp(); let mut last_contact_info_save = timestamp(); let mut entrypoints_processed = false; - let recycler = PacketBatchRecycler::default(); + let recycler = StandardPacketBatchRecycler::default(); let crds_data = vec![ CrdsData::Version(Version::new(self.id())), CrdsData::NodeInstance( @@ -1840,9 +1842,9 @@ impl ClusterInfo { // from address, crds filter, caller contact info requests: Vec<(SocketAddr, CrdsFilter, CrdsValue)>, thread_pool: &ThreadPool, - recycler: &PacketBatchRecycler, + recycler: &StandardPacketBatchRecycler, stakes: &HashMap, - response_sender: &PacketBatchSender, + response_sender: &StandardPacketSender, ) { let _st = ScopedTimer::from(&self.stats.handle_batch_pull_requests_time); if requests.is_empty() { @@ -1904,7 +1906,7 @@ impl ClusterInfo { &'a self, now: Instant, mut rng: &'a mut R, - packet_batch: &'a mut PacketBatch, + packet_batch: &'a mut StandardPacketBatch, ) -> impl FnMut(&PullData) -> bool + 'a where R: Rng + CryptoRng, @@ -1944,10 +1946,10 @@ impl ClusterInfo { fn handle_pull_requests( &self, thread_pool: &ThreadPool, - recycler: &PacketBatchRecycler, + recycler: &StandardPacketBatchRecycler, requests: Vec, stakes: &HashMap, - ) -> PacketBatch { + ) -> StandardPacketBatch { const DEFAULT_EPOCH_DURATION_MS: u64 = DEFAULT_SLOTS_PER_EPOCH * DEFAULT_MS_PER_SLOT; let mut time = Measure::start("handle_pull_requests"); let callers = crds_value::filter_current(requests.iter().map(|r| &r.caller)); @@ -1958,8 +1960,11 @@ impl ClusterInfo { } let output_size_limit = self.update_data_budget(stakes.len()) / PULL_RESPONSE_MIN_SERIALIZED_SIZE; - let mut packet_batch = - PacketBatch::new_unpinned_with_recycler(recycler.clone(), 64, "handle_pull_requests"); + let mut packet_batch = StandardPacketBatch::new_unpinned_with_recycler( + recycler.clone(), + 64, + "handle_pull_requests", + ); let (caller_and_filters, addrs): (Vec<_>, Vec<_>) = { let mut rng = rand::thread_rng(); let check_pull_request = @@ -2164,8 +2169,8 @@ impl ClusterInfo { fn handle_batch_ping_messages( &self, pings: I, - recycler: &PacketBatchRecycler, - response_sender: &PacketBatchSender, + recycler: &StandardPacketBatchRecycler, + response_sender: &StandardPacketSender, ) where I: IntoIterator, { @@ -2178,8 +2183,8 @@ impl ClusterInfo { fn handle_ping_messages( &self, pings: I, - recycler: &PacketBatchRecycler, - ) -> Option + recycler: &StandardPacketBatchRecycler, + ) -> Option where I: IntoIterator, { @@ -2201,7 +2206,7 @@ impl ClusterInfo { if packets.is_empty() { None } else { - let packet_batch = PacketBatch::new_unpinned_with_recycler_data( + let packet_batch = StandardPacketBatch::new_unpinned_with_recycler_data( recycler, "handle_ping_messages", packets, @@ -2229,9 +2234,9 @@ impl ClusterInfo { &self, messages: Vec<(Pubkey, Vec)>, thread_pool: &ThreadPool, - recycler: &PacketBatchRecycler, + recycler: &StandardPacketBatchRecycler, stakes: &HashMap, - response_sender: &PacketBatchSender, + response_sender: &StandardPacketSender, ) { let _st = ScopedTimer::from(&self.stats.handle_batch_push_messages_time); if messages.is_empty() { @@ -2349,8 +2354,8 @@ impl ClusterInfo { &self, packets: VecDeque<(/*from:*/ SocketAddr, Protocol)>, thread_pool: &ThreadPool, - recycler: &PacketBatchRecycler, - response_sender: &PacketBatchSender, + recycler: &StandardPacketBatchRecycler, + response_sender: &StandardPacketSender, stakes: &HashMap, _feature_set: Option<&FeatureSet>, epoch_duration: Duration, @@ -2467,7 +2472,7 @@ impl ClusterInfo { // handling of requests/messages. fn run_socket_consume( &self, - receiver: &PacketBatchReceiver, + receiver: &StandardPacketReceiver, sender: &Sender>, thread_pool: &ThreadPool, ) -> Result<(), GossipError> { @@ -2507,10 +2512,10 @@ impl ClusterInfo { /// Process messages from the network fn run_listen( &self, - recycler: &PacketBatchRecycler, + recycler: &StandardPacketBatchRecycler, bank_forks: Option<&RwLock>, receiver: &Receiver>, - response_sender: &PacketBatchSender, + response_sender: &StandardPacketSender, thread_pool: &ThreadPool, last_print: &mut Instant, should_check_duplicate_instance: bool, @@ -2558,7 +2563,7 @@ impl ClusterInfo { pub(crate) fn start_socket_consume_thread( self: Arc, - receiver: PacketBatchReceiver, + receiver: StandardPacketReceiver, sender: Sender>, exit: Arc, ) -> JoinHandle<()> { @@ -2588,12 +2593,12 @@ impl ClusterInfo { self: Arc, bank_forks: Option>>, requests_receiver: Receiver>, - response_sender: PacketBatchSender, + response_sender: StandardPacketSender, should_check_duplicate_instance: bool, exit: Arc, ) -> JoinHandle<()> { let mut last_print = Instant::now(); - let recycler = PacketBatchRecycler::default(); + let recycler = StandardPacketBatchRecycler::default(); let thread_pool = ThreadPoolBuilder::new() .num_threads(get_thread_count().min(8)) .thread_name(|i| format!("sol-gossip-work-{}", i)) @@ -2737,6 +2742,8 @@ pub struct Sockets { pub tvu_forwards: Vec, pub tpu: Vec, pub tpu_forwards: Vec, + pub tpu_extended: Vec, + pub tpu_extended_forwards: Vec, pub tpu_vote: Vec, pub broadcast: Vec, pub repair: UdpSocket, @@ -2765,6 +2772,8 @@ impl Node { let tvu = UdpSocket::bind("127.0.0.1:0").unwrap(); let tvu_forwards = UdpSocket::bind("127.0.0.1:0").unwrap(); let tpu_forwards = UdpSocket::bind("127.0.0.1:0").unwrap(); + let tpu_extended = UdpSocket::bind("127.0.0.1:0").unwrap(); + let tpu_extended_forwards = UdpSocket::bind("127.0.0.1:0").unwrap(); let tpu_vote = UdpSocket::bind("127.0.0.1:0").unwrap(); let repair = UdpSocket::bind("127.0.0.1:0").unwrap(); let rpc_port = find_available_port_in_range(bind_ip_addr, (1024, 65535)).unwrap(); @@ -2786,6 +2795,8 @@ impl Node { repair: repair.local_addr().unwrap(), tpu: tpu.local_addr().unwrap(), tpu_forwards: tpu_forwards.local_addr().unwrap(), + tpu_extended: tpu_extended.local_addr().unwrap(), + tpu_extended_forwards: tpu_extended_forwards.local_addr().unwrap(), tpu_vote: tpu_vote.local_addr().unwrap(), rpc: rpc_addr, rpc_pubsub: rpc_pubsub_addr, @@ -2802,6 +2813,8 @@ impl Node { tvu_forwards: vec![tvu_forwards], tpu: vec![tpu], tpu_forwards: vec![tpu_forwards], + tpu_extended: vec![tpu_extended], + tpu_extended_forwards: vec![tpu_extended_forwards], tpu_vote: vec![tpu_vote], broadcast, repair, @@ -2844,6 +2857,9 @@ impl Node { let (tvu_forwards_port, tvu_forwards) = Self::bind(bind_ip_addr, port_range); let (tpu_port, tpu) = Self::bind(bind_ip_addr, port_range); let (tpu_forwards_port, tpu_forwards) = Self::bind(bind_ip_addr, port_range); + let (tpu_extended_port, tpu_extended) = Self::bind(bind_ip_addr, port_range); + let (tpu_extended_forwards_port, tpu_extended_forwards) = + Self::bind(bind_ip_addr, port_range); let (tpu_vote_port, tpu_vote) = Self::bind(bind_ip_addr, port_range); let (_, retransmit_socket) = Self::bind(bind_ip_addr, port_range); let (repair_port, repair) = Self::bind(bind_ip_addr, port_range); @@ -2862,6 +2878,8 @@ impl Node { repair: SocketAddr::new(gossip_addr.ip(), repair_port), tpu: SocketAddr::new(gossip_addr.ip(), tpu_port), tpu_forwards: SocketAddr::new(gossip_addr.ip(), tpu_forwards_port), + tpu_extended: SocketAddr::new(gossip_addr.ip(), tpu_extended_port), + tpu_extended_forwards: SocketAddr::new(gossip_addr.ip(), tpu_extended_forwards_port), tpu_vote: SocketAddr::new(gossip_addr.ip(), tpu_vote_port), rpc: SocketAddr::new(gossip_addr.ip(), rpc_port), rpc_pubsub: SocketAddr::new(gossip_addr.ip(), rpc_pubsub_port), @@ -2880,6 +2898,8 @@ impl Node { tvu_forwards: vec![tvu_forwards], tpu: vec![tpu], tpu_forwards: vec![tpu_forwards], + tpu_extended: vec![tpu_extended], + tpu_extended_forwards: vec![tpu_extended_forwards], tpu_vote: vec![tpu_vote], broadcast: vec![broadcast], repair, @@ -2911,6 +2931,16 @@ impl Node { let (tpu_forwards_port, tpu_forwards_sockets) = multi_bind_in_range(bind_ip_addr, port_range, 8).expect("tpu_forwards multi_bind"); + // TODO: How many sockets (which translates to threds) should we set aside here ? + // Currently have half as many as `tpu_sockets` + let (tpu_extended_port, tpu_extended_sockets) = + multi_bind_in_range(bind_ip_addr, port_range, 16).expect("tpu_extended multi_bind"); + + // TODO: How many sockets (which translates to threds) should we set aside here ? + // Currently have half as many as `tpu_forwards_sockets` + let (tpu_extended_forwards_port, tpu_extended_forwards_sockets) = + multi_bind_in_range(bind_ip_addr, port_range, 4).expect("tpu_forwards multi_bind"); + let (tpu_vote_port, tpu_vote_sockets) = multi_bind_in_range(bind_ip_addr, port_range, 1).expect("tpu_vote multi_bind"); @@ -2933,6 +2963,8 @@ impl Node { repair: SocketAddr::new(gossip_addr.ip(), repair_port), tpu: SocketAddr::new(gossip_addr.ip(), tpu_port), tpu_forwards: SocketAddr::new(gossip_addr.ip(), tpu_forwards_port), + tpu_extended: SocketAddr::new(gossip_addr.ip(), tpu_extended_port), + tpu_extended_forwards: SocketAddr::new(gossip_addr.ip(), tpu_extended_forwards_port), tpu_vote: SocketAddr::new(gossip_addr.ip(), tpu_vote_port), rpc: socketaddr_any!(), rpc_pubsub: socketaddr_any!(), @@ -2950,6 +2982,8 @@ impl Node { tvu_forwards: tvu_forwards_sockets, tpu: tpu_sockets, tpu_forwards: tpu_forwards_sockets, + tpu_extended: tpu_extended_sockets, + tpu_extended_forwards: tpu_extended_forwards_sockets, tpu_vote: tpu_vote_sockets, broadcast, repair, @@ -2971,7 +3005,8 @@ pub fn push_messages_to_peer( let reqs: Vec<_> = ClusterInfo::split_gossip_messages(PUSH_MESSAGE_MAX_PAYLOAD_SIZE, messages) .map(move |payload| (peer_gossip, Protocol::PushMessage(self_id, payload))) .collect(); - let packet_batch = to_packet_batch_with_destination(PacketBatchRecycler::default(), &reqs); + let packet_batch = + to_packet_batch_with_destination(StandardPacketBatchRecycler::default(), &reqs); let sock = UdpSocket::bind("0.0.0.0:0").unwrap(); packet::send_to(&packet_batch, &sock, socket_addr_space)?; Ok(()) @@ -3222,7 +3257,7 @@ mod tests { .iter() .map(|ping| Pong::new(ping, &this_node).unwrap()) .collect(); - let recycler = PacketBatchRecycler::default(); + let recycler = StandardPacketBatchRecycler::default(); let packets = cluster_info .handle_ping_messages( remote_nodes diff --git a/gossip/src/contact_info.rs b/gossip/src/contact_info.rs index a614fee7e584d0..0c17e239f2eb1b 100644 --- a/gossip/src/contact_info.rs +++ b/gossip/src/contact_info.rs @@ -35,6 +35,13 @@ pub struct ContactInfo { pub rpc_pubsub: SocketAddr, /// address to send repair requests to pub serve_repair: SocketAddr, + // TODO: Logically, I'd like to put tpu_extended fields right after tpu / tpu_forwards; + // however, it isn't clear to me if re-ordering will break things ? + // TODO: How do we "broadcast" the extended ports ? + /// address to send extended transactions to + pub tpu_extended: SocketAddr, + /// address to forward unprocessed extended transactions to + pub tpu_extended_forwards: SocketAddr, /// latest wallclock picked pub wallclock: u64, /// node shred version @@ -80,6 +87,8 @@ impl Default for ContactInfo { rpc: socketaddr_any!(), rpc_pubsub: socketaddr_any!(), serve_repair: socketaddr_any!(), + tpu_extended: socketaddr_any!(), + tpu_extended_forwards: socketaddr_any!(), wallclock: 0, shred_version: 0, } @@ -100,6 +109,8 @@ impl ContactInfo { rpc: socketaddr!("127.0.0.1:1241"), rpc_pubsub: socketaddr!("127.0.0.1:1242"), serve_repair: socketaddr!("127.0.0.1:1243"), + tpu_extended: socketaddr!("127.0.0.1:1244"), + tpu_extended_forwards: socketaddr!("127.0.0.1:1245"), wallclock: now, shred_version: 0, } @@ -130,6 +141,8 @@ impl ContactInfo { rpc: addr, rpc_pubsub: addr, serve_repair: addr, + tpu_extended: addr, + tpu_extended_forwards: addr, wallclock: 0, shred_version: 0, } @@ -153,6 +166,8 @@ impl ContactInfo { let rpc_pubsub = SocketAddr::new(bind_addr.ip(), rpc_port::DEFAULT_RPC_PUBSUB_PORT); let serve_repair = next_port(bind_addr, 6); let tpu_vote = next_port(bind_addr, 7); + let tpu_extended = next_port(bind_addr, 8); + let tpu_extended_forwards = next_port(bind_addr, 9); Self { id: *pubkey, gossip, @@ -165,6 +180,8 @@ impl ContactInfo { rpc, rpc_pubsub, serve_repair, + tpu_extended, + tpu_extended_forwards, wallclock: timestamp(), shred_version: 0, } @@ -207,18 +224,18 @@ impl ContactInfo { Self::is_valid_tvu_address(addr) && socket_addr_space.check(addr) } - pub fn client_facing_addr(&self) -> (SocketAddr, SocketAddr) { - (self.rpc, self.tpu) + pub fn client_facing_addr(&self) -> (SocketAddr, SocketAddr, SocketAddr) { + (self.rpc, self.tpu, self.tpu_extended) } pub fn valid_client_facing_addr( &self, socket_addr_space: &SocketAddrSpace, - ) -> Option<(SocketAddr, SocketAddr)> { + ) -> Option<(SocketAddr, SocketAddr, SocketAddr)> { if ContactInfo::is_valid_address(&self.rpc, socket_addr_space) && ContactInfo::is_valid_address(&self.tpu, socket_addr_space) { - Some((self.rpc, self.tpu)) + Some((self.rpc, self.tpu, self.tpu_extended)) } else { None } @@ -329,6 +346,8 @@ mod tests { assert_eq!(d1.repair, socketaddr!("127.0.0.1:1239")); assert_eq!(d1.serve_repair, socketaddr!("127.0.0.1:1240")); assert_eq!(d1.tpu_vote, socketaddr!("127.0.0.1:1241")); + assert_eq!(d1.tpu_extended, socketaddr!("127.0.0.1:1242")); + assert_eq!(d1.tpu_extended_forwards, socketaddr!("127.0.0.1:1243")); } #[test] diff --git a/gossip/src/gossip_service.rs b/gossip/src/gossip_service.rs index db9e295b72bf08..c9b7ba14e633bd 100644 --- a/gossip/src/gossip_service.rs +++ b/gossip/src/gossip_service.rs @@ -228,6 +228,7 @@ pub fn get_multi_client( .collect(); let rpc_addrs: Vec<_> = addrs.iter().map(|addr| addr.0).collect(); let tpu_addrs: Vec<_> = addrs.iter().map(|addr| addr.1).collect(); + let tpu_extended_addrs: Vec<_> = addrs.iter().map(|addr| addr.2).collect(); let (_, transactions_socket) = solana_net_utils::bind_in_range( IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), VALIDATOR_PORT_RANGE, @@ -235,7 +236,7 @@ pub fn get_multi_client( .unwrap(); let num_nodes = tpu_addrs.len(); ( - ThinClient::new_from_addrs(rpc_addrs, tpu_addrs, transactions_socket), + ThinClient::new_from_addrs(rpc_addrs, tpu_addrs, tpu_extended_addrs, transactions_socket), num_nodes, ) } diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index e9f01b4f8f7254..bee626c46763ec 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -233,7 +233,7 @@ fn output_slot( for transaction in entry.transactions { let tx_signature = transaction.signatures[0]; let sanitize_result = - SanitizedTransaction::try_create(transaction, Hash::default(), None, |_| { + SanitizedTransaction::try_create(transaction, Hash::default(), None, None, |_| { Err(TransactionError::UnsupportedVersion) }); @@ -789,7 +789,7 @@ fn compute_slot_cost(blockstore: &Blockstore, slot: Slot) -> Result<(), String> .transactions .into_iter() .filter_map(|transaction| { - SanitizedTransaction::try_create(transaction, Hash::default(), None, |_| { + SanitizedTransaction::try_create(transaction, Hash::default(), None, None, |_| { Err(TransactionError::UnsupportedVersion) }) .map_err(|err| { diff --git a/ledger/src/sigverify_shreds.rs b/ledger/src/sigverify_shreds.rs index a8e8868cf0be09..78d9af5dbf4cb0 100644 --- a/ledger/src/sigverify_shreds.rs +++ b/ledger/src/sigverify_shreds.rs @@ -36,6 +36,8 @@ lazy_static! { .unwrap(); } +type ShredPackets = PacketBatch; + /// Assuming layout is /// signature: Signature /// signed_msg: { @@ -77,7 +79,7 @@ pub fn verify_shred_cpu(packet: &Packet, slot_leaders: &HashMap) } fn verify_shreds_cpu( - batches: &[PacketBatch], + batches: &[ShredPackets], slot_leaders: &HashMap, ) -> Vec> { use rayon::prelude::*; @@ -103,7 +105,7 @@ fn slot_key_data_for_gpu< T: Sync + Sized + Default + std::fmt::Debug + Eq + std::hash::Hash + Clone + Copy + AsRef<[u8]>, >( offset_start: usize, - batches: &[PacketBatch], + batches: &[ShredPackets], slot_keys: &HashMap, recycler_cache: &RecyclerCache, ) -> (PinnedVec, TxOffset, usize) { @@ -188,7 +190,7 @@ fn resize_vec(keyvec: &mut PinnedVec) -> usize { fn shred_gpu_offsets( mut pubkeys_end: usize, - batches: &[PacketBatch], + batches: &[ShredPackets], recycler_cache: &RecyclerCache, ) -> (TxOffset, TxOffset, TxOffset, Vec>) { let mut signature_offsets = recycler_cache.offsets().allocate("shred_signatures"); @@ -226,7 +228,7 @@ fn shred_gpu_offsets( } pub fn verify_shreds_gpu( - batches: &[PacketBatch], + batches: &[ShredPackets], slot_leaders: &HashMap, recycler_cache: &RecyclerCache, ) -> Vec> { @@ -252,13 +254,15 @@ pub fn verify_shreds_gpu( out.set_pinnable(); elems.push(perf_libs::Elems { #[allow(clippy::cast_ptr_alignment)] - elems: pubkeys.as_ptr() as *const solana_sdk::packet::Packet, + // todo: verify + elems: pubkeys.as_ptr().cast::() as *const u8, num: num_packets as u32, }); for batch in batches { elems.push(perf_libs::Elems { - elems: batch.packets.as_ptr(), + // todo: verify + elems: batch.packets.as_ptr().cast::(), num: batch.packets.len() as u32, }); let mut v = Vec::new(); @@ -321,7 +325,7 @@ fn sign_shred_cpu(keypair: &Keypair, packet: &mut Packet) { packet.data[0..sig_end].copy_from_slice(signature.as_ref()); } -pub fn sign_shreds_cpu(keypair: &Keypair, batches: &mut [PacketBatch]) { +pub fn sign_shreds_cpu(keypair: &Keypair, batches: &mut [ShredPackets]) { use rayon::prelude::*; let packet_count = count_packets_in_batches(batches); debug!("CPU SHRED ECDSA for {}", packet_count); @@ -355,7 +359,7 @@ pub fn sign_shreds_gpu_pinned_keypair(keypair: &Keypair, cache: &RecyclerCache) pub fn sign_shreds_gpu( keypair: &Keypair, pinned_keypair: &Option>>, - batches: &mut [PacketBatch], + batches: &mut [ShredPackets], recycler_cache: &RecyclerCache, ) { let sig_size = size_of::(); @@ -389,13 +393,15 @@ pub fn sign_shreds_gpu( signatures_out.resize(total_sigs * sig_size, 0); elems.push(perf_libs::Elems { #[allow(clippy::cast_ptr_alignment)] - elems: pinned_keypair.as_ptr() as *const solana_sdk::packet::Packet, + // todo: verify + elems: pinned_keypair.as_ptr().cast::() as *const u8, num: num_keypair_packets as u32, }); for batch in batches.iter() { elems.push(perf_libs::Elems { - elems: batch.packets.as_ptr(), + // todo: verify + elems: batch.packets.as_ptr().cast::(), num: batch.packets.len() as u32, }); let mut v = Vec::new(); diff --git a/net/init-metrics.sh b/net/init-metrics.sh index e5862420d660f1..b6e8db3371af50 100755 --- a/net/init-metrics.sh +++ b/net/init-metrics.sh @@ -13,15 +13,12 @@ usage() { fi cat <, +pub struct PacketBatch { + pub packets: PinnedVec

, } -pub type PacketBatchRecycler = Recycler>; +// TODO: Do we want to run with "Standard" for naming ? If so, rename "Packet" too ? +pub type StandardPacketBatch = PacketBatch; +pub type ExtendedPacketBatch = PacketBatch; + +pub type PacketBatchRecycler

= Recycler>; +pub type StandardPacketBatchRecycler = PacketBatchRecycler; -impl PacketBatch { - pub fn new(packets: Vec) -> Self { +impl PacketBatch

{ + pub fn new(packets: Vec

) -> Self { let packets = PinnedVec::from_vec(packets); Self { packets } } pub fn with_capacity(capacity: usize) -> Self { let packets = PinnedVec::with_capacity(capacity); - PacketBatch { packets } + Self { packets } } pub fn new_unpinned_with_recycler( - recycler: PacketBatchRecycler, + recycler: PacketBatchRecycler

, size: usize, name: &'static str, ) -> Self { let mut packets = recycler.allocate(name); packets.reserve(size); - PacketBatch { packets } + Self { packets } } pub fn new_with_recycler( - recycler: PacketBatchRecycler, + recycler: PacketBatchRecycler

, size: usize, name: &'static str, ) -> Self { let mut packets = recycler.allocate(name); packets.reserve_and_pin(size); - PacketBatch { packets } + Self { packets } } pub fn new_with_recycler_data( - recycler: &PacketBatchRecycler, + recycler: &PacketBatchRecycler

, name: &'static str, - mut packets: Vec, + mut packets: Vec

, ) -> Self { let mut batch = Self::new_with_recycler(recycler.clone(), packets.len(), name); batch.packets.append(&mut packets); @@ -61,9 +68,9 @@ impl PacketBatch { } pub fn new_unpinned_with_recycler_data( - recycler: &PacketBatchRecycler, + recycler: &PacketBatchRecycler

, name: &'static str, - mut packets: Vec, + mut packets: Vec

, ) -> Self { let mut batch = Self::new_unpinned_with_recycler(recycler.clone(), packets.len(), name); batch.packets.append(&mut packets); @@ -71,8 +78,8 @@ impl PacketBatch { } pub fn set_addr(&mut self, addr: &SocketAddr) { - for p in self.packets.iter_mut() { - p.meta.set_addr(addr); + for packet in self.packets.iter_mut() { + packet.get_meta_mut().set_addr(addr); } } @@ -81,17 +88,20 @@ impl PacketBatch { } } -pub fn to_packet_batches(xs: &[T], chunks: usize) -> Vec { - let mut out = vec![]; +pub fn to_packet_batches( + xs: &[T], + chunks: usize, +) -> Vec> { + let mut batches = vec![]; for x in xs.chunks(chunks) { let mut batch = PacketBatch::with_capacity(x.len()); - batch.packets.resize(x.len(), Packet::default()); + batch.packets.resize(x.len(), P::default()); for (i, packet) in x.iter().zip(batch.packets.iter_mut()) { - Packet::populate_packet(packet, None, i).expect("serialize request"); + P::populate_packet(packet, None, i).expect("serialize request"); } - out.push(batch); + batches.push(batch); } - out + batches } #[cfg(test)] @@ -99,19 +109,19 @@ pub fn to_packet_batches_for_tests(xs: &[T]) -> Vec { to_packet_batches(xs, NUM_PACKETS) } -pub fn to_packet_batch_with_destination( - recycler: PacketBatchRecycler, +pub fn to_packet_batch_with_destination( + recycler: PacketBatchRecycler

, dests_and_data: &[(SocketAddr, T)], -) -> PacketBatch { +) -> PacketBatch

{ let mut out = PacketBatch::new_unpinned_with_recycler( recycler, dests_and_data.len(), "to_packet_batch_with_destination", ); - out.packets.resize(dests_and_data.len(), Packet::default()); + out.packets.resize(dests_and_data.len(), P::default()); for (dest_and_data, o) in dests_and_data.iter().zip(out.packets.iter_mut()) { if !dest_and_data.0.ip().is_unspecified() && dest_and_data.0.port() != 0 { - if let Err(e) = Packet::populate_packet(o, Some(&dest_and_data.0), &dest_and_data.1) { + if let Err(e) = P::populate_packet(o, Some(&dest_and_data.0), &dest_and_data.1) { // TODO: This should never happen. Instead the caller should // break the payload into smaller messages, and here any errors // should be propagated. @@ -128,8 +138,12 @@ pub fn limited_deserialize(data: &[u8]) -> bincode::Result where T: serde::de::DeserializeOwned, { + // TODO verify. It seems that if one were using this with a regular packet + // the smaller data field would imply the original PACKET_DATA_SIZE bound anyway. + // Is there any reason we would not want to use EXTENDED_PACKET_DATA_SIZE here to + // be able to work with ExtendPacket as well? bincode::options() - .with_limit(PACKET_DATA_SIZE as u64) + .with_limit(EXTENDED_PACKET_DATA_SIZE as u64) .with_fixint_encoding() .allow_trailing_bytes() .deserialize_from(data) diff --git a/perf/src/perf_libs.rs b/perf/src/perf_libs.rs index 1f0505b4a4e7dd..ea25434969dc19 100644 --- a/perf/src/perf_libs.rs +++ b/perf/src/perf_libs.rs @@ -3,7 +3,6 @@ use { dlopen::symbor::{Container, SymBorApi, Symbol}, dlopen_derive::SymBorApi, log::*, - solana_sdk::packet::Packet, std::{ env, ffi::OsStr, @@ -16,7 +15,7 @@ use { #[repr(C)] pub struct Elems { - pub elems: *const Packet, + pub elems: *const u8, pub num: u32, } diff --git a/perf/src/sigverify.rs b/perf/src/sigverify.rs index 3da88f9dd02d42..71240d94592264 100644 --- a/perf/src/sigverify.rs +++ b/perf/src/sigverify.rs @@ -9,7 +9,7 @@ use solana_sdk::transaction::Transaction; use { crate::{ cuda_runtime::PinnedVec, - packet::{Packet, PacketBatch}, + packet::{Packet, PacketBatch, PacketInterface}, perf_libs, recycler::Recycler, }, @@ -107,28 +107,28 @@ pub fn init() { } } -fn verify_packet(packet: &mut Packet, reject_non_vote: bool) { +fn verify_packet(packet: &mut P, reject_non_vote: bool) { let packet_offsets = get_packet_offsets(packet, 0, reject_non_vote); let mut sig_start = packet_offsets.sig_start as usize; let mut pubkey_start = packet_offsets.pubkey_start as usize; let msg_start = packet_offsets.msg_start as usize; // If this packet was already marked as discard, drop it - if packet.meta.discard { + if packet.get_meta().discard { return; } if packet_offsets.sig_len == 0 { - packet.meta.discard = true; + packet.get_meta_mut().discard = true; return; } - if packet.meta.size <= msg_start { - packet.meta.discard = true; + if packet.get_meta().size <= msg_start { + packet.get_meta_mut().discard = true; return; } - let msg_end = packet.meta.size; + let msg_end = packet.get_meta().size; for _ in 0..packet_offsets.sig_len { let pubkey_end = pubkey_start.saturating_add(size_of::()); let sig_end = sig_start.saturating_add(size_of::()); @@ -136,21 +136,21 @@ fn verify_packet(packet: &mut Packet, reject_non_vote: bool) { // get_packet_offsets should ensure pubkey_end and sig_end do // not overflow packet.meta.size - let signature = Signature::new(&packet.data[sig_start..sig_end]); + let signature = Signature::new(&packet.get_data()[sig_start..sig_end]); if !signature.verify( - &packet.data[pubkey_start..pubkey_end], - &packet.data[msg_start..msg_end], + &packet.get_data()[pubkey_start..pubkey_end], + &packet.get_data()[msg_start..msg_end], ) { - packet.meta.discard = true; + packet.get_meta_mut().discard = true; return; } // Check for tracer pubkey - if !packet.meta.is_tracer_tx - && &packet.data[pubkey_start..pubkey_end] == TRACER_KEY.as_ref() + if !packet.get_meta().is_tracer_tx + && &packet.get_data()[pubkey_start..pubkey_end] == TRACER_KEY.as_ref() { - packet.meta.is_tracer_tx = true; + packet.get_meta_mut().is_tracer_tx = true; } pubkey_start = pubkey_end; @@ -158,24 +158,24 @@ fn verify_packet(packet: &mut Packet, reject_non_vote: bool) { } } -pub fn count_packets_in_batches(batches: &[PacketBatch]) -> usize { +pub fn count_packets_in_batches(batches: &[PacketBatch

]) -> usize { batches.iter().map(|batch| batch.packets.len()).sum() } // internal function to be unit-tested; should be used only by get_packet_offsets -fn do_get_packet_offsets( - packet: &Packet, +fn do_get_packet_offsets( + packet: &P, current_offset: usize, ) -> Result { // should have at least 1 signature and sig lengths let _ = 1usize .checked_add(size_of::()) - .filter(|v| *v <= packet.meta.size) + .filter(|v| *v <= packet.get_meta().size) .ok_or(PacketError::InvalidLen)?; // read the length of Transaction.signatures (serialized with short_vec) let (sig_len_untrusted, sig_size) = - decode_shortu16_len(&packet.data).map_err(|_| PacketError::InvalidShortVec)?; + decode_shortu16_len(packet.get_data()).map_err(|_| PacketError::InvalidShortVec)?; // Using msg_start_offset which is based on sig_len_untrusted introduces uncertainty. // Ultimately, the actual sigverify will determine the uncertainty. @@ -187,14 +187,14 @@ fn do_get_packet_offsets( // Determine the start of the message header by checking the message prefix bit. let msg_header_offset = { // Packet should have data for prefix bit - if msg_start_offset >= packet.meta.size { + if msg_start_offset >= packet.get_meta().size { return Err(PacketError::InvalidSignatureLen); } // next byte indicates if the transaction is versioned. If the top bit // is set, the remaining bits encode a version number. If the top bit is // not set, this byte is the first byte of the message header. - let message_prefix = packet.data[msg_start_offset]; + let message_prefix = packet.get_data()[msg_start_offset]; if message_prefix & MESSAGE_VERSION_PREFIX != 0 { let version = message_prefix & !MESSAGE_VERSION_PREFIX; match version { @@ -220,11 +220,11 @@ fn do_get_packet_offsets( // Packet should have data at least for MessageHeader and 1 byte for Message.account_keys.len let _ = msg_header_offset_plus_one .checked_add(MESSAGE_HEADER_LENGTH) - .filter(|v| *v <= packet.meta.size) + .filter(|v| *v <= packet.get_meta().size) .ok_or(PacketError::InvalidSignatureLen)?; // read MessageHeader.num_required_signatures (serialized with u8) - let sig_len_maybe_trusted = packet.data[msg_header_offset]; + let sig_len_maybe_trusted = packet.get_data()[msg_header_offset]; let message_account_keys_len_offset = msg_header_offset .checked_add(MESSAGE_HEADER_LENGTH) @@ -235,7 +235,7 @@ fn do_get_packet_offsets( // num_readonly_signed_accounts, the first account is not debitable, and cannot be charged // required transaction fees. let readonly_signer_offset = msg_header_offset_plus_one; - if sig_len_maybe_trusted <= packet.data[readonly_signer_offset] { + if sig_len_maybe_trusted <= packet.get_data()[readonly_signer_offset] { return Err(PacketError::PayerNotWritable); } @@ -245,7 +245,7 @@ fn do_get_packet_offsets( // read the length of Message.account_keys (serialized with short_vec) let (pubkey_len, pubkey_len_size) = - decode_shortu16_len(&packet.data[message_account_keys_len_offset..]) + decode_shortu16_len(&packet.get_data()[message_account_keys_len_offset..]) .map_err(|_| PacketError::InvalidShortVec)?; let pubkey_start = message_account_keys_len_offset @@ -255,7 +255,7 @@ fn do_get_packet_offsets( let _ = pubkey_len .checked_mul(size_of::()) .and_then(|v| v.checked_add(pubkey_start)) - .filter(|v| *v <= packet.meta.size) + .filter(|v| *v <= packet.get_meta().size) .ok_or(PacketError::InvalidPubkeyLen)?; if pubkey_len < sig_len_untrusted { @@ -281,15 +281,15 @@ fn do_get_packet_offsets( )) } -fn get_packet_offsets( - packet: &mut Packet, +fn get_packet_offsets( + packet: &mut P, current_offset: usize, reject_non_vote: bool, ) -> PacketOffsets { let unsanitized_packet_offsets = do_get_packet_offsets(packet, current_offset); if let Ok(offsets) = unsanitized_packet_offsets { check_for_simple_vote_transaction(packet, &offsets, current_offset).ok(); - if !reject_non_vote || packet.meta.is_simple_vote_tx { + if !reject_non_vote || packet.get_meta().is_simple_vote_tx { return offsets; } } @@ -297,8 +297,8 @@ fn get_packet_offsets( PacketOffsets::new(0, 0, 0, 0, 0) } -fn check_for_simple_vote_transaction( - packet: &mut Packet, +fn check_for_simple_vote_transaction( + packet: &mut P, packet_offsets: &PacketOffsets, current_offset: usize, ) -> Result<(), PacketError> { @@ -321,11 +321,11 @@ fn check_for_simple_vote_transaction( // Packet should have at least 1 more byte for instructions.len let _ = instructions_len_offset .checked_add(1usize) - .filter(|v| *v <= packet.meta.size) + .filter(|v| *v <= packet.get_meta().size) .ok_or(PacketError::InvalidLen)?; let (instruction_len, instruction_len_size) = - decode_shortu16_len(&packet.data[instructions_len_offset..]) + decode_shortu16_len(&packet.get_data()[instructions_len_offset..]) .map_err(|_| PacketError::InvalidLen)?; // skip if has more than 1 instruction @@ -340,10 +340,10 @@ fn check_for_simple_vote_transaction( // Packet should have at least 1 more byte for one instructions_program_id let _ = instruction_start .checked_add(1usize) - .filter(|v| *v <= packet.meta.size) + .filter(|v| *v <= packet.get_meta().size) .ok_or(PacketError::InvalidLen)?; - let instruction_program_id_index: usize = usize::from(packet.data[instruction_start]); + let instruction_program_id_index: usize = usize::from(packet.get_data()[instruction_start]); if instruction_program_id_index >= packet_offsets.pubkey_len as usize { return Err(PacketError::InvalidProgramIdIndex); @@ -357,16 +357,16 @@ fn check_for_simple_vote_transaction( .checked_add(size_of::()) .ok_or(PacketError::InvalidLen)?; - if &packet.data[instruction_program_id_start..instruction_program_id_end] + if &packet.get_data()[instruction_program_id_start..instruction_program_id_end] == solana_sdk::vote::program::id().as_ref() { - packet.meta.is_simple_vote_tx = true; + packet.get_meta_mut().is_simple_vote_tx = true; } Ok(()) } -pub fn generate_offsets( - batches: &mut [PacketBatch], +pub fn generate_offsets( + batches: &mut [PacketBatch

], recycler: &Recycler, reject_non_vote: bool, ) -> TxOffsets { @@ -392,7 +392,7 @@ pub fn generate_offsets( let mut pubkey_offset = packet_offsets.pubkey_start; let mut sig_offset = packet_offsets.sig_start; - let msg_size = current_offset.saturating_add(packet.meta.size) as u32; + let msg_size = current_offset.saturating_add(packet.get_meta().size) as u32; for _ in 0..packet_offsets.sig_len { signature_offsets.push(sig_offset); sig_offset = sig_offset.saturating_add(size_of::() as u32); @@ -418,7 +418,10 @@ pub fn generate_offsets( ) } -pub fn ed25519_verify_cpu(batches: &mut [PacketBatch], reject_non_vote: bool) { +pub fn ed25519_verify_cpu( + batches: &mut [PacketBatch

], + reject_non_vote: bool, +) { use rayon::prelude::*; let packet_count = count_packets_in_batches(batches); debug!("CPU ECDSA for {}", packet_count); @@ -433,7 +436,7 @@ pub fn ed25519_verify_cpu(batches: &mut [PacketBatch], reject_non_vote: bool) { inc_new_counter_debug!("ed25519_verify_cpu", packet_count); } -pub fn ed25519_verify_disabled(batches: &mut [PacketBatch]) { +pub fn ed25519_verify_disabled(batches: &mut [PacketBatch

]) { use rayon::prelude::*; let packet_count = count_packets_in_batches(batches); debug!("disabled ECDSA for {}", packet_count); @@ -441,7 +444,7 @@ pub fn ed25519_verify_disabled(batches: &mut [PacketBatch]) { batch .packets .par_iter_mut() - .for_each(|p| p.meta.discard = false) + .for_each(|p| p.get_meta_mut().discard = false) }); inc_new_counter_debug!("ed25519_verify_disabled", packet_count); } @@ -497,16 +500,27 @@ pub fn get_checked_scalar(scalar: &[u8; 32]) -> Result<[u8; 32], PacketError> { Ok(out) } -pub fn mark_disabled(batches: &mut [PacketBatch], r: &[Vec]) { +pub fn mark_disabled(batches: &mut [PacketBatch

], r: &[Vec]) { batches.iter_mut().zip(r).for_each(|(b, v)| { b.packets.iter_mut().zip(v).for_each(|(p, f)| { - p.meta.discard = *f == 0; + p.get_meta_mut().discard = *f == 0; }) }); } -pub fn ed25519_verify( - batches: &mut [PacketBatch], +#[cfg(test)] +pub fn make_packet_from_transaction(tx: Transaction) -> Packet { + use bincode::serialize; + + let tx_bytes = serialize(&tx).unwrap(); + let mut packet = Packet::default(); + packet.meta.size = tx_bytes.len(); + packet.data[..packet.meta.size].copy_from_slice(&tx_bytes); + packet +} + +pub fn ed25519_verify( + batches: &mut [PacketBatch

], recycler: &Recycler, recycler_out: &Recycler>, reject_non_vote: bool, @@ -542,7 +556,8 @@ pub fn ed25519_verify( let mut num_packets: usize = 0; for batch in batches.iter() { elems.push(perf_libs::Elems { - elems: batch.packets.as_ptr(), + // todo: verify + elems: batch.packets.as_ptr().cast::(), num: batch.packets.len() as u32, }); let mut v = Vec::new(); @@ -551,16 +566,24 @@ pub fn ed25519_verify( num_packets = num_packets.saturating_add(batch.packets.len()); } out.resize(signature_offsets.len(), 0); - trace!("Starting verify num packets: {}", num_packets); - trace!("elem len: {}", elems.len() as u32); - trace!("packet sizeof: {}", size_of::() as u32); + warn!( + "Starting verify {}, num packets: {}", + P::get_packet_type_name(), + num_packets + ); + warn!("elem len: {}", elems.len() as u32); + warn!("packet sizeof: {}", size_of::

() as u32); + // todo: figure out what this means; the data field max size + // does not actually seem to be used in figuring out any of the offsets + // (which would make sense, as we don't modify the meta + // directly on the GPU)... trace!("len offset: {}", PACKET_DATA_SIZE as u32); const USE_NON_DEFAULT_STREAM: u8 = 1; unsafe { let res = (api.ed25519_verify_many)( elems.as_ptr(), elems.len() as u32, - size_of::() as u32, + size_of::

() as u32, num_packets as u32, signature_offsets.len() as u32, msg_sizes.as_ptr(), diff --git a/rpc/src/cluster_tpu_info.rs b/rpc/src/cluster_tpu_info.rs index 5421bc7b2b9b3d..c8e266a06e6e69 100644 --- a/rpc/src/cluster_tpu_info.rs +++ b/rpc/src/cluster_tpu_info.rs @@ -13,7 +13,7 @@ use { pub struct ClusterTpuInfo { cluster_info: Arc, poh_recorder: Arc>, - recent_peers: HashMap, + recent_peers: HashMap, } impl ClusterTpuInfo { @@ -32,11 +32,11 @@ impl TpuInfo for ClusterTpuInfo { .cluster_info .tpu_peers() .into_iter() - .map(|ci| (ci.id, ci.tpu)) + .map(|ci| (ci.id, (ci.tpu, ci.tpu_extended))) .collect(); } - fn get_leader_tpus(&self, max_count: u64) -> Vec<&SocketAddr> { + fn get_leader_tpus(&self, extended: bool, max_count: u64) -> Vec { let recorder = self.poh_recorder.lock().unwrap(); let leaders: Vec<_> = (0..max_count) .filter_map(|i| recorder.leader_after_n_slots(i * NUM_CONSECUTIVE_LEADER_SLOTS)) @@ -44,7 +44,10 @@ impl TpuInfo for ClusterTpuInfo { drop(recorder); let mut unique_leaders = vec![]; for leader in leaders.iter() { - if let Some(addr) = self.recent_peers.get(leader) { + // TOOD: There was some funny business with pulling SocketAddr out of + // (SocketAddr, SocketAddr); figure that out and use borrows again + if let Some(addr_pair) = self.recent_peers.get(leader) { + let addr = if extended { addr_pair.1 } else { addr_pair.0 }; if !unique_leaders.contains(&addr) { unique_leaders.push(addr); } diff --git a/rpc/src/rpc.rs b/rpc/src/rpc.rs index 987980cb535def..d467da16a10aaa 100644 --- a/rpc/src/rpc.rs +++ b/rpc/src/rpc.rs @@ -37,7 +37,6 @@ use { leader_schedule_cache::LeaderScheduleCache, }, solana_metrics::inc_new_counter_info, - solana_perf::packet::PACKET_DATA_SIZE, solana_runtime::{ accounts::AccountAddressFilter, accounts_index::{AccountIndex, AccountSecondaryIndexes, IndexKey, ScanConfig}, @@ -61,6 +60,7 @@ use { fee_calculator::FeeCalculator, hash::Hash, message::{Message, SanitizedMessage}, + packet::MAX_TRANSACTION_SIZE, pubkey::{Pubkey, PUBKEY_BYTES}, signature::{Keypair, Signature, Signer}, stake::state::{StakeActivationStatus, StakeState}, @@ -301,9 +301,11 @@ impl JsonRpcRequestProcessor { socket_addr_space, )); let tpu_address = cluster_info.my_contact_info().tpu; + let tpu_extended_address = cluster_info.my_contact_info().tpu_extended; let (sender, receiver) = channel(); SendTransactionService::new::( tpu_address, + tpu_extended_address, &bank_forks, None, receiver, @@ -4177,8 +4179,12 @@ pub mod rpc_obsolete_v1_7 { } } -const MAX_BASE58_SIZE: usize = 1683; // Golden, bump if PACKET_DATA_SIZE changes -const MAX_BASE64_SIZE: usize = 1644; // Golden, bump if PACKET_DATA_SIZE changes +// The below contants are based on MAX_TRANSACTION_SIZE; any changes to +// MAX_TRANSACTION_SIZE will require an update to the below constants. +// The correct values can be found by hand or by encoding MAX_TRANSACTION_SIZE bytes. +// `test_max_encoded_tx_len_goldens` ensures these values are correct +const MAX_BASE58_SIZE: usize = 3365; +const MAX_BASE64_SIZE: usize = 3288; fn decode_and_deserialize( encoded: String, encoding: UiTransactionEncoding, @@ -4195,7 +4201,7 @@ where type_name::(), encoded.len(), MAX_BASE58_SIZE, - PACKET_DATA_SIZE, + MAX_TRANSACTION_SIZE, ))); } bs58::decode(encoded) @@ -4210,7 +4216,7 @@ where type_name::(), encoded.len(), MAX_BASE64_SIZE, - PACKET_DATA_SIZE, + MAX_TRANSACTION_SIZE, ))); } base64::decode(encoded).map_err(|e| Error::invalid_params(format!("{:?}", e)))? @@ -4222,18 +4228,18 @@ where ))) } }; - if wire_output.len() > PACKET_DATA_SIZE { + if wire_output.len() > MAX_TRANSACTION_SIZE { let err = format!( "encoded {} too large: {} bytes (max: {} bytes)", type_name::(), wire_output.len(), - PACKET_DATA_SIZE + MAX_TRANSACTION_SIZE ); info!("{}", err); return Err(Error::invalid_params(&err)); } bincode::options() - .with_limit(PACKET_DATA_SIZE as u64) + .with_limit(MAX_TRANSACTION_SIZE as u64) .with_fixint_encoding() .allow_trailing_bytes() .deserialize_from(&wire_output[..]) @@ -4246,7 +4252,7 @@ where fn sanitize_transaction(transaction: VersionedTransaction) -> Result { let message_hash = transaction.message.hash(); - SanitizedTransaction::try_create(transaction, message_hash, None, |_| { + SanitizedTransaction::try_create(transaction, message_hash, None, None, |_| { Err(TransactionError::UnsupportedVersion) }) .map_err(|err| Error::invalid_params(format!("invalid transaction: {}", err))) @@ -7961,8 +7967,8 @@ pub mod tests { } #[test] - fn test_worst_case_encoded_tx_goldens() { - let ff_tx = vec![0xffu8; PACKET_DATA_SIZE]; + fn test_max_encoded_tx_len_goldens() { + let ff_tx = vec![0xffu8; MAX_TRANSACTION_SIZE]; let tx58 = bs58::encode(&ff_tx).into_string(); assert_eq!(tx58.len(), MAX_BASE58_SIZE); let tx64 = base64::encode(&ff_tx); @@ -7971,14 +7977,16 @@ pub mod tests { #[test] fn test_decode_and_deserialize_too_large_payloads_fail() { - // +2 because +1 still fits in base64 encoded worst-case - let too_big = PACKET_DATA_SIZE + 2; + // 3 bytes (base16) is 24 bits which can be represented by 4 base64 digits + // Those 4 base64 digits will be generated even if there are only 1 or 2 bytes + // So, to guarantee that there will be extra base64 digits, add 3 extra bytes + let too_big = MAX_TRANSACTION_SIZE + 3; let tx_ser = vec![0xffu8; too_big]; let tx58 = bs58::encode(&tx_ser).into_string(); let tx58_len = tx58.len(); let expect58 = Error::invalid_params(format!( "encoded solana_sdk::transaction::Transaction too large: {} bytes (max: encoded/raw {}/{})", - tx58_len, MAX_BASE58_SIZE, PACKET_DATA_SIZE, + tx58_len, MAX_BASE58_SIZE, MAX_TRANSACTION_SIZE, )); assert_eq!( decode_and_deserialize::(tx58, UiTransactionEncoding::Base58).unwrap_err(), @@ -7988,18 +7996,18 @@ pub mod tests { let tx64_len = tx64.len(); let expect64 = Error::invalid_params(format!( "encoded solana_sdk::transaction::Transaction too large: {} bytes (max: encoded/raw {}/{})", - tx64_len, MAX_BASE64_SIZE, PACKET_DATA_SIZE, + tx64_len, MAX_BASE64_SIZE, MAX_TRANSACTION_SIZE, )); assert_eq!( decode_and_deserialize::(tx64, UiTransactionEncoding::Base64).unwrap_err(), expect64 ); - let too_big = PACKET_DATA_SIZE + 1; + let too_big = MAX_TRANSACTION_SIZE + 1; let tx_ser = vec![0x00u8; too_big]; let tx58 = bs58::encode(&tx_ser).into_string(); let expect = Error::invalid_params(format!( "encoded solana_sdk::transaction::Transaction too large: {} bytes (max: {} bytes)", - too_big, PACKET_DATA_SIZE + too_big, MAX_TRANSACTION_SIZE )); assert_eq!( decode_and_deserialize::(tx58, UiTransactionEncoding::Base58).unwrap_err(), diff --git a/rpc/src/rpc_service.rs b/rpc/src/rpc_service.rs index 14e167de402a93..63daad2068278a 100644 --- a/rpc/src/rpc_service.rs +++ b/rpc/src/rpc_service.rs @@ -323,6 +323,7 @@ impl JsonRpcService { ))); let tpu_address = cluster_info.my_contact_info().tpu; + let tpu_extended_address = cluster_info.my_contact_info().tpu_extended; // sadly, some parts of our current rpc implemention block the jsonrpc's // _socket-listening_ event loop for too long, due to (blocking) long IO or intesive CPU, @@ -403,6 +404,7 @@ impl JsonRpcService { poh_recorder.map(|recorder| ClusterTpuInfo::new(cluster_info.clone(), recorder)); let _send_transaction_service = Arc::new(SendTransactionService::new_with_config( tpu_address, + tpu_extended_address, &bank_forks, leader_info, receiver, diff --git a/rpc/src/transaction_status_service.rs b/rpc/src/transaction_status_service.rs index 9523df641ee765..4337d2a6a01748 100644 --- a/rpc/src/transaction_status_service.rs +++ b/rpc/src/transaction_status_service.rs @@ -308,7 +308,7 @@ pub(crate) mod tests { let transaction = build_test_transaction_legacy(); let transaction = VersionedTransaction::from(transaction); let transaction = - SanitizedTransaction::try_create(transaction, message_hash, Some(true), |_| { + SanitizedTransaction::try_create(transaction, message_hash, Some(true), None, |_| { Err(TransactionError::UnsupportedVersion) }) .unwrap(); diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index d7ac6745658ce0..b412d0da41886c 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -109,7 +109,7 @@ use { native_loader, native_token::sol_to_lamports, nonce, nonce_account, - packet::PACKET_DATA_SIZE, + packet::EXTENDED_PACKET_DATA_SIZE, precompiles::get_precompiles, program_utils::limited_deserialize, pubkey::Pubkey, @@ -3080,7 +3080,7 @@ impl Bank { .into_iter() .map(|tx| { let message_hash = tx.message.hash(); - SanitizedTransaction::try_create(tx, message_hash, None, |_| { + SanitizedTransaction::try_create(tx, message_hash, None, None, |_| { Err(TransactionError::UnsupportedVersion) }) }) @@ -5232,7 +5232,7 @@ impl Bank { let sanitized_tx = { let size = bincode::serialized_size(&tx).map_err(|_| TransactionError::SanitizeFailure)?; - if size > PACKET_DATA_SIZE as u64 { + if size > EXTENDED_PACKET_DATA_SIZE as u64 { return Err(TransactionError::SanitizeFailure); } let message_hash = if verification_mode == TransactionVerificationMode::FullVerification @@ -5242,7 +5242,7 @@ impl Bank { tx.message.hash() }; - SanitizedTransaction::try_create(tx, message_hash, None, |_| { + SanitizedTransaction::try_create(tx, message_hash, None, Some(size), |_| { Err(TransactionError::UnsupportedVersion) }) }?; diff --git a/sdk/src/packet.rs b/sdk/src/packet.rs index 19b0bae996d493..405e71529f4d98 100644 --- a/sdk/src/packet.rs +++ b/sdk/src/packet.rs @@ -14,6 +14,54 @@ use { /// 8 bytes is the size of the fragment header pub const PACKET_DATA_SIZE: usize = 1280 - 40 - 8; +pub const EXTENDED_PACKET_DATA_SIZE: usize = PACKET_DATA_SIZE * 2; + +pub const MAX_TRANSACTION_SIZE: usize = EXTENDED_PACKET_DATA_SIZE; +#[cfg(test)] +static_assertions::const_assert_eq!(EXTENDED_PACKET_DATA_SIZE > PACKET_DATA_SIZE); + +// Send + Sync needed to allow Rayon par_iter() +/// Generic interface to support variable sized packtes +pub trait PacketInterface: Clone + Default + Sized + Send + Sync + fmt::Debug { + fn get_data(&self) -> &[u8]; + + fn get_data_mut(&mut self) -> &mut [u8]; + + fn get_meta(&self) -> &Meta; + + fn get_meta_mut(&mut self) -> &mut Meta; + + fn from_data(dest: Option<&SocketAddr>, data: T) -> Result { + let mut packet = Self::default(); + Self::populate_packet(&mut packet, dest, &data)?; + Ok(packet) + } + + fn populate_packet( + packet: &mut Self, + dest: Option<&SocketAddr>, + data: &T, + ) -> Result<()> { + let mut wr = io::Cursor::new(&mut *packet.get_data_mut()); + bincode::serialize_into(&mut wr, data)?; + let len = wr.position() as usize; + packet.get_meta_mut().size = len; + if let Some(dest) = dest { + packet.get_meta_mut().set_addr(dest); + } + Ok(()) + } + + // Hack to allow the introduction of special logic + // in necessary places and work around Rust's lack of generic specialization + // or similar "compile-time conditionals" + // TODO: is there a better way to do this (perhaps a macro of some sort?)? + fn is_extended() -> bool; + + // todo: ideally, replace this with a macro or something + fn get_packet_type_name() -> &'static str; +} + #[derive(Clone, Default, Debug, PartialEq)] #[repr(C)] pub struct Meta { @@ -37,30 +85,63 @@ pub struct Packet { pub meta: Meta, } -impl Packet { - pub fn new(data: [u8; PACKET_DATA_SIZE], meta: Meta) -> Self { - Self { data, meta } +// TODO: can we de-duplicate some of this Packet and ExtendedPacket code? +#[derive(Clone)] +#[repr(C)] +pub struct ExtendedPacket { + pub data: [u8; EXTENDED_PACKET_DATA_SIZE], + pub meta: Meta, +} + +impl PacketInterface for ExtendedPacket { + fn get_data(&self) -> &[u8] { + &self.data } - pub fn from_data(dest: Option<&SocketAddr>, data: T) -> Result { - let mut packet = Packet::default(); - Self::populate_packet(&mut packet, dest, &data)?; - Ok(packet) + fn get_data_mut(&mut self) -> &mut [u8] { + &mut self.data } - pub fn populate_packet( - packet: &mut Packet, - dest: Option<&SocketAddr>, - data: &T, - ) -> Result<()> { - let mut wr = io::Cursor::new(&mut packet.data[..]); - bincode::serialize_into(&mut wr, data)?; - let len = wr.position() as usize; - packet.meta.size = len; - if let Some(dest) = dest { - packet.meta.set_addr(dest); - } - Ok(()) + fn get_meta(&self) -> &Meta { + &self.meta + } + + fn get_meta_mut(&mut self) -> &mut Meta { + &mut self.meta + } + + fn is_extended() -> bool { + true + } + + fn get_packet_type_name() -> &'static str { + "ExtendedPacket" + } +} + +impl PacketInterface for Packet { + fn get_data(&self) -> &[u8] { + &self.data + } + + fn get_data_mut(&mut self) -> &mut [u8] { + &mut self.data + } + + fn get_meta(&self) -> &Meta { + &self.meta + } + + fn get_meta_mut(&mut self) -> &mut Meta { + &mut self.meta + } + + fn is_extended() -> bool { + false + } + + fn get_packet_type_name() -> &'static str { + "Packet" } } @@ -93,6 +174,37 @@ impl PartialEq for Packet { } } +impl fmt::Debug for ExtendedPacket { + // It may be useful to know the type of Packet in the debug + // print + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "ExtendedPacket {{ size: {:?}, addr: {:?} }}", + self.meta.size, + self.meta.addr() + ) + } +} + +#[allow(clippy::uninit_assumed_init)] +impl Default for ExtendedPacket { + fn default() -> ExtendedPacket { + ExtendedPacket { + data: unsafe { std::mem::MaybeUninit::uninit().assume_init() }, + meta: Meta::default(), + } + } +} + +impl PartialEq for ExtendedPacket { + fn eq(&self, other: &ExtendedPacket) -> bool { + let self_data: &[u8] = self.data.as_ref(); + let other_data: &[u8] = other.data.as_ref(); + self.meta == other.meta && self_data[..self.meta.size] == other_data[..self.meta.size] + } +} + impl Meta { pub fn addr(&self) -> SocketAddr { if !self.v6 { diff --git a/sdk/src/program_utils.rs b/sdk/src/program_utils.rs index 590e81ae39007b..a069d1ed9b16f8 100644 --- a/sdk/src/program_utils.rs +++ b/sdk/src/program_utils.rs @@ -6,7 +6,11 @@ pub fn limited_deserialize(instruction_data: &[u8]) -> Result, } @@ -46,10 +50,18 @@ impl SanitizedTransaction { tx: VersionedTransaction, message_hash: Hash, is_simple_vote_tx: Option, + serialized_size: Option, address_loader: impl Fn(&v0::Message) -> Result, ) -> Result { tx.sanitize()?; + let size = match serialized_size { + Some(size) => size, + None => bincode::serialized_size(&tx).map_err(|_| TransactionError::SanitizeFailure)? + }; + + let is_large_transaction = size > (PACKET_DATA_SIZE as u64); + let signatures = tx.signatures; let message = match tx.message { VersionedMessage::Legacy(message) => SanitizedMessage::Legacy(message), @@ -72,6 +84,7 @@ impl SanitizedTransaction { message, message_hash, is_simple_vote_tx, + is_large_transaction, signatures, }) } @@ -87,6 +100,7 @@ impl SanitizedTransaction { message_hash: tx.message.hash(), message: SanitizedMessage::Legacy(tx.message), is_simple_vote_tx: false, + is_large_transaction: false, signatures: tx.signatures, }) } @@ -122,6 +136,10 @@ impl SanitizedTransaction { &self.message_hash } + pub fn is_large_transaction(&self) -> bool { + self.is_large_transaction + } + /// Returns true if this transaction is a simple vote pub fn is_simple_vote_transaction(&self) -> bool { self.is_simple_vote_tx diff --git a/send-transaction-service/src/send_transaction_service.rs b/send-transaction-service/src/send_transaction_service.rs index ae498e8f9ac4ca..0ec2099d19ac34 100644 --- a/send-transaction-service/src/send_transaction_service.rs +++ b/send-transaction-service/src/send_transaction_service.rs @@ -3,7 +3,9 @@ use { log::*, solana_metrics::{datapoint_warn, inc_new_counter_info}, solana_runtime::{bank::Bank, bank_forks::BankForks}, - solana_sdk::{hash::Hash, nonce_account, pubkey::Pubkey, signature::Signature}, + solana_sdk::{ + hash::Hash, nonce_account, packet::PACKET_DATA_SIZE, pubkey::Pubkey, signature::Signature, + }, std::{ collections::HashMap, net::{SocketAddr, UdpSocket}, @@ -89,6 +91,7 @@ impl Default for Config { impl SendTransactionService { pub fn new( tpu_address: SocketAddr, + tpu_extended_address: SocketAddr, bank_forks: &Arc>, leader_info: Option, receiver: Receiver, @@ -100,11 +103,19 @@ impl SendTransactionService { leader_forward_count, ..Config::default() }; - Self::new_with_config(tpu_address, bank_forks, leader_info, receiver, config) + Self::new_with_config( + tpu_address, + tpu_extended_address, + bank_forks, + leader_info, + receiver, + config, + ) } pub fn new_with_config( tpu_address: SocketAddr, + tpu_extended_address: SocketAddr, bank_forks: &Arc>, leader_info: Option, receiver: Receiver, @@ -112,6 +123,7 @@ impl SendTransactionService { ) -> Self { let thread = Self::retry_thread( tpu_address, + tpu_extended_address, receiver, bank_forks.clone(), leader_info, @@ -122,6 +134,7 @@ impl SendTransactionService { fn retry_thread( tpu_address: SocketAddr, + tpu_extended_address: SocketAddr, receiver: Receiver, bank_forks: Arc>, mut leader_info: Option, @@ -144,22 +157,34 @@ impl SendTransactionService { Err(RecvTimeoutError::Timeout) => {} Ok(transaction_info) => { inc_new_counter_info!("send_transaction_service-recv-tx", 1); + let extended = Self::requires_tpu_extended(&transaction_info); + let choose_tpu = if extended { + tpu_extended_address + } else { + tpu_address + }; let addresses = leader_info.as_ref().map(|leader_info| { - leader_info.get_leader_tpus(config.leader_forward_count) + leader_info.get_leader_tpus(extended, config.leader_forward_count) }); let addresses = addresses .map(|address_list| { if address_list.is_empty() { - vec![&tpu_address] + vec![choose_tpu] } else { address_list } }) - .unwrap_or_else(|| vec![&tpu_address]); + .unwrap_or_else(|| vec![choose_tpu]); + if extended { + warn!("retry_thread: Forwarding large transaction to extended packet addresses {:?}", addresses); + } + else { + warn!("retry_thread: Forwarding transaction to standard packet addresses {:?}", addresses); + } for address in addresses { Self::send_transaction( &send_socket, - address, + &address, &transaction_info.wire_transaction, ); } @@ -191,6 +216,7 @@ impl SendTransactionService { &root_bank, &send_socket, &tpu_address, + &tpu_extended_address, &mut transactions, &leader_info, &config, @@ -213,6 +239,7 @@ impl SendTransactionService { root_bank: &Arc, send_socket: &UdpSocket, tpu_address: &SocketAddr, + tpu_extended_address: &SocketAddr, transactions: &mut HashMap, leader_info: &Option, config: &Config, @@ -269,22 +296,34 @@ impl SendTransactionService { result.retried += 1; transaction_info.retries += 1; inc_new_counter_info!("send_transaction_service-retry", 1); + let extended = Self::requires_tpu_extended(transaction_info); + let choose_tpu = if extended { + tpu_extended_address + } else { + tpu_address + }; let addresses = leader_info.as_ref().map(|leader_info| { - leader_info.get_leader_tpus(config.leader_forward_count) + leader_info.get_leader_tpus(extended, config.leader_forward_count) }); let addresses = addresses .map(|address_list| { if address_list.is_empty() { - vec![tpu_address] + vec![*choose_tpu] } else { address_list } }) - .unwrap_or_else(|| vec![tpu_address]); + .unwrap_or_else(|| vec![*choose_tpu]); + if extended { + warn!("process_transactions: Forwarding large transaction to extended packet addresses {:?}", addresses); + } + else { + warn!("process_transactions: Forwarding transaction to standard packet addresses {:?}", addresses); + } for address in addresses { Self::send_transaction( send_socket, - address, + &address, &transaction_info.wire_transaction, ); } @@ -307,6 +346,10 @@ impl SendTransactionService { result } + fn requires_tpu_extended(transaction_info: &TransactionInfo) -> bool { + transaction_info.wire_transaction.len() > PACKET_DATA_SIZE + } + fn send_transaction( send_socket: &UdpSocket, tpu_address: &SocketAddr, diff --git a/send-transaction-service/src/tpu_info.rs b/send-transaction-service/src/tpu_info.rs index 02bccb96414a1f..c58273b37fb0dc 100644 --- a/send-transaction-service/src/tpu_info.rs +++ b/send-transaction-service/src/tpu_info.rs @@ -2,14 +2,14 @@ use std::net::SocketAddr; pub trait TpuInfo { fn refresh_recent_peers(&mut self); - fn get_leader_tpus(&self, max_count: u64) -> Vec<&SocketAddr>; + fn get_leader_tpus(&self, extended: bool, max_count: u64) -> Vec; } pub struct NullTpuInfo; impl TpuInfo for NullTpuInfo { fn refresh_recent_peers(&mut self) {} - fn get_leader_tpus(&self, _max_count: u64) -> Vec<&SocketAddr> { + fn get_leader_tpus(&self, _extended: bool, _max_count: u64) -> Vec { vec![] } } diff --git a/streamer/Cargo.toml b/streamer/Cargo.toml index bce936e59cf6b2..4fa40168ff18a2 100644 --- a/streamer/Cargo.toml +++ b/streamer/Cargo.toml @@ -19,6 +19,11 @@ thiserror = "1.0" libc = "0.2.112" nix = "0.23.1" solana-perf = { path = "../perf", version = "=1.10.0" } +quinn = "0.8.0" +rustls = {version = "0.20.0", features = ["dangerous_configuration"]} +anyhow = "1.0.22" +bytes = "1" +tokio = { version = "1", features = ["full"] } [dev-dependencies] solana-logger = { path = "../logger", version = "=1.10.0" } diff --git a/streamer/src/packet.rs b/streamer/src/packet.rs index b0abe551a3a2ef..e4f7b59866e912 100644 --- a/streamer/src/packet.rs +++ b/streamer/src/packet.rs @@ -12,10 +12,14 @@ pub use { limited_deserialize, to_packet_batches, PacketBatch, PacketBatchRecycler, NUM_PACKETS, PACKETS_PER_BATCH, }, - solana_sdk::packet::{Meta, Packet, PACKET_DATA_SIZE}, + solana_sdk::packet::{Meta, Packet, PacketInterface, PACKET_DATA_SIZE}, }; -pub fn recv_from(batch: &mut PacketBatch, socket: &UdpSocket, max_wait_ms: u64) -> Result { +pub fn recv_from( + batch: &mut PacketBatch

, + socket: &UdpSocket, + max_wait_ms: u64, +) -> Result { let mut i = 0; //DOCUMENTED SIDE-EFFECT //Performance out of the IO without poll @@ -29,7 +33,7 @@ pub fn recv_from(batch: &mut PacketBatch, socket: &UdpSocket, max_wait_ms: u64) loop { batch.packets.resize( std::cmp::min(i + NUM_RCVMMSGS, PACKETS_PER_BATCH), - Packet::default(), + P::default(), ); match recv_mmsg(socket, &mut batch.packets[i..]) { Err(_) if i > 0 => { @@ -60,15 +64,15 @@ pub fn recv_from(batch: &mut PacketBatch, socket: &UdpSocket, max_wait_ms: u64) Ok(i) } -pub fn send_to( - batch: &PacketBatch, +pub fn send_to( + batch: &PacketBatch

, socket: &UdpSocket, socket_addr_space: &SocketAddrSpace, ) -> Result<()> { for p in &batch.packets { - let addr = p.meta.addr(); + let addr = p.get_meta().addr(); if socket_addr_space.check(&addr) { - socket.send_to(&p.data[..p.meta.size], &addr)?; + socket.send_to(&p.get_data()[..p.get_meta().size], &addr)?; } } Ok(()) diff --git a/streamer/src/recvmmsg.rs b/streamer/src/recvmmsg.rs index 897ba07990002f..12c1f3bed26208 100644 --- a/streamer/src/recvmmsg.rs +++ b/streamer/src/recvmmsg.rs @@ -2,7 +2,7 @@ pub use solana_perf::packet::NUM_RCVMMSGS; use { - crate::packet::Packet, + crate::packet::PacketInterface, std::{cmp, io, net::UdpSocket}, }; #[cfg(target_os = "linux")] @@ -14,13 +14,16 @@ use { }; #[cfg(not(target_os = "linux"))] -pub fn recv_mmsg(socket: &UdpSocket, packets: &mut [Packet]) -> io::Result<(usize, usize)> { +pub fn recv_mmsg( + socket: &UdpSocket, + packets: &mut [P], +) -> io::Result<(usize, usize)> { let mut i = 0; let count = cmp::min(NUM_RCVMMSGS, packets.len()); let mut total_size = 0; for p in packets.iter_mut().take(count) { - p.meta.size = 0; - match socket.recv_from(&mut p.data) { + p.get_meta_mut().size = 0; + match socket.recv_from(p.get_data_mut()) { Err(_) if i > 0 => { break; } @@ -29,8 +32,8 @@ pub fn recv_mmsg(socket: &UdpSocket, packets: &mut [Packet]) -> io::Result<(usiz } Ok((nrecv, from)) => { total_size += nrecv; - p.meta.size = nrecv; - p.meta.set_addr(&from); + p.get_meta_mut().size = nrecv; + p.get_meta_mut().set_addr(&from); if i == 0 { socket.set_nonblocking(true)?; } @@ -67,7 +70,10 @@ fn cast_socket_addr(addr: &sockaddr_storage, hdr: &mmsghdr) -> Option #[cfg(target_os = "linux")] #[allow(clippy::uninit_assumed_init)] -pub fn recv_mmsg(sock: &UdpSocket, packets: &mut [Packet]) -> io::Result<(usize, usize)> { +pub fn recv_mmsg( + sock: &UdpSocket, + packets: &mut [P], +) -> io::Result<(usize, usize)> { const SOCKADDR_STORAGE_SIZE: usize = mem::size_of::(); let mut hdrs: [mmsghdr; NUM_RCVMMSGS] = unsafe { mem::zeroed() }; @@ -81,8 +87,8 @@ pub fn recv_mmsg(sock: &UdpSocket, packets: &mut [Packet]) -> io::Result<(usize, izip!(packets.iter_mut(), &mut hdrs, &mut iovs, &mut addrs).take(count) { *iov = iovec { - iov_base: packet.data.as_mut_ptr() as *mut libc::c_void, - iov_len: packet.data.len(), + iov_base: packet.get_data_mut().as_mut_ptr() as *mut libc::c_void, + iov_len: packet.get_data().len(), }; hdr.msg_hdr.msg_name = addr as *mut _ as *mut _; hdr.msg_hdr.msg_namelen = SOCKADDR_STORAGE_SIZE as socklen_t; @@ -109,11 +115,21 @@ pub fn recv_mmsg(sock: &UdpSocket, packets: &mut [Packet]) -> io::Result<(usize, }) .zip(packets.iter_mut()) .for_each(|((addr, hdr), pkt)| { - pkt.meta.size = hdr.msg_len as usize; - pkt.meta.set_addr(&addr); + pkt.get_meta_mut().size = hdr.msg_len as usize; + pkt.get_meta_mut().set_addr(&addr); npkts += 1; }); - let total_size = packets.iter().take(npkts).map(|pkt| pkt.meta.size).sum(); + // TODO: Not part of supporting larger transactions but revisit / make an issue for this + // - Could total_size be accumulated in above loop instead of running through packets below? + // - If cast_socket_addr() returns a None, then wouldn't doing filter_map with zip afterwards + // misalign (addr, hdr) and packet? If we get a None, there will be fewer zipped (addr, hdr) + // pairs, and when we zip with packets, anything after the bad (addr, hdr) will be zipped + // with the wrong packet ... I think + let total_size = packets + .iter() + .take(npkts) + .map(|pkt| pkt.get_meta().size) + .sum(); Ok((total_size, npkts)) } diff --git a/streamer/src/sendmmsg.rs b/streamer/src/sendmmsg.rs index 6e434f5ed5e27e..84ce24c7655b6b 100644 --- a/streamer/src/sendmmsg.rs +++ b/streamer/src/sendmmsg.rs @@ -13,8 +13,15 @@ use { io, iter::repeat, net::{SocketAddr, UdpSocket}, + sync::Arc, }, thiserror::Error, + quinn::{ClientConfig, Endpoint, NewConnection, EndpointConfig}, + anyhow::{Context, Result, anyhow}, + bytes::Bytes, + solana_sdk::packet::MAX_TRANSACTION_SIZE, + solana_sdk::packet::PacketInterface, + rustls::*, }; #[derive(Debug, Error)] @@ -48,6 +55,147 @@ where } } + +/// Dummy certificate verifier that treats any certificate as valid. +struct SkipServerVerification; + +impl SkipServerVerification { + fn new() -> Arc { + Arc::new(Self) + } +} + +impl rustls::client::ServerCertVerifier for SkipServerVerification { + fn verify_server_cert( + &self, + _end_entity: &rustls::Certificate, + _intermediates: &[rustls::Certificate], + _server_name: &rustls::ServerName, + _scts: &mut dyn Iterator, + _ocsp_response: &[u8], + _now: std::time::SystemTime, + ) -> Result { + Ok(rustls::client::ServerCertVerified::assertion()) + } +} + +fn configure_client() -> ClientConfig { + let crypto = rustls::ClientConfig::builder() + .with_safe_defaults() + .with_custom_certificate_verifier(SkipServerVerification::new()) + .with_no_client_auth(); + + ClientConfig::new(Arc::new(crypto)) +} + +#[tokio::main] +pub async fn batch_send_quic(sock: &UdpSocket, packets: Vec<&P>, dest: &SocketAddr) -> Result<()> +{ + /*let url = options.url; + let remote = (url.host_str().unwrap(), url.port().unwrap_or(4433)) + .to_socket_addrs()? + .next() + .ok_or_else(|| anyhow!("couldn't resolve to an address"))?;*/ +/* + let mut roots = rustls::RootCertStore::empty(); + if let Some(ca_path) = options.ca { + roots.add(&rustls::Certificate(fs::read(&ca_path)?))?; + } else { + let dirs = directories_next::ProjectDirs::from("org", "quinn", "quinn-examples").unwrap(); + match fs::read(dirs.data_local_dir().join("cert.der")) { + Ok(cert) => { + roots.add(&rustls::Certificate(cert))?; + } + Err(ref e) if e.kind() == io::ErrorKind::NotFound => { + info!("local server certificate not found"); + } + Err(e) => { + error!("failed to open local server certificate: {}", e); + } + } + }*/ + let client_crypto = configure_client(); + + /* + client_crypto.alpn_protocols = common::ALPN_QUIC_HTTP.iter().map(|&x| x.into()).collect(); + if options.keylog { + client_crypto.key_log = Arc::new(rustls::KeyLogFile::new()); + }*/ + + let mut endpoint = quinn::Endpoint::new(EndpointConfig::default(), None, sock.try_clone()?)?.0; + endpoint.set_default_client_config(client_crypto); + + //let request = format!("GET {}\r\n", url.path()); + //let start = Instant::now(); + //let rebind = options.rebind; + /* + let host = options + .host + .as_ref() + .map_or_else(|| url.host_str(), |x| Some(x)) + .ok_or_else(|| anyhow!("no hostname specified"))?; + + eprintln!("connecting to {} at {}", host, remote); + */ + let new_conn = endpoint + .connect(*dest, "host")? + .await + .map_err(|e| anyhow!("failed to connect: {}", e))?; + //eprintln!("connected at {:?}", start.elapsed()); + let quinn::NewConnection { + connection: conn, .. + } = new_conn; + /*let (mut send, recv) = conn + .open_bi() + .await + .map_err(|e| anyhow!("failed to open stream: {}", e))?;*/ + + let max_dgram_size = conn.max_datagram_size(); + assert!(max_dgram_size.is_some() && max_dgram_size.unwrap() >= MAX_TRANSACTION_SIZE); + + /* + if rebind { + let socket = std::net::UdpSocket::bind("[::]:0").unwrap(); + let addr = socket.local_addr().unwrap(); + eprintln!("rebinding to {}", addr); + endpoint.rebind(socket).expect("rebind failed"); + } +*/ +/* + send.write_all(request.as_bytes()) + .await + .map_err(|e| anyhow!("failed to send request: {}", e))?; + send.finish() + .await + .map_err(|e| anyhow!("failed to shutdown stream: {}", e))?; + let response_start = Instant::now(); + eprintln!("request sent at {:?}", response_start - start); + let resp = recv + .read_to_end(usize::max_value()) + .await + .map_err(|e| anyhow!("failed to read response: {}", e))?; + let duration = response_start.elapsed(); + eprintln!( + "response received in {:?} - {} KiB/s", + duration, + resp.len() as f32 / (duration_secs(&duration) * 1024.0) + ); + io::stdout().write_all(&resp).unwrap(); + io::stdout().flush().unwrap(); +*/ + //todo: is there a way to get rid of this copy? + packets.iter().try_for_each(|packet| {conn.send_datagram(Bytes::copy_from_slice(&packet.get_data()[0..packet.get_meta().size]))})?; + + conn.close(0u32.into(), b"done"); + + // Give the server a fair chance to receive the close packet + endpoint.wait_idle().await; + + Ok(()) +} + + + #[cfg(target_os = "linux")] fn mmsghdr_for_packet( packet: &[u8], diff --git a/streamer/src/streamer.rs b/streamer/src/streamer.rs index 58365f190f7ab3..ecdd769eb9d1d5 100644 --- a/streamer/src/streamer.rs +++ b/streamer/src/streamer.rs @@ -3,7 +3,7 @@ use { crate::{ - packet::{self, send_to, PacketBatch, PacketBatchRecycler, PACKETS_PER_BATCH}, + packet::{self, send_to, PacketBatch, PacketBatchRecycler, PACKETS_PER_BATCH, PacketInterface}, recvmmsg::NUM_RCVMMSGS, socket::SocketAddrSpace, }, @@ -24,11 +24,14 @@ use { thiserror::Error, }; -pub type PacketBatchReceiver = Receiver; -pub type PacketBatchSender = Sender; +pub type PacketBatchReceiver = Receiver>; +pub type PacketBatchSender = Sender>; + +pub type StandardPacketReceiver = PacketBatchReceiver; +pub type StandardPacketSender = PacketBatchSender; #[derive(Error, Debug)] -pub enum StreamerError { +pub enum StreamerError { #[error("I/O error")] Io(#[from] std::io::Error), @@ -36,20 +39,20 @@ pub enum StreamerError { RecvTimeout(#[from] RecvTimeoutError), #[error("send packets error")] - Send(#[from] SendError), + Send(#[from] SendError>), } -pub type Result = std::result::Result; +pub type Result = std::result::Result>; -fn recv_loop( +fn recv_loop( sock: &UdpSocket, exit: Arc, - channel: &PacketBatchSender, - recycler: &PacketBatchRecycler, + channel: &PacketBatchSender

, + recycler: &PacketBatchRecycler

, name: &'static str, coalesce_ms: u64, use_pinned_memory: bool, -) -> Result<()> { +) -> Result<(), P> { let mut recv_count = 0; let mut call_count = 0; let mut now = Instant::now(); @@ -94,11 +97,11 @@ fn recv_loop( } } -pub fn receiver( +pub fn receiver( sock: Arc, exit: &Arc, - packet_sender: PacketBatchSender, - recycler: PacketBatchRecycler, + packet_sender: PacketBatchSender

, + recycler: PacketBatchRecycler

, name: &'static str, coalesce_ms: u64, use_pinned_memory: bool, @@ -122,6 +125,7 @@ pub fn receiver( .unwrap() } + #[derive(Debug, Default)] struct SendStats { bytes: u64, @@ -224,19 +228,19 @@ impl StreamerSendStats { }; } - fn record(&mut self, pkt: &Packet) { - let ent = self.host_map.entry(pkt.meta.addr).or_default(); + fn record(&mut self, pkt: &P) { + let ent = self.host_map.entry(pkt.get_meta().addr).or_default(); ent.count += 1; - ent.bytes += pkt.data.len() as u64; + ent.bytes += pkt.get_data().len() as u64; } } -fn recv_send( +fn recv_send( sock: &UdpSocket, - r: &PacketBatchReceiver, + r: &PacketBatchReceiver

, socket_addr_space: &SocketAddrSpace, stats: &mut Option, -) -> Result<()> { +) -> Result<(), P> { let timer = Duration::new(1, 0); let packet_batch = r.recv_timeout(timer)?; if let Some(stats) = stats { @@ -246,9 +250,9 @@ fn recv_send( Ok(()) } -pub fn recv_packet_batches( - recvr: &PacketBatchReceiver, -) -> Result<(Vec, usize, Duration)> { +pub fn recv_packet_batches( + recvr: &PacketBatchReceiver

, +) -> Result<(Vec>, usize, Duration), P> { let timer = Duration::new(1, 0); let packet_batch = recvr.recv_timeout(timer)?; let recv_start = Instant::now(); @@ -269,10 +273,10 @@ pub fn recv_packet_batches( Ok((packet_batches, num_packets, recv_duration)) } -pub fn responder( +pub fn responder( name: &'static str, sock: Arc, - r: PacketBatchReceiver, + r: PacketBatchReceiver

, socket_addr_space: SocketAddrSpace, stats_reporter_sender: Option>>, ) -> JoinHandle<()> { @@ -287,7 +291,6 @@ pub fn responder( if stats_reporter_sender.is_some() { stats = Some(StreamerSendStats::default()); } - loop { if let Err(e) = recv_send(&sock, &r, &socket_addr_space, &mut stats) { match e { @@ -383,7 +386,6 @@ mod test { Arc::new(send), r_responder, SocketAddrSpace::Unspecified, - None, ); let mut packet_batch = PacketBatch::default(); for i in 0..5 { diff --git a/transaction-dos/src/main.rs b/transaction-dos/src/main.rs index 78cb3c67116403..7bd6277098b59f 100644 --- a/transaction-dos/src/main.rs +++ b/transaction-dos/src/main.rs @@ -378,7 +378,7 @@ fn run_transactions_dos( let tx = Transaction::new(&signers, message, blockhash); if !tested_size.load(Ordering::Relaxed) { let ser_size = bincode::serialized_size(&tx).unwrap(); - assert!(ser_size < 1200, "{}", ser_size); + assert!(ser_size < 1200 * 2, "{}", ser_size); tested_size.store(true, Ordering::Relaxed); } tx diff --git a/validator/src/admin_rpc_service.rs b/validator/src/admin_rpc_service.rs index e330f2a81a3378..b83b3b6b289c38 100644 --- a/validator/src/admin_rpc_service.rs +++ b/validator/src/admin_rpc_service.rs @@ -69,6 +69,8 @@ impl From for AdminRpcContactInfo { serve_repair, wallclock, shred_version, + tpu_extended, + tpu_extended_forwards, } = contact_info; Self { id: id.to_string(), diff --git a/validator/src/bootstrap.rs b/validator/src/bootstrap.rs index 1f3b72f1371d95..4567782cd12bd2 100644 --- a/validator/src/bootstrap.rs +++ b/validator/src/bootstrap.rs @@ -150,6 +150,12 @@ fn verify_reachable_ports( if ContactInfo::is_valid_address(&node.info.tpu_forwards, socket_addr_space) { udp_sockets.extend(node.sockets.tpu_forwards.iter()); } + if ContactInfo::is_valid_address(&node.info.tpu_extended, socket_addr_space) { + udp_sockets.extend(node.sockets.tpu_extended.iter()); + } + if ContactInfo::is_valid_address(&node.info.tpu_extended_forwards, socket_addr_space) { + udp_sockets.extend(node.sockets.tpu_extended_forwards.iter()); + } if ContactInfo::is_valid_address(&node.info.tpu_vote, socket_addr_space) { udp_sockets.extend(node.sockets.tpu_vote.iter()); } diff --git a/validator/src/main.rs b/validator/src/main.rs index d2da166d702320..95717d68b1eb3f 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -2607,6 +2607,7 @@ pub fn main() { // requests initiated by the node. All other ports are unused. node.info.tpu = any; node.info.tpu_forwards = any; + node.info.tpu_extended_forwards = any; node.info.tvu = any; node.info.tvu_forwards = any; node.info.serve_repair = any; diff --git a/web3.js/src/transaction.ts b/web3.js/src/transaction.ts index 5466464f5a9488..6e565fd3768ce0 100644 --- a/web3.js/src/transaction.ts +++ b/web3.js/src/transaction.ts @@ -30,7 +30,7 @@ const DEFAULT_SIGNATURE = Buffer.alloc(64).fill(0); * 40 bytes is the size of the IPv6 header * 8 bytes is the size of the fragment header */ -export const PACKET_DATA_SIZE = 1280 - 40 - 8; +export const PACKET_DATA_SIZE = (1280 - 40 - 8)*2; const SIGNATURE_LENGTH = 64;