Skip to content

Commit 950d496

Browse files
lijunwangslijunwangs
authored
make QUIC tpu QOS parameters configurable (anza-xyz#4170)
* make QUIC tpu QOS parameters configurable * Use max_connections_per_ipaddr_per_min * set max_unstaked_connections to 0 for tpu-fwd and vote in testing * fixed some clippy complaint * missing max-streams-per-ms * missing tpu_max_streams_per_ms * vote does not accept unstaked connections * Addressed some feedback from Alessandro * re-export some constants moved/renamed and mark them deprecated * removed duplicated code definition, use 'use'
1 parent 1a18c26 commit 950d496

File tree

14 files changed

+259
-151
lines changed

14 files changed

+259
-151
lines changed

‎bench-tps/src/cli.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ use {
1111
pubkey::Pubkey,
1212
signature::{read_keypair_file, Keypair},
1313
},
14-
solana_streamer::nonblocking::quic::DEFAULT_MAX_CONNECTIONS_PER_IPADDR_PER_MINUTE,
14+
solana_streamer::quic::DEFAULT_MAX_CONNECTIONS_PER_IPADDR_PER_MINUTE,
1515
solana_tpu_client::tpu_client::{DEFAULT_TPU_CONNECTION_POOL_SIZE, DEFAULT_TPU_USE_QUIC},
1616
std::{
1717
net::{IpAddr, Ipv4Addr},

‎core/src/tpu.rs

+13-32
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,12 @@
22
//! multi-stage transaction processing pipeline in software.
33
44
pub use solana_sdk::net::DEFAULT_TPU_COALESCE;
5+
// allow multiple connections for NAT and any open/close overlap
6+
#[deprecated(
7+
since = "2.2.0",
8+
note = "Use solana_streamer::quic::DEFAULT_MAX_QUIC_CONNECTIONS_PER_PEER instead"
9+
)]
10+
pub use solana_streamer::quic::DEFAULT_MAX_QUIC_CONNECTIONS_PER_PEER as MAX_QUIC_CONNECTIONS_PER_PEER;
511
use {
612
crate::{
713
banking_stage::BankingStage,
@@ -37,10 +43,7 @@ use {
3743
},
3844
solana_sdk::{clock::Slot, pubkey::Pubkey, quic::NotifyKeyUpdate, signature::Keypair},
3945
solana_streamer::{
40-
quic::{
41-
spawn_server_multi, QuicServerParams, SpawnServerResult, MAX_STAKED_CONNECTIONS,
42-
MAX_UNSTAKED_CONNECTIONS,
43-
},
46+
quic::{spawn_server_multi, QuicServerParams, SpawnServerResult},
4447
streamer::StakedNodes,
4548
},
4649
solana_turbine::broadcast_stage::{BroadcastStage, BroadcastStageType},
@@ -54,9 +57,6 @@ use {
5457
tokio::sync::mpsc::Sender as AsyncSender,
5558
};
5659

57-
// allow multiple connections for NAT and any open/close overlap
58-
pub const MAX_QUIC_CONNECTIONS_PER_PEER: usize = 8;
59-
6060
pub struct TpuSockets {
6161
pub transactions: Vec<UdpSocket>,
6262
pub transaction_forwards: Vec<UdpSocket>,
@@ -115,7 +115,9 @@ impl Tpu {
115115
banking_tracer: Arc<BankingTracer>,
116116
tracer_thread_hdl: TracerThread,
117117
tpu_enable_udp: bool,
118-
tpu_max_connections_per_ipaddr_per_minute: u64,
118+
tpu_quic_server_config: QuicServerParams,
119+
tpu_fwd_quic_server_config: QuicServerParams,
120+
vote_quic_server_config: QuicServerParams,
119121
prioritization_fee_cache: &Arc<PrioritizationFeeCache>,
120122
block_production_method: BlockProductionMethod,
121123
transaction_struct: TransactionStructure,
@@ -179,15 +181,7 @@ impl Tpu {
179181
vote_packet_sender.clone(),
180182
exit.clone(),
181183
staked_nodes.clone(),
182-
QuicServerParams {
183-
max_connections_per_peer: 1,
184-
max_connections_per_ipaddr_per_min: tpu_max_connections_per_ipaddr_per_minute,
185-
coalesce: tpu_coalesce,
186-
max_staked_connections: MAX_STAKED_CONNECTIONS
187-
.saturating_add(MAX_UNSTAKED_CONNECTIONS),
188-
max_unstaked_connections: 0,
189-
..QuicServerParams::default()
190-
},
184+
vote_quic_server_config,
191185
)
192186
.unwrap();
193187

@@ -204,12 +198,7 @@ impl Tpu {
204198
packet_sender,
205199
exit.clone(),
206200
staked_nodes.clone(),
207-
QuicServerParams {
208-
max_connections_per_peer: MAX_QUIC_CONNECTIONS_PER_PEER,
209-
max_connections_per_ipaddr_per_min: tpu_max_connections_per_ipaddr_per_minute,
210-
coalesce: tpu_coalesce,
211-
..QuicServerParams::default()
212-
},
201+
tpu_quic_server_config,
213202
)
214203
.unwrap();
215204

@@ -226,15 +215,7 @@ impl Tpu {
226215
forwarded_packet_sender,
227216
exit.clone(),
228217
staked_nodes.clone(),
229-
QuicServerParams {
230-
max_connections_per_peer: MAX_QUIC_CONNECTIONS_PER_PEER,
231-
max_staked_connections: MAX_STAKED_CONNECTIONS
232-
.saturating_add(MAX_UNSTAKED_CONNECTIONS),
233-
max_unstaked_connections: 0, // Prevent unstaked nodes from forwarding transactions
234-
max_connections_per_ipaddr_per_min: tpu_max_connections_per_ipaddr_per_minute,
235-
coalesce: tpu_coalesce,
236-
..QuicServerParams::default()
237-
},
218+
tpu_fwd_quic_server_config,
238219
)
239220
.unwrap();
240221

‎core/src/validator.rs

+49-23
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,10 @@ use {
124124
timing::timestamp,
125125
},
126126
solana_send_transaction_service::send_transaction_service,
127-
solana_streamer::{socket::SocketAddrSpace, streamer::StakedNodes},
127+
solana_streamer::{quic::QuicServerParams, socket::SocketAddrSpace, streamer::StakedNodes},
128+
solana_tpu_client::tpu_client::{
129+
DEFAULT_TPU_CONNECTION_POOL_SIZE, DEFAULT_TPU_USE_QUIC, DEFAULT_VOTE_USE_QUIC,
130+
},
128131
solana_turbine::{self, broadcast_stage::BroadcastStageType},
129132
solana_unified_scheduler_pool::DefaultSchedulerPool,
130133
solana_vote_program::vote_state,
@@ -510,8 +513,42 @@ pub struct ValidatorTpuConfig {
510513
pub tpu_connection_pool_size: usize,
511514
/// Controls if to enable UDP for TPU tansactions.
512515
pub tpu_enable_udp: bool,
513-
/// Controls the new maximum connections per IpAddr per minute
514-
pub tpu_max_connections_per_ipaddr_per_minute: u64,
516+
/// QUIC server config for regular TPU
517+
pub tpu_quic_server_config: QuicServerParams,
518+
/// QUIC server config for TPU forward
519+
pub tpu_fwd_quic_server_config: QuicServerParams,
520+
/// QUIC server config for Vote
521+
pub vote_quic_server_config: QuicServerParams,
522+
}
523+
524+
impl ValidatorTpuConfig {
525+
/// A convenient function to build a ValidatorTpuConfig for testing with good
526+
/// default.
527+
pub fn new_for_tests(tpu_enable_udp: bool) -> Self {
528+
let tpu_quic_server_config = QuicServerParams {
529+
max_connections_per_ipaddr_per_min: 32,
530+
..Default::default()
531+
};
532+
533+
let tpu_fwd_quic_server_config = QuicServerParams {
534+
max_connections_per_ipaddr_per_min: 32,
535+
max_unstaked_connections: 0,
536+
..Default::default()
537+
};
538+
539+
// vote and tpu_fwd share the same characteristics -- disallow non-staked connections:
540+
let vote_quic_server_config = tpu_fwd_quic_server_config.clone();
541+
542+
ValidatorTpuConfig {
543+
use_quic: DEFAULT_TPU_USE_QUIC,
544+
vote_use_quic: DEFAULT_VOTE_USE_QUIC,
545+
tpu_connection_pool_size: DEFAULT_TPU_CONNECTION_POOL_SIZE,
546+
tpu_enable_udp,
547+
tpu_quic_server_config,
548+
tpu_fwd_quic_server_config,
549+
vote_quic_server_config,
550+
}
551+
}
515552
}
516553

517554
pub struct Validator {
@@ -573,7 +610,9 @@ impl Validator {
573610
vote_use_quic,
574611
tpu_connection_pool_size,
575612
tpu_enable_udp,
576-
tpu_max_connections_per_ipaddr_per_minute,
613+
tpu_quic_server_config,
614+
tpu_fwd_quic_server_config,
615+
vote_quic_server_config,
577616
} = tpu_config;
578617

579618
let start_time = Instant::now();
@@ -1548,7 +1587,9 @@ impl Validator {
15481587
banking_tracer,
15491588
tracer_thread,
15501589
tpu_enable_udp,
1551-
tpu_max_connections_per_ipaddr_per_minute,
1590+
tpu_quic_server_config,
1591+
tpu_fwd_quic_server_config,
1592+
vote_quic_server_config,
15521593
&prioritization_fee_cache,
15531594
config.block_production_method.clone(),
15541595
config.transaction_struct.clone(),
@@ -2751,10 +2792,7 @@ mod tests {
27512792
get_tmp_ledger_path_auto_delete,
27522793
},
27532794
solana_sdk::{genesis_config::create_genesis_config, poh_config::PohConfig},
2754-
solana_tpu_client::tpu_client::{
2755-
DEFAULT_TPU_CONNECTION_POOL_SIZE, DEFAULT_TPU_ENABLE_UDP, DEFAULT_TPU_USE_QUIC,
2756-
DEFAULT_VOTE_USE_QUIC,
2757-
},
2795+
solana_tpu_client::tpu_client::DEFAULT_TPU_ENABLE_UDP,
27582796
std::{fs::remove_dir_all, thread, time::Duration},
27592797
};
27602798

@@ -2792,13 +2830,7 @@ mod tests {
27922830
None, // rpc_to_plugin_manager_receiver
27932831
start_progress.clone(),
27942832
SocketAddrSpace::Unspecified,
2795-
ValidatorTpuConfig {
2796-
use_quic: DEFAULT_TPU_USE_QUIC,
2797-
vote_use_quic: DEFAULT_VOTE_USE_QUIC,
2798-
tpu_connection_pool_size: DEFAULT_TPU_CONNECTION_POOL_SIZE,
2799-
tpu_enable_udp: DEFAULT_TPU_ENABLE_UDP,
2800-
tpu_max_connections_per_ipaddr_per_minute: 32, // max connections per IpAddr per minute for test
2801-
},
2833+
ValidatorTpuConfig::new_for_tests(DEFAULT_TPU_ENABLE_UDP),
28022834
Arc::new(RwLock::new(None)),
28032835
)
28042836
.expect("assume successful validator start");
@@ -3014,13 +3046,7 @@ mod tests {
30143046
None, // rpc_to_plugin_manager_receiver
30153047
Arc::new(RwLock::new(ValidatorStartProgress::default())),
30163048
SocketAddrSpace::Unspecified,
3017-
ValidatorTpuConfig {
3018-
use_quic: DEFAULT_TPU_USE_QUIC,
3019-
vote_use_quic: DEFAULT_VOTE_USE_QUIC,
3020-
tpu_connection_pool_size: DEFAULT_TPU_CONNECTION_POOL_SIZE,
3021-
tpu_enable_udp: DEFAULT_TPU_ENABLE_UDP,
3022-
tpu_max_connections_per_ipaddr_per_minute: 32, // max connections per IpAddr per minute for test
3023-
},
3049+
ValidatorTpuConfig::new_for_tests(DEFAULT_TPU_ENABLE_UDP),
30243050
Arc::new(RwLock::new(None)),
30253051
)
30263052
.expect("assume successful validator start")

‎local-cluster/src/local_cluster.rs

+5-23
Original file line numberDiff line numberDiff line change
@@ -351,15 +351,9 @@ impl LocalCluster {
351351
None, // rpc_to_plugin_manager_receiver
352352
Arc::new(RwLock::new(ValidatorStartProgress::default())),
353353
socket_addr_space,
354-
ValidatorTpuConfig {
355-
use_quic: DEFAULT_TPU_USE_QUIC,
356-
vote_use_quic: DEFAULT_VOTE_USE_QUIC,
357-
tpu_connection_pool_size: DEFAULT_TPU_CONNECTION_POOL_SIZE,
358-
// We are turning tpu_enable_udp to true in order to prevent concurrent local cluster tests
359-
// to use the same QUIC ports due to SO_REUSEPORT.
360-
tpu_enable_udp: true,
361-
tpu_max_connections_per_ipaddr_per_minute: 32, // max connections per IpAddr per minute
362-
},
354+
// We are turning tpu_enable_udp to true in order to prevent concurrent local cluster tests
355+
// to use the same QUIC ports due to SO_REUSEPORT.
356+
ValidatorTpuConfig::new_for_tests(true),
363357
Arc::new(RwLock::new(None)),
364358
)
365359
.expect("assume successful validator start");
@@ -568,13 +562,7 @@ impl LocalCluster {
568562
None, // rpc_to_plugin_manager_receiver
569563
Arc::new(RwLock::new(ValidatorStartProgress::default())),
570564
socket_addr_space,
571-
ValidatorTpuConfig {
572-
use_quic: DEFAULT_TPU_USE_QUIC,
573-
vote_use_quic: DEFAULT_VOTE_USE_QUIC,
574-
tpu_connection_pool_size: DEFAULT_TPU_CONNECTION_POOL_SIZE,
575-
tpu_enable_udp: DEFAULT_TPU_ENABLE_UDP,
576-
tpu_max_connections_per_ipaddr_per_minute: 32, // max connections per IpAddr per mintute
577-
},
565+
ValidatorTpuConfig::new_for_tests(DEFAULT_TPU_ENABLE_UDP),
578566
Arc::new(RwLock::new(None)),
579567
)
580568
.expect("assume successful validator start");
@@ -1100,13 +1088,7 @@ impl Cluster for LocalCluster {
11001088
None, // rpc_to_plugin_manager_receiver
11011089
Arc::new(RwLock::new(ValidatorStartProgress::default())),
11021090
socket_addr_space,
1103-
ValidatorTpuConfig {
1104-
use_quic: DEFAULT_TPU_USE_QUIC,
1105-
vote_use_quic: DEFAULT_VOTE_USE_QUIC,
1106-
tpu_connection_pool_size: DEFAULT_TPU_CONNECTION_POOL_SIZE,
1107-
tpu_enable_udp: DEFAULT_TPU_ENABLE_UDP,
1108-
tpu_max_connections_per_ipaddr_per_minute: 32, // max connections per IpAddr per minute, use higher value because of tests
1109-
},
1091+
ValidatorTpuConfig::new_for_tests(DEFAULT_TPU_ENABLE_UDP),
11101092
Arc::new(RwLock::new(None)),
11111093
)
11121094
.expect("assume successful validator start");

‎streamer/src/nonblocking/quic.rs

+11-4
Original file line numberDiff line numberDiff line change
@@ -87,13 +87,20 @@ const CONNECTION_CLOSE_REASON_TOO_MANY: &[u8] = b"too_many";
8787
const CONNECTION_CLOSE_CODE_INVALID_STREAM: u32 = 5;
8888
const CONNECTION_CLOSE_REASON_INVALID_STREAM: &[u8] = b"invalid_stream";
8989

90-
/// Limit to 250K PPS
91-
pub const DEFAULT_MAX_STREAMS_PER_MS: u64 = 250;
92-
9390
/// The new connections per minute from a particular IP address.
9491
/// Heuristically set to the default maximum concurrent connections
9592
/// per IP address. Might be adjusted later.
96-
pub const DEFAULT_MAX_CONNECTIONS_PER_IPADDR_PER_MINUTE: u64 = 8;
93+
#[deprecated(
94+
since = "2.2.0",
95+
note = "Use solana_streamer::quic::DEFAULT_MAX_CONNECTIONS_PER_IPADDR_PER_MINUTE"
96+
)]
97+
pub use crate::quic::DEFAULT_MAX_CONNECTIONS_PER_IPADDR_PER_MINUTE;
98+
/// Limit to 250K PPS
99+
#[deprecated(
100+
since = "2.2.0",
101+
note = "Use solana_streamer::quic::DEFAULT_MAX_STREAMS_PER_MS"
102+
)]
103+
pub use crate::quic::DEFAULT_MAX_STREAMS_PER_MS;
97104

98105
/// Total new connection counts per second. Heuristically taken from
99106
/// the default staked and unstaked connection limits. Might be adjusted

‎streamer/src/nonblocking/stream_throttle.rs

+7-9
Original file line numberDiff line numberDiff line change
@@ -236,10 +236,8 @@ pub mod test {
236236
use {
237237
super::*,
238238
crate::{
239-
nonblocking::{
240-
quic::DEFAULT_MAX_STREAMS_PER_MS, stream_throttle::STREAM_LOAD_EMA_INTERVAL_MS,
241-
},
242-
quic::{StreamerStats, MAX_UNSTAKED_CONNECTIONS},
239+
nonblocking::stream_throttle::STREAM_LOAD_EMA_INTERVAL_MS,
240+
quic::{StreamerStats, DEFAULT_MAX_STREAMS_PER_MS, DEFAULT_MAX_UNSTAKED_CONNECTIONS},
243241
},
244242
std::{
245243
sync::{atomic::Ordering, Arc},
@@ -251,7 +249,7 @@ pub mod test {
251249
fn test_max_streams_for_unstaked_connection() {
252250
let load_ema = Arc::new(StakedStreamLoadEMA::new(
253251
Arc::new(StreamerStats::default()),
254-
MAX_UNSTAKED_CONNECTIONS,
252+
DEFAULT_MAX_UNSTAKED_CONNECTIONS,
255253
DEFAULT_MAX_STREAMS_PER_MS,
256254
));
257255
// 25K packets per ms * 20% / 500 max unstaked connections
@@ -268,7 +266,7 @@ pub mod test {
268266
fn test_max_streams_for_staked_connection() {
269267
let load_ema = Arc::new(StakedStreamLoadEMA::new(
270268
Arc::new(StreamerStats::default()),
271-
MAX_UNSTAKED_CONNECTIONS,
269+
DEFAULT_MAX_UNSTAKED_CONNECTIONS,
272270
DEFAULT_MAX_STREAMS_PER_MS,
273271
));
274272

@@ -448,7 +446,7 @@ pub mod test {
448446
fn test_update_ema() {
449447
let stream_load_ema = Arc::new(StakedStreamLoadEMA::new(
450448
Arc::new(StreamerStats::default()),
451-
MAX_UNSTAKED_CONNECTIONS,
449+
DEFAULT_MAX_UNSTAKED_CONNECTIONS,
452450
DEFAULT_MAX_STREAMS_PER_MS,
453451
));
454452
stream_load_ema
@@ -477,7 +475,7 @@ pub mod test {
477475
fn test_update_ema_missing_interval() {
478476
let stream_load_ema = Arc::new(StakedStreamLoadEMA::new(
479477
Arc::new(StreamerStats::default()),
480-
MAX_UNSTAKED_CONNECTIONS,
478+
DEFAULT_MAX_UNSTAKED_CONNECTIONS,
481479
DEFAULT_MAX_STREAMS_PER_MS,
482480
));
483481
stream_load_ema
@@ -497,7 +495,7 @@ pub mod test {
497495
fn test_update_ema_if_needed() {
498496
let stream_load_ema = Arc::new(StakedStreamLoadEMA::new(
499497
Arc::new(StreamerStats::default()),
500-
MAX_UNSTAKED_CONNECTIONS,
498+
DEFAULT_MAX_UNSTAKED_CONNECTIONS,
501499
DEFAULT_MAX_STREAMS_PER_MS,
502500
));
503501
stream_load_ema

‎streamer/src/nonblocking/testing_utilities.rs

+5-5
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,13 @@
22
use {
33
super::quic::{
44
spawn_server_multi, SpawnNonBlockingServerResult, ALPN_TPU_PROTOCOL_ID,
5-
DEFAULT_MAX_CONNECTIONS_PER_IPADDR_PER_MINUTE, DEFAULT_MAX_STREAMS_PER_MS,
65
DEFAULT_WAIT_FOR_CHUNK_TIMEOUT,
76
},
87
crate::{
98
quic::{
10-
QuicServerParams, StreamerStats, DEFAULT_TPU_COALESCE, MAX_STAKED_CONNECTIONS,
11-
MAX_UNSTAKED_CONNECTIONS,
9+
QuicServerParams, StreamerStats, DEFAULT_MAX_CONNECTIONS_PER_IPADDR_PER_MINUTE,
10+
DEFAULT_MAX_STAKED_CONNECTIONS, DEFAULT_MAX_STREAMS_PER_MS,
11+
DEFAULT_MAX_UNSTAKED_CONNECTIONS, DEFAULT_TPU_COALESCE,
1212
},
1313
streamer::StakedNodes,
1414
},
@@ -64,8 +64,8 @@ impl Default for TestServerConfig {
6464
fn default() -> Self {
6565
Self {
6666
max_connections_per_peer: 1,
67-
max_staked_connections: MAX_STAKED_CONNECTIONS,
68-
max_unstaked_connections: MAX_UNSTAKED_CONNECTIONS,
67+
max_staked_connections: DEFAULT_MAX_STAKED_CONNECTIONS,
68+
max_unstaked_connections: DEFAULT_MAX_UNSTAKED_CONNECTIONS,
6969
max_streams_per_ms: DEFAULT_MAX_STREAMS_PER_MS,
7070
max_connections_per_ipaddr_per_min: DEFAULT_MAX_CONNECTIONS_PER_IPADDR_PER_MINUTE,
7171
}

0 commit comments

Comments
 (0)