diff --git a/Cargo.lock b/Cargo.lock index 7f0f523e70039e..df691372a46be1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7943,7 +7943,6 @@ dependencies = [ "solana-pubkey 4.0.0", "solana-pubsub-client", "solana-quic-client", - "solana-quic-definitions", "solana-rpc-client", "solana-rpc-client-api", "solana-rpc-client-nonce-utils", @@ -7951,6 +7950,7 @@ dependencies = [ "solana-signer", "solana-streamer", "solana-time-utils", + "solana-tls-utils", "solana-tpu-client", "solana-transaction", "solana-transaction-error", @@ -8274,7 +8274,6 @@ dependencies = [ "solana-program-runtime", "solana-pubkey 4.0.0", "solana-quic-client", - "solana-quic-definitions", "solana-rayon-threadlimit", "solana-rent", "solana-rpc", @@ -8911,7 +8910,6 @@ dependencies = [ "solana-packet 4.0.0", "solana-perf", "solana-pubkey 4.0.0", - "solana-quic-definitions", "solana-rayon-threadlimit", "solana-rpc-client", "solana-runtime", @@ -9999,7 +9997,6 @@ dependencies = [ "solana-perf", "solana-pubkey 4.0.0", "solana-quic-client", - "solana-quic-definitions", "solana-rpc-client-api", "solana-signer", "solana-streamer", @@ -10010,15 +10007,6 @@ dependencies = [ "tokio-util 0.7.18", ] -[[package]] -name = "solana-quic-definitions" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15319accf7d3afd845817aeffa6edd8cc185f135cefbc6b985df29cfd8c09609" -dependencies = [ - "solana-keypair", -] - [[package]] name = "solana-rayon-threadlimit" version = "4.0.0-alpha.0" @@ -10141,7 +10129,6 @@ dependencies = [ "solana-program-pack", "solana-program-runtime", "solana-pubkey 4.0.0", - "solana-quic-definitions", "solana-rayon-threadlimit", "solana-rent", "solana-rpc", @@ -10162,6 +10149,7 @@ dependencies = [ "solana-system-transaction", "solana-sysvar", "solana-time-utils", + "solana-tls-utils", "solana-tpu-client", "solana-transaction", "solana-transaction-context", @@ -10657,13 +10645,13 @@ dependencies = [ "solana-nonce", "solana-nonce-account", "solana-pubkey 4.0.0", - "solana-quic-definitions", "solana-runtime", "solana-signature", "solana-signer", "solana-system-interface 3.0.0", "solana-system-transaction", "solana-time-utils", + "solana-tls-utils", "solana-tpu-client-next", "tokio", "tokio-util 0.7.18", @@ -10965,7 +10953,6 @@ dependencies = [ "solana-packet 4.0.0", "solana-perf", "solana-pubkey 4.0.0", - "solana-quic-definitions", "solana-signature", "solana-signer", "solana-streamer", @@ -11884,9 +11871,9 @@ dependencies = [ "solana-net-utils", "solana-perf", "solana-pubkey 4.0.0", - "solana-quic-definitions", "solana-signer", "solana-streamer", + "solana-tls-utils", "solana-transaction-metrics-tracker", "solana-version", "thiserror 2.0.17", diff --git a/Cargo.toml b/Cargo.toml index 87658a563d0a80..b31a130710152b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -505,7 +505,6 @@ solana-program-test = { path = "program-test", version = "=4.0.0-alpha.0", featu solana-pubkey = { version = "4.0.0", default-features = false } solana-pubsub-client = { path = "pubsub-client", version = "=4.0.0-alpha.0" } solana-quic-client = { path = "quic-client", version = "=4.0.0-alpha.0", features = ["agave-unstable-api"] } -solana-quic-definitions = "3.0.0" solana-rayon-threadlimit = { path = "rayon-threadlimit", version = "=4.0.0-alpha.0", features = ["agave-unstable-api"] } solana-remote-wallet = { path = "remote-wallet", version = "=4.0.0-alpha.0", default-features = false, features = ["agave-unstable-api"] } solana-rent = "3.0.0" diff --git a/client/Cargo.toml b/client/Cargo.toml index 18f0312cd76962..ea083c6c2e4b85 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -41,7 +41,6 @@ solana-net-utils = { workspace = true } solana-pubkey = { workspace = true } solana-pubsub-client = { workspace = true } solana-quic-client = { workspace = true } -solana-quic-definitions = { workspace = true } solana-rpc-client = { workspace = true, features = ["default"] } solana-rpc-client-api = { workspace = true } solana-rpc-client-nonce-utils = { workspace = true } @@ -49,6 +48,7 @@ solana-signature = { workspace = true } solana-signer = { workspace = true } solana-streamer = { workspace = true } solana-time-utils = { workspace = true } +solana-tls-utils = { workspace = true } solana-tpu-client = { workspace = true, features = ["default"] } solana-transaction = { workspace = true } solana-transaction-error = { workspace = true } diff --git a/client/src/connection_cache.rs b/client/src/connection_cache.rs index 1fdc27ac922979..7358ca057d5013 100644 --- a/client/src/connection_cache.rs +++ b/client/src/connection_cache.rs @@ -10,8 +10,8 @@ use { solana_keypair::Keypair, solana_pubkey::Pubkey, solana_quic_client::{QuicConfig, QuicConnectionManager, QuicPool}, - solana_quic_definitions::NotifyKeyUpdate, solana_streamer::streamer::StakedNodes, + solana_tls_utils::NotifyKeyUpdate, solana_transaction_error::TransportResult, solana_udp_client::{UdpConfig, UdpConnectionManager, UdpPool}, std::{ diff --git a/core/Cargo.toml b/core/Cargo.toml index 282f37af168dbe..1c2b43a2caf5c1 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -135,7 +135,6 @@ solana-poh = { workspace = true } solana-poh-config = { workspace = true } solana-pubkey = { workspace = true } solana-quic-client = { workspace = true } -solana-quic-definitions = { workspace = true } solana-rayon-threadlimit = { workspace = true } solana-rent = { workspace = true } solana-rpc = { workspace = true } diff --git a/core/src/admin_rpc_post_init.rs b/core/src/admin_rpc_post_init.rs index 1081fec6099a27..03e1e0375dca3a 100644 --- a/core/src/admin_rpc_post_init.rs +++ b/core/src/admin_rpc_post_init.rs @@ -6,8 +6,8 @@ use { }, solana_gossip::{cluster_info::ClusterInfo, node::NodeMultihoming}, solana_pubkey::Pubkey, - solana_quic_definitions::NotifyKeyUpdate, solana_runtime::{bank_forks::BankForks, snapshot_controller::SnapshotController}, + solana_tls_utils::NotifyKeyUpdate, std::{ collections::{HashMap, HashSet}, net::UdpSocket, diff --git a/core/src/forwarding_stage.rs b/core/src/forwarding_stage.rs index e3f88ba73e573d..779a04c8e18e9d 100644 --- a/core/src/forwarding_stage.rs +++ b/core/src/forwarding_stage.rs @@ -18,7 +18,6 @@ use { solana_packet as packet, solana_perf::data_budget::DataBudget, solana_poh::poh_recorder::PohRecorder, - solana_quic_definitions::NotifyKeyUpdate, solana_runtime::{ bank::{Bank, CollectorFeeDetails}, bank_forks::SharableBanks, @@ -27,6 +26,7 @@ use { runtime_transaction::RuntimeTransaction, transaction_meta::StaticMeta, }, solana_streamer::sendmmsg::{batch_send, SendPktsError}, + solana_tls_utils::NotifyKeyUpdate, solana_tpu_client_next::{ connection_workers_scheduler::{ BindTarget, ConnectionWorkersSchedulerConfig, Fanout, StakeIdentity, diff --git a/dev-bins/Cargo.lock b/dev-bins/Cargo.lock index 7c8b03fd89c2fb..5cce5385fd30f7 100644 --- a/dev-bins/Cargo.lock +++ b/dev-bins/Cargo.lock @@ -6701,7 +6701,6 @@ dependencies = [ "solana-pubkey 4.0.0", "solana-pubsub-client", "solana-quic-client", - "solana-quic-definitions", "solana-rpc-client", "solana-rpc-client-api", "solana-rpc-client-nonce-utils", @@ -6709,6 +6708,7 @@ dependencies = [ "solana-signer", "solana-streamer", "solana-time-utils", + "solana-tls-utils", "solana-tpu-client", "solana-transaction", "solana-transaction-error", @@ -6951,7 +6951,6 @@ dependencies = [ "solana-poh-config", "solana-pubkey 4.0.0", "solana-quic-client", - "solana-quic-definitions", "solana-rayon-threadlimit", "solana-rent", "solana-rpc", @@ -7502,7 +7501,6 @@ dependencies = [ "solana-packet 4.0.0", "solana-perf", "solana-pubkey 4.0.0", - "solana-quic-definitions", "solana-rayon-threadlimit", "solana-rpc-client", "solana-runtime", @@ -8380,7 +8378,6 @@ dependencies = [ "solana-metrics", "solana-net-utils", "solana-pubkey 4.0.0", - "solana-quic-definitions", "solana-rpc-client-api", "solana-signer", "solana-streamer", @@ -8390,15 +8387,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "solana-quic-definitions" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15319accf7d3afd845817aeffa6edd8cc185f135cefbc6b985df29cfd8c09609" -dependencies = [ - "solana-keypair", -] - [[package]] name = "solana-rayon-threadlimit" version = "4.0.0-alpha.0" @@ -8502,7 +8490,6 @@ dependencies = [ "solana-poh-config", "solana-program-pack", "solana-pubkey 4.0.0", - "solana-quic-definitions", "solana-rayon-threadlimit", "solana-rpc-client-api", "solana-runtime", @@ -8517,6 +8504,7 @@ dependencies = [ "solana-system-transaction", "solana-sysvar", "solana-time-utils", + "solana-tls-utils", "solana-tpu-client", "solana-transaction", "solana-transaction-context", @@ -8907,10 +8895,10 @@ dependencies = [ "solana-metrics", "solana-nonce-account", "solana-pubkey 4.0.0", - "solana-quic-definitions", "solana-runtime", "solana-signature", "solana-time-utils", + "solana-tls-utils", "solana-tpu-client-next", "tokio", "tokio-util 0.7.18", @@ -9163,7 +9151,6 @@ dependencies = [ "solana-packet 4.0.0", "solana-perf", "solana-pubkey 4.0.0", - "solana-quic-definitions", "solana-signature", "solana-signer", "solana-time-utils", diff --git a/gossip/Cargo.toml b/gossip/Cargo.toml index 97a5a64749a9f8..2ef418357095a4 100644 --- a/gossip/Cargo.toml +++ b/gossip/Cargo.toml @@ -78,7 +78,6 @@ solana-net-utils = { workspace = true, features = ["agave-unstable-api"] } solana-packet = { workspace = true } solana-perf = { workspace = true } solana-pubkey = { workspace = true, features = ["rand"] } -solana-quic-definitions = { workspace = true } solana-rayon-threadlimit = { workspace = true } solana-rpc-client = { workspace = true } solana-runtime = { workspace = true } diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index 0665690fa78f58..83d67abd3b51bc 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -6510,7 +6510,6 @@ dependencies = [ "solana-pubkey 4.0.0", "solana-pubsub-client", "solana-quic-client", - "solana-quic-definitions", "solana-rpc-client", "solana-rpc-client-api", "solana-rpc-client-nonce-utils", @@ -6518,6 +6517,7 @@ dependencies = [ "solana-signer", "solana-streamer", "solana-time-utils", + "solana-tls-utils", "solana-tpu-client", "solana-transaction", "solana-transaction-error", @@ -6760,7 +6760,6 @@ dependencies = [ "solana-poh-config", "solana-pubkey 4.0.0", "solana-quic-client", - "solana-quic-definitions", "solana-rayon-threadlimit", "solana-rent", "solana-rpc", @@ -7253,7 +7252,6 @@ dependencies = [ "solana-packet 4.0.0", "solana-perf", "solana-pubkey 4.0.0", - "solana-quic-definitions", "solana-rayon-threadlimit", "solana-rpc-client", "solana-runtime", @@ -8109,7 +8107,6 @@ dependencies = [ "solana-metrics", "solana-net-utils", "solana-pubkey 4.0.0", - "solana-quic-definitions", "solana-rpc-client-api", "solana-signer", "solana-streamer", @@ -8119,15 +8116,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "solana-quic-definitions" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15319accf7d3afd845817aeffa6edd8cc185f135cefbc6b985df29cfd8c09609" -dependencies = [ - "solana-keypair", -] - [[package]] name = "solana-rayon-threadlimit" version = "4.0.0-alpha.0" @@ -8231,7 +8219,6 @@ dependencies = [ "solana-poh-config", "solana-program-pack", "solana-pubkey 4.0.0", - "solana-quic-definitions", "solana-rayon-threadlimit", "solana-rpc-client-api", "solana-runtime", @@ -8246,6 +8233,7 @@ dependencies = [ "solana-system-transaction", "solana-sysvar", "solana-time-utils", + "solana-tls-utils", "solana-tpu-client", "solana-transaction", "solana-transaction-context", @@ -9398,10 +9386,10 @@ dependencies = [ "solana-metrics", "solana-nonce-account", "solana-pubkey 4.0.0", - "solana-quic-definitions", "solana-runtime", "solana-signature", "solana-time-utils", + "solana-tls-utils", "solana-tpu-client-next", "tokio", "tokio-util 0.7.18", @@ -9652,7 +9640,6 @@ dependencies = [ "solana-packet 4.0.0", "solana-perf", "solana-pubkey 4.0.0", - "solana-quic-definitions", "solana-signature", "solana-signer", "solana-time-utils", diff --git a/quic-client/Cargo.toml b/quic-client/Cargo.toml index a1c30363c339d4..4adfa595a2b47f 100644 --- a/quic-client/Cargo.toml +++ b/quic-client/Cargo.toml @@ -27,7 +27,6 @@ solana-measure = { workspace = true } solana-metrics = { workspace = true } solana-net-utils = { workspace = true } solana-pubkey = { workspace = true, default-features = false } -solana-quic-definitions = { workspace = true } solana-rpc-client-api = { workspace = true } solana-signer = { workspace = true } solana-streamer = { workspace = true } diff --git a/quic-client/src/nonblocking/quic_client.rs b/quic-client/src/nonblocking/quic_client.rs index 276011d0e56a0d..415c0784cb57ef 100644 --- a/quic-client/src/nonblocking/quic_client.rs +++ b/quic-client/src/nonblocking/quic_client.rs @@ -18,11 +18,8 @@ use { solana_keypair::Keypair, solana_measure::measure::Measure, solana_net_utils::sockets, - solana_quic_definitions::{ - QUIC_CONNECTION_HANDSHAKE_TIMEOUT, QUIC_KEEP_ALIVE, QUIC_MAX_TIMEOUT, QUIC_SEND_FAIRNESS, - }, solana_rpc_client_api::client_error::ErrorKind as ClientErrorKind, - solana_streamer::nonblocking::quic::ALPN_TPU_PROTOCOL_ID, + solana_streamer::{nonblocking::quic::ALPN_TPU_PROTOCOL_ID, quic::QUIC_MAX_TIMEOUT}, solana_tls_utils::{ new_dummy_x509_certificate, socket_addr_to_quic_server_name, tls_client_config_builder, QuicClientCertificate, @@ -32,11 +29,19 @@ use { net::{SocketAddr, UdpSocket}, sync::{atomic::Ordering, Arc}, thread, + time::Duration, }, thiserror::Error, tokio::{sync::OnceCell, time::timeout}, }; +const QUIC_KEEP_ALIVE: Duration = Duration::from_secs(1); + +// Based on commonly-used handshake timeouts for various TCP +// applications. Different applications vary, but most seem to +// be in the 30-60 second range +pub const QUIC_CONNECTION_HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(60); + /// A lazy-initialized Quic Endpoint pub struct QuicLazyInitializedEndpoint { endpoint: OnceCell>, @@ -107,7 +112,7 @@ impl QuicLazyInitializedEndpoint { let timeout = IdleTimeout::try_from(QUIC_MAX_TIMEOUT).unwrap(); transport_config.max_idle_timeout(Some(timeout)); transport_config.keep_alive_interval(Some(QUIC_KEEP_ALIVE)); - transport_config.send_fairness(QUIC_SEND_FAIRNESS); + transport_config.send_fairness(false); config.transport_config(Arc::new(transport_config)); endpoint.set_default_client_config(config); diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index 4f05b430ce9241..c8d90ec4da000f 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -70,7 +70,6 @@ solana-poh = { workspace = true } solana-poh-config = { workspace = true } solana-program-pack = { workspace = true } solana-pubkey = { workspace = true } -solana-quic-definitions = { workspace = true } solana-rayon-threadlimit = { workspace = true } solana-rpc-client-api = { workspace = true } solana-runtime = { workspace = true } @@ -85,6 +84,7 @@ solana-system-interface = { workspace = true } solana-system-transaction = { workspace = true } solana-sysvar = { workspace = true } solana-time-utils = { workspace = true } +solana-tls-utils = { workspace = true } solana-tpu-client = { workspace = true } solana-transaction = { workspace = true } solana-transaction-context = { workspace = true } diff --git a/rpc/src/rpc_service.rs b/rpc/src/rpc_service.rs index ba18ef14dd81e4..e2cddfc02cd5a2 100644 --- a/rpc/src/rpc_service.rs +++ b/rpc/src/rpc_service.rs @@ -34,7 +34,6 @@ use { solana_metrics::inc_new_counter_info, solana_perf::thread::renice_this_thread, solana_poh::poh_recorder::PohRecorder, - solana_quic_definitions::NotifyKeyUpdate, solana_runtime::{ bank::Bank, bank_forks::BankForks, commitment::BlockCommitmentCache, non_circulating_supply::calculate_non_circulating_supply, @@ -45,6 +44,7 @@ use { transaction_client::{TpuClientNextClient, TransactionClient}, }, solana_storage_bigtable::CredentialType, + solana_tls_utils::NotifyKeyUpdate, solana_validator_exit::Exit, std::{ net::{SocketAddr, UdpSocket}, diff --git a/send-transaction-service/Cargo.toml b/send-transaction-service/Cargo.toml index ec6ae2ba2a3b27..d2062670164383 100644 --- a/send-transaction-service/Cargo.toml +++ b/send-transaction-service/Cargo.toml @@ -30,10 +30,10 @@ solana-metrics = { workspace = true } solana-net-utils = { workspace = true, optional = true } solana-nonce-account = { workspace = true } solana-pubkey = { workspace = true } -solana-quic-definitions = { workspace = true } solana-runtime = { workspace = true } solana-signature = { workspace = true } solana-time-utils = { workspace = true } +solana-tls-utils = { workspace = true } solana-tpu-client-next = { workspace = true, features = ["metrics"] } tokio = { workspace = true, features = ["full"] } tokio-util = { workspace = true } diff --git a/send-transaction-service/src/transaction_client.rs b/send-transaction-service/src/transaction_client.rs index 5e36b3378280f4..2862ef64cd4edb 100644 --- a/send-transaction-service/src/transaction_client.rs +++ b/send-transaction-service/src/transaction_client.rs @@ -4,7 +4,7 @@ use { log::warn, solana_keypair::Keypair, solana_measure::measure::Measure, - solana_quic_definitions::NotifyKeyUpdate, + solana_tls_utils::NotifyKeyUpdate, solana_tpu_client_next::{ connection_workers_scheduler::{ BindTarget, ConnectionWorkersSchedulerConfig, Fanout, StakeIdentity, diff --git a/streamer/Cargo.toml b/streamer/Cargo.toml index 376a29b8190ced..e1a72a74ca3f85 100644 --- a/streamer/Cargo.toml +++ b/streamer/Cargo.toml @@ -49,7 +49,6 @@ solana-net-utils = { workspace = true, features = ["agave-unstable-api"] } solana-packet = { workspace = true } solana-perf = { workspace = true } solana-pubkey = { workspace = true } -solana-quic-definitions = { workspace = true } solana-signature = { workspace = true } solana-signer = { workspace = true } solana-time-utils = { workspace = true } diff --git a/streamer/src/nonblocking/swqos.rs b/streamer/src/nonblocking/swqos.rs index 99a8dd8582de1c..cb7ba04f0d97d2 100644 --- a/streamer/src/nonblocking/swqos.rs +++ b/streamer/src/nonblocking/swqos.rs @@ -23,10 +23,6 @@ use { }, percentage::Percentage, quinn::{Connection, VarInt}, - solana_quic_definitions::{ - QUIC_MAX_STAKED_CONCURRENT_STREAMS, QUIC_MAX_UNSTAKED_CONCURRENT_STREAMS, - QUIC_MIN_STAKED_CONCURRENT_STREAMS, QUIC_TOTAL_STAKED_CONCURRENT_STREAMS, - }, solana_time_utils as timing, std::{ future::Future, @@ -39,6 +35,20 @@ use { tokio_util::sync::CancellationToken, }; +// Empirically found max number of concurrent streams +// that seems to maximize TPS on GCE (higher values don't seem to +// give significant improvement or seem to impact stability) +pub const QUIC_MAX_UNSTAKED_CONCURRENT_STREAMS: usize = 128; +pub const QUIC_MIN_STAKED_CONCURRENT_STREAMS: usize = 128; + +// Set the maximum concurrent stream numbers to avoid excessive streams. +// The value was lowered from 2048 to reduce contention of the limited +// receive_window among the streams which is observed in CI bench-tests with +// forwarded packets from staked nodes. +pub const QUIC_MAX_STAKED_CONCURRENT_STREAMS: usize = 512; + +pub const QUIC_TOTAL_STAKED_CONCURRENT_STREAMS: usize = 100_000; + #[derive(Clone)] pub struct SwQosConfig { pub max_streams_per_ms: u64, diff --git a/streamer/src/nonblocking/testing_utilities.rs b/streamer/src/nonblocking/testing_utilities.rs index 36ca9e95fb3ed5..b9f5b7fe37fc37 100644 --- a/streamer/src/nonblocking/testing_utilities.rs +++ b/streamer/src/nonblocking/testing_utilities.rs @@ -6,7 +6,7 @@ use { quic::spawn_server, swqos::{SwQos, SwQosConfig}, }, - quic::{QuicServerError, QuicStreamerConfig, StreamerStats}, + quic::{QuicServerError, QuicStreamerConfig, StreamerStats, QUIC_MAX_TIMEOUT}, streamer::StakedNodes, }, crossbeam_channel::{unbounded, Receiver, Sender}, @@ -20,7 +20,6 @@ use { SocketConfiguration as SocketConfig, }, solana_perf::packet::PacketBatch, - solana_quic_definitions::{QUIC_KEEP_ALIVE, QUIC_MAX_TIMEOUT, QUIC_SEND_FAIRNESS}, solana_tls_utils::{new_dummy_x509_certificate, tls_client_config_builder}, std::{ net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket}, @@ -31,6 +30,11 @@ use { tokio_util::sync::CancellationToken, }; +/// Duration for QUIC keep-alive in tests. Typically tests are running for shorter duration that +/// connection timeout and keep-alive is not strictly necessary. But for longer running tests, it +/// makes sense to have keep-alive enable and set the value to be around half of the connection timeout. +const QUIC_KEEP_ALIVE_FOR_TESTS: Duration = Duration::from_secs(5); + /// Spawn a streamer instance in the current tokio runtime. pub fn spawn_stake_weighted_qos_server( name: &'static str, @@ -80,8 +84,8 @@ pub fn get_client_config(keypair: &Keypair) -> ClientConfig { let mut transport_config = TransportConfig::default(); let timeout = IdleTimeout::try_from(QUIC_MAX_TIMEOUT).unwrap(); transport_config.max_idle_timeout(Some(timeout)); - transport_config.keep_alive_interval(Some(QUIC_KEEP_ALIVE)); - transport_config.send_fairness(QUIC_SEND_FAIRNESS); + transport_config.keep_alive_interval(Some(QUIC_KEEP_ALIVE_FOR_TESTS)); + transport_config.send_fairness(false); config.transport_config(Arc::new(transport_config)); config diff --git a/streamer/src/quic.rs b/streamer/src/quic.rs index 78bd9ebff378d7..4ce03cd9888eb8 100644 --- a/streamer/src/quic.rs +++ b/streamer/src/quic.rs @@ -18,8 +18,7 @@ use { solana_keypair::Keypair, solana_packet::PACKET_DATA_SIZE, solana_perf::packet::PacketBatch, - solana_quic_definitions::{NotifyKeyUpdate, QUIC_MAX_TIMEOUT}, - solana_tls_utils::{new_dummy_x509_certificate, tls_server_config_builder}, + solana_tls_utils::{new_dummy_x509_certificate, tls_server_config_builder, NotifyKeyUpdate}, std::{ net::UdpSocket, num::NonZeroUsize, @@ -34,6 +33,10 @@ use { tokio_util::sync::CancellationToken, }; +/// QUIC connection idle timeout. The connection will be closed if +/// there are no activities on it within the timeout window. +pub const QUIC_MAX_TIMEOUT: Duration = Duration::from_secs(60); + // allow multiple connections for NAT and any open/close overlap pub const DEFAULT_MAX_QUIC_CONNECTIONS_PER_UNSTAKED_PEER: usize = 8; diff --git a/tls-utils/src/lib.rs b/tls-utils/src/lib.rs index d61c22e8f8c0e3..9fd1c2440ada8d 100644 --- a/tls-utils/src/lib.rs +++ b/tls-utils/src/lib.rs @@ -27,3 +27,6 @@ pub use skip_server_verification::SkipServerVerification; mod skip_client_verification; pub use skip_client_verification::SkipClientVerification; + +pub mod notify_key_update; +pub use notify_key_update::NotifyKeyUpdate; diff --git a/tls-utils/src/notify_key_update.rs b/tls-utils/src/notify_key_update.rs new file mode 100644 index 00000000000000..978fcb9445e695 --- /dev/null +++ b/tls-utils/src/notify_key_update.rs @@ -0,0 +1,9 @@ +use solana_keypair::Keypair; + +/// [`NotifyKeyUpdate`] is a trait used for updating the certificate used for QUIC connections. +/// +/// When validator receives signal to update its identity through the admin_rpc, we need to change +/// the keypair used for QUIC connections. This trait provides an interface for that. +pub trait NotifyKeyUpdate { + fn update_key(&self, key: &Keypair) -> Result<(), Box>; +} diff --git a/vortexor/Cargo.toml b/vortexor/Cargo.toml index e8a7dd15659b79..24d66fe5bc3ef7 100644 --- a/vortexor/Cargo.toml +++ b/vortexor/Cargo.toml @@ -57,9 +57,9 @@ solana-metrics = { workspace = true } solana-net-utils = { workspace = true } solana-perf = { workspace = true } solana-pubkey = { workspace = true } -solana-quic-definitions = { workspace = true } solana-signer = { workspace = true } solana-streamer = { workspace = true } +solana-tls-utils = { workspace = true } solana-transaction-metrics-tracker = { workspace = true } solana-version = { workspace = true } thiserror = { workspace = true } diff --git a/vortexor/src/vortexor.rs b/vortexor/src/vortexor.rs index 8481b22913d8a7..e66c20ee9ad70a 100644 --- a/vortexor/src/vortexor.rs +++ b/vortexor/src/vortexor.rs @@ -9,7 +9,6 @@ use { multi_bind_in_range_with_config, SocketConfiguration as SocketConfig, }, solana_perf::packet::PacketBatch, - solana_quic_definitions::NotifyKeyUpdate, solana_streamer::{ nonblocking::{quic::DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, swqos::SwQosConfig}, quic::{ @@ -18,6 +17,7 @@ use { }, streamer::StakedNodes, }, + solana_tls_utils::NotifyKeyUpdate, std::{ net::{SocketAddr, UdpSocket}, sync::{Arc, Mutex, RwLock},