Skip to content

Support receiving verified transactions from the vortexor #5321

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lijunwangs merged 9 commits into from
Apr 21, 2025
Merged

Support receiving verified transactions from the vortexor #5321

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
73965a3
Support receiving verified transactions from the vortexor
lijunwangs Mar 17, 2025
8061476
vortexor receiver adapter
lijunwangs Mar 17, 2025
3fd83f0
Update some comments
lijunwangs Apr 9, 2025
053ae73
Update some comments
lijunwangs Apr 9, 2025
08c8e6b
Gaurd against 0 packets
lijunwangs Apr 9, 2025
9c8b817
update some debug messages
lijunwangs Apr 9, 2025
bfb49f3
disable regular tpu streamers if vortexor is in-place
lijunwangs Apr 11, 2025
bb8280e
disable regular tpu streamers if vortexor is in-place
lijunwangs Apr 11, 2025
59b8dca
Addressed some comments from Andrew. Hide the tpu-vortexor-receiver-a…
lijunwangs Apr 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,7 @@ members = [
"validator",
"version",
"vortexor",
"vortexor-receiver",
"vote",
"watchtower",
"wen-restart",
Expand Down Expand Up @@ -567,6 +568,7 @@ solana-type-overrides = { path = "type-overrides", version = "=2.3.0" }
solana-udp-client = { path = "udp-client", version = "=2.3.0" }
solana-validator-exit = "2.2.1"
solana-version = { path = "version", version = "=2.3.0" }
solana-vortexor-receiver = { path = "vortexor-receiver", version = "=2.3.0" }
solana-vote = { path = "vote", version = "=2.3.0" }
solana-vote-interface = "2.2.4"
solana-vote-program = { path = "programs/vote", version = "=2.3.0", default-features = false }
Expand Down
1 change: 1 addition & 0 deletions core/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ solana-transaction-status = { workspace = true }
solana-turbine = { workspace = true }
solana-unified-scheduler-pool = { workspace = true }
solana-version = { workspace = true }
solana-vortexor-receiver = { workspace = true }
solana-vote = { workspace = true }
solana-vote-program = { workspace = true }
solana-wen-restart = { workspace = true }
Expand Down
1 change: 1 addition & 0 deletions core/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ mod tpu_entry_notifier;
pub mod tvu;
pub mod unfrozen_gossip_verified_vote_hashes;
pub mod validator;
mod vortexor_receiver_adapter;
pub mod vote_simulator;
pub mod voting_service;
pub mod warm_quic_cache_service;
Expand Down
138 changes: 96 additions & 42 deletions core/src/tpu.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ use {
staked_nodes_updater_service::StakedNodesUpdaterService,
tpu_entry_notifier::TpuEntryNotifier,
validator::{BlockProductionMethod, GeneratorConfig, TransactionStructure},
vortexor_receiver_adapter::VortexorReceiverAdapter,
},
bytes::Bytes,
crossbeam_channel::{bounded, unbounded, Receiver},
Expand Down Expand Up @@ -73,18 +74,34 @@ pub struct TpuSockets {
pub vote_quic: Vec<UdpSocket>,
/// Client-side socket for the forwarding votes.
pub vote_forwards_client: UdpSocket,
pub vortexor_receivers: Option<Vec<UdpSocket>>,
}

/// The `SigVerifier` enum is used to determine whether to use a local or remote signature verifier.
enum SigVerifier {
Local(SigVerifyStage),
Remote(VortexorReceiverAdapter),
}

impl SigVerifier {
fn join(self) -> thread::Result<()> {
match self {
SigVerifier::Local(sig_verify_stage) => sig_verify_stage.join(),
SigVerifier::Remote(vortexor_receiver_adapter) => vortexor_receiver_adapter.join(),
}
}
}

pub struct Tpu {
fetch_stage: FetchStage,
sigverify_stage: SigVerifyStage,
sig_verifier: SigVerifier,
vote_sigverify_stage: SigVerifyStage,
banking_stage: BankingStage,
forwarding_stage: JoinHandle<()>,
cluster_info_vote_listener: ClusterInfoVoteListener,
broadcast_stage: BroadcastStage,
tpu_quic_t: thread::JoinHandle<()>,
tpu_forwards_quic_t: thread::JoinHandle<()>,
tpu_quic_t: Option<thread::JoinHandle<()>>,
tpu_forwards_quic_t: Option<thread::JoinHandle<()>>,
tpu_entry_notifier: Option<TpuEntryNotifier>,
staked_nodes_updater_service: StakedNodesUpdaterService,
tracer_thread_hdl: TracerThread,
Expand Down Expand Up @@ -143,6 +160,7 @@ impl Tpu {
transactions_forwards_quic: transactions_forwards_quic_sockets,
vote_quic: tpu_vote_quic_sockets,
vote_forwards_client: vote_forwards_client_socket,
vortexor_receivers,
} = sockets;

let (packet_sender, packet_receiver) = unbounded();
Expand Down Expand Up @@ -196,47 +214,75 @@ impl Tpu {
)
.unwrap();

// Streamer for TPU
let SpawnServerResult {
endpoints: _,
thread: tpu_quic_t,
key_updater,
} = spawn_server_multi(
"solQuicTpu",
"quic_streamer_tpu",
transactions_quic_sockets,
keypair,
packet_sender,
exit.clone(),
staked_nodes.clone(),
tpu_quic_server_config,
)
.unwrap();
let (tpu_quic_t, key_updater) = if vortexor_receivers.is_none() {
// Streamer for TPU
let SpawnServerResult {
endpoints: _,
thread: tpu_quic_t,
key_updater,
} = spawn_server_multi(
"solQuicTpu",
"quic_streamer_tpu",
transactions_quic_sockets,
keypair,
packet_sender,
exit.clone(),
staked_nodes.clone(),
tpu_quic_server_config,
)
.unwrap();
(Some(tpu_quic_t), Some(key_updater))
} else {
(None, None)
};

// Streamer for TPU forward
let SpawnServerResult {
endpoints: _,
thread: tpu_forwards_quic_t,
key_updater: forwards_key_updater,
} = spawn_server_multi(
"solQuicTpuFwd",
"quic_streamer_tpu_forwards",
transactions_forwards_quic_sockets,
keypair,
forwarded_packet_sender,
exit.clone(),
staked_nodes.clone(),
tpu_fwd_quic_server_config,
)
.unwrap();
let (tpu_forwards_quic_t, forwards_key_updater) = if vortexor_receivers.is_none() {
// Streamer for TPU forward
let SpawnServerResult {
endpoints: _,
thread: tpu_forwards_quic_t,
key_updater: forwards_key_updater,
} = spawn_server_multi(
"solQuicTpuFwd",
"quic_streamer_tpu_forwards",
transactions_forwards_quic_sockets,
keypair,
forwarded_packet_sender,
exit.clone(),
staked_nodes.clone(),
tpu_fwd_quic_server_config,
)
.unwrap();
(Some(tpu_forwards_quic_t), Some(forwards_key_updater))
} else {
(None, None)
};

let (forward_stage_sender, forward_stage_receiver) = bounded(1024);
let sigverify_stage = {
let sig_verifier = if let Some(vortexor_receivers) = vortexor_receivers {

@apfitzge apfitzge Apr 10, 2025

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should optionally spawn vortexor receiver, but we should still spawn the local sigverify stage.

AFAICT the quic and fetch stages still exist, and even if the ports are not advertised on gossip it's possible to still send to them.

This also makes upgrades tied together. I can't restart my vortexor instance because my validator is relying on it.
We should let the operator switch their advertised tpu port(s) at runtime, so they can switch back to local mode to upgrade vortexor without screwing up the validator.
Or switch to another vortexor instance.

@lijunwangs lijunwangs Apr 10, 2025

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Great point. We plan to support the dynamic management of subscriptions via Admin RPC on the validator. See https://github.com/anza-xyz/agave/blob/master/vortexor/Readme.md

@apfitzge apfitzge Apr 10, 2025

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right - but we shouldn't just let the quic & fetch stages send to a SV that doesn't exist. It's likely we will cause a panic in that case soon, and then it's a vulnerability if I can guess the ports your tpu is on.

Even if we can't switch off of it in this PR, we shouldn't remove SV imo

@lijunwangs lijunwangs Apr 10, 2025

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point! Will address

@lijunwangs lijunwangs Apr 12, 2025

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Disable TPU streamers when vortexor receiver is configured.

@apfitzge apfitzge Apr 16, 2025

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry for the delay, but this seems the wrong direction to me.
If the vortexor goes down, the operator is forced to restart their node to go back to normal TPU. This seems like a necessary feature to me, not something that we should do as follow-up.

I'm happy to hear other's opinions, as maybe I'm being overly cautious about this.

@lijunwangs lijunwangs Apr 16, 2025 •
edited
Loading

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This feature is only enabled when tpu-vortexor-receiver-address enabled. Auto fullback and heartbeat feature will be delivered in follow-on PRs.

info!("starting vortexor adapter");
let sockets = vortexor_receivers.into_iter().map(Arc::new).collect();
let adapter = VortexorReceiverAdapter::new(
sockets,
Duration::from_millis(5),
tpu_coalesce,
non_vote_sender,
enable_block_production_forwarding.then(|| forward_stage_sender.clone()),
exit.clone(),
);
SigVerifier::Remote(adapter)
} else {
info!("starting regular sigverify stage");
let verifier = TransactionSigVerifier::new(
non_vote_sender,
enable_block_production_forwarding.then(|| forward_stage_sender.clone()),
);
SigVerifyStage::new(packet_receiver, verifier, "solSigVerTpu", "tpu-verifier")
SigVerifier::Local(SigVerifyStage::new(
packet_receiver,
verifier,
"solSigVerTpu",
"tpu-verifier",
))
};

let vote_sigverify_stage = {

@apfitzge apfitzge Apr 16, 2025

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Votes do not go through vortexor? will they in the future? if I want to offload the sigverify task it seems it should be fully offloaded.

@lijunwangs lijunwangs Apr 16, 2025

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed. I think we can offload votes to vortexor as well. To be done in future PRs.

Expand Down Expand Up @@ -318,10 +364,18 @@ impl Tpu {
turbine_quic_endpoint_sender,
);

let mut key_updaters: Vec<Arc<dyn NotifyKeyUpdate + Send + Sync>> = Vec::new();
if let Some(key_updater) = key_updater {
key_updaters.push(key_updater);
}
if let Some(forwards_key_updater) = forwards_key_updater {
key_updaters.push(forwards_key_updater);
}
key_updaters.push(vote_streamer_key_updater);
(
Self {
fetch_stage,
sigverify_stage,
sig_verifier,
vote_sigverify_stage,
banking_stage,
forwarding_stage,
Expand All @@ -334,21 +388,21 @@ impl Tpu {
tracer_thread_hdl,
tpu_vote_quic_t,
},
vec![key_updater, forwards_key_updater, vote_streamer_key_updater],
key_updaters,
)
}

pub fn join(self) -> thread::Result<()> {
let results = vec![
self.fetch_stage.join(),
self.sigverify_stage.join(),
self.sig_verifier.join(),
self.vote_sigverify_stage.join(),
self.cluster_info_vote_listener.join(),
self.banking_stage.join(),
self.forwarding_stage.join(),
self.staked_nodes_updater_service.join(),
self.tpu_quic_t.join(),
self.tpu_forwards_quic_t.join(),
self.tpu_quic_t.map_or(Ok(()), |t| t.join()),
self.tpu_forwards_quic_t.map_or(Ok(()), |t| t.join()),
self.tpu_vote_quic_t.join(),
];
let broadcast_result = self.broadcast_stage.join();
Expand Down
1 change: 1 addition & 0 deletions core/src/validator.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1580,6 +1580,7 @@ impl Validator {
transactions_forwards_quic: node.sockets.tpu_forwards_quic,
vote_quic: node.sockets.tpu_vote_quic,
vote_forwards_client: node.sockets.tpu_vote_forwards_client,
vortexor_receivers: node.sockets.vortexor_receivers,
},
&rpc_subscriptions,
transaction_status_sender,
Expand Down
Loading