From 808865fdcd2c09e1203a080856f6d2b3b9e004d4 Mon Sep 17 00:00:00 2001 From: kamuik16 Date: Fri, 7 Feb 2025 14:00:18 +0530 Subject: [PATCH 1/2] feat: add cli flags to delay publishing --- beacon_node/beacon_chain/src/chain_config.rs | 6 ++++++ beacon_node/http_api/src/publish_blocks.rs | 16 +++++++++++++- beacon_node/src/cli.rs | 22 ++++++++++++++++++++ beacon_node/src/config.rs | 8 +++++++ 4 files changed, 51 insertions(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/src/chain_config.rs b/beacon_node/beacon_chain/src/chain_config.rs index b8a607c8864..71ac406337a 100644 --- a/beacon_node/beacon_chain/src/chain_config.rs +++ b/beacon_node/beacon_chain/src/chain_config.rs @@ -94,6 +94,10 @@ pub struct ChainConfig { /// The delay in milliseconds applied by the node between sending each blob or data column batch. /// This doesn't apply if the node is the block proposer. pub blob_publication_batch_interval: Duration, + /// Artificial delay for block publishing. For PeerDAS testing only. + pub block_publishing_delay: Option, + /// Artificial delay for data column publishing. For PeerDAS testing only. + pub data_column_publishing_delay: Option, } impl Default for ChainConfig { @@ -129,6 +133,8 @@ impl Default for ChainConfig { enable_sampling: false, blob_publication_batches: 4, blob_publication_batch_interval: Duration::from_millis(300), + block_publishing_delay: None, + data_column_publishing_delay: None, } } } diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 60d4b2f16ed..2e6ced92d90 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -86,6 +86,8 @@ pub async fn publish_block>( network_globals: Arc>, ) -> Result { let seen_timestamp = timestamp_now(); + let block_publishing_delay = chain.config.block_publishing_delay; + let data_column_publishing_delay = chain.config.data_column_publishing_delay; let (unverified_block, unverified_blobs, is_locally_built_block) = match provenanced_block { ProvenancedBlock::Local(block, blobs, _) => (block, blobs, true), @@ -103,8 +105,13 @@ pub async fn publish_block>( let publish_block_p2p = move |block: Arc>, sender, log, - seen_timestamp| + seen_timestamp, + block_publishing_delay| -> Result<(), BlockError> { + // Add delay before publishing the block to the network. + if let Some(block_publishing_delay) = block_publishing_delay { + std::thread::sleep(block_publishing_delay); + } let publish_timestamp = timestamp_now(); let publish_delay = publish_timestamp .checked_sub(seen_timestamp) @@ -152,6 +159,7 @@ pub async fn publish_block>( sender_clone.clone(), log.clone(), seen_timestamp, + block_publishing_delay, ) .map_err(|_| warp_utils::reject::custom_server_error("unable to publish".into()))?; } @@ -167,6 +175,7 @@ pub async fn publish_block>( sender_clone.clone(), log.clone(), seen_timestamp, + block_publishing_delay, )?, BroadcastValidation::ConsensusAndEquivocation => { check_slashable(&chain, block_root, &block_to_publish, &log)?; @@ -175,6 +184,7 @@ pub async fn publish_block>( sender_clone.clone(), log.clone(), seen_timestamp, + block_publishing_delay, )?; } }; @@ -207,6 +217,10 @@ pub async fn publish_block>( } if gossip_verified_columns.iter().map(Option::is_some).count() > 0 { + // Add delay before publishing the data columns to the network. + if let Some(data_column_publishing_delay) = data_column_publishing_delay { + tokio::time::sleep(data_column_publishing_delay).await; + } publish_column_sidecars(network_tx, &gossip_verified_columns, &chain).map_err(|_| { warp_utils::reject::custom_server_error("unable to publish data column sidecars".into()) })?; diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index 29faa7f2203..4c46284f40f 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -1590,5 +1590,27 @@ pub fn cli_app() -> Command { .action(ArgAction::Set) .display_order(0) ) + .arg( + Arg::new("delay-block-publishing") + .long("delay-block-publishing") + .value_name("SECONDS") + .action(ArgAction::Set) + .help_heading(FLAG_HEADER) + .help("TESTING ONLY: Artificially delay block publishing by the specified number of seconds. \ + DO NOT USE IN PRODUCTION.") + .hide(true) + .display_order(0) + ) + .arg( + Arg::new("delay-data-column-publishing") + .long("delay-data-column-publishing") + .value_name("SECONDS") + .action(ArgAction::Set) + .help_heading(FLAG_HEADER) + .help("TESTING ONLY: Artificially delay data column publishing by the specified number of seconds. \ + DO NOT USE IN PRODUCTION.") + .hide(true) + .display_order(0) + ) .group(ArgGroup::new("enable_http").args(["http", "gui", "staking"]).multiple(true)) } diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 0f8f3a8012a..f303365bd32 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -887,6 +887,14 @@ pub fn get_config( .max_gossip_aggregate_batch_size = clap_utils::parse_required(cli_args, "beacon-processor-aggregate-batch-size")?; + if let Some(delay) = clap_utils::parse_optional(cli_args, "delay-block-publishing")? { + client_config.chain.block_publishing_delay = Some(Duration::from_secs(delay)); + } + + if let Some(delay) = clap_utils::parse_optional(cli_args, "delay-data-column-publishing")? { + client_config.chain.data_column_publishing_delay = Some(Duration::from_secs(delay)); + } + Ok(client_config) } From e3957d30996773e873bd20d69c6705c1443b85ce Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Wed, 19 Feb 2025 17:03:28 +1100 Subject: [PATCH 2/2] Remove use of std::thread::sleep in async context. --- beacon_node/http_api/src/publish_blocks.rs | 41 ++++++++++++++-------- beacon_node/src/cli.rs | 5 ++- beacon_node/src/config.rs | 4 +-- lighthouse/tests/beacon_node.rs | 26 ++++++++++++++ 4 files changed, 59 insertions(+), 17 deletions(-) diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 2e6ced92d90..072ae5dc03c 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -86,8 +86,8 @@ pub async fn publish_block>( network_globals: Arc>, ) -> Result { let seen_timestamp = timestamp_now(); - let block_publishing_delay = chain.config.block_publishing_delay; - let data_column_publishing_delay = chain.config.data_column_publishing_delay; + let block_publishing_delay_for_testing = chain.config.block_publishing_delay; + let data_column_publishing_delay_for_testing = chain.config.data_column_publishing_delay; let (unverified_block, unverified_blobs, is_locally_built_block) = match provenanced_block { ProvenancedBlock::Local(block, blobs, _) => (block, blobs, true), @@ -105,13 +105,8 @@ pub async fn publish_block>( let publish_block_p2p = move |block: Arc>, sender, log, - seen_timestamp, - block_publishing_delay| + seen_timestamp| -> Result<(), BlockError> { - // Add delay before publishing the block to the network. - if let Some(block_publishing_delay) = block_publishing_delay { - std::thread::sleep(block_publishing_delay); - } let publish_timestamp = timestamp_now(); let publish_delay = publish_timestamp .checked_sub(seen_timestamp) @@ -154,12 +149,19 @@ pub async fn publish_block>( let should_publish_block = gossip_verified_block_result.is_ok(); if BroadcastValidation::Gossip == validation_level && should_publish_block { + if let Some(block_publishing_delay) = block_publishing_delay_for_testing { + debug!( + log, + "Publishing block with artificial delay"; + "block_publishing_delay" => ?block_publishing_delay + ); + tokio::time::sleep(block_publishing_delay).await; + } publish_block_p2p( block.clone(), sender_clone.clone(), log.clone(), seen_timestamp, - block_publishing_delay, ) .map_err(|_| warp_utils::reject::custom_server_error("unable to publish".into()))?; } @@ -175,7 +177,6 @@ pub async fn publish_block>( sender_clone.clone(), log.clone(), seen_timestamp, - block_publishing_delay, )?, BroadcastValidation::ConsensusAndEquivocation => { check_slashable(&chain, block_root, &block_to_publish, &log)?; @@ -184,7 +185,6 @@ pub async fn publish_block>( sender_clone.clone(), log.clone(), seen_timestamp, - block_publishing_delay, )?; } }; @@ -217,9 +217,22 @@ pub async fn publish_block>( } if gossip_verified_columns.iter().map(Option::is_some).count() > 0 { - // Add delay before publishing the data columns to the network. - if let Some(data_column_publishing_delay) = data_column_publishing_delay { - tokio::time::sleep(data_column_publishing_delay).await; + if let Some(data_column_publishing_delay) = data_column_publishing_delay_for_testing { + // Subtract block publishing delay if it is also used. + // Note: if `data_column_publishing_delay` is less than `block_publishing_delay`, it + // will still be delayed by `block_publishing_delay`. This could be solved with spawning + // async tasks but the limitation is minor and I believe it's probably not worth + // affecting the mainnet code path. + let block_publishing_delay = block_publishing_delay_for_testing.unwrap_or_default(); + let delay = data_column_publishing_delay.saturating_sub(block_publishing_delay); + if !delay.is_zero() { + debug!( + log, + "Publishing data columns with artificial delay"; + "data_column_publishing_delay" => ?data_column_publishing_delay + ); + tokio::time::sleep(delay).await; + } } publish_column_sidecars(network_tx, &gossip_verified_columns, &chain).map_err(|_| { warp_utils::reject::custom_server_error("unable to publish data column sidecars".into()) diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index 4c46284f40f..f5211ced954 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -1597,7 +1597,8 @@ pub fn cli_app() -> Command { .action(ArgAction::Set) .help_heading(FLAG_HEADER) .help("TESTING ONLY: Artificially delay block publishing by the specified number of seconds. \ - DO NOT USE IN PRODUCTION.") + This only works for if `BroadcastValidation::Gossip` is used (default). \ + DO NOT USE IN PRODUCTION.") .hide(true) .display_order(0) ) @@ -1608,6 +1609,8 @@ pub fn cli_app() -> Command { .action(ArgAction::Set) .help_heading(FLAG_HEADER) .help("TESTING ONLY: Artificially delay data column publishing by the specified number of seconds. \ + Limitation: If `delay-block-publishing` is also used, data columns will be delayed for a \ + minimum of `delay-block-publishing` seconds. DO NOT USE IN PRODUCTION.") .hide(true) .display_order(0) diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index f303365bd32..5a74d261da6 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -888,11 +888,11 @@ pub fn get_config( clap_utils::parse_required(cli_args, "beacon-processor-aggregate-batch-size")?; if let Some(delay) = clap_utils::parse_optional(cli_args, "delay-block-publishing")? { - client_config.chain.block_publishing_delay = Some(Duration::from_secs(delay)); + client_config.chain.block_publishing_delay = Some(Duration::from_secs_f64(delay)); } if let Some(delay) = clap_utils::parse_optional(cli_args, "delay-data-column-publishing")? { - client_config.chain.data_column_publishing_delay = Some(Duration::from_secs(delay)); + client_config.chain.data_column_publishing_delay = Some(Duration::from_secs_f64(delay)); } Ok(client_config) diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index 1063a80ff40..255a366a0ae 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -2702,3 +2702,29 @@ fn beacon_node_backend_override() { assert_eq!(config.store.backend, BeaconNodeBackend::LevelDb); }); } + +#[test] +fn block_publishing_delay_for_testing() { + CommandLineTest::new() + .flag("delay-block-publishing", Some("2.5")) + .run_with_zero_port() + .with_config(|config| { + assert_eq!( + config.chain.block_publishing_delay, + Some(Duration::from_secs_f64(2.5f64)) + ); + }); +} + +#[test] +fn data_column_publishing_delay_for_testing() { + CommandLineTest::new() + .flag("delay-data-column-publishing", Some("3.5")) + .run_with_zero_port() + .with_config(|config| { + assert_eq!( + config.chain.data_column_publishing_delay, + Some(Duration::from_secs_f64(3.5f64)) + ); + }); +}