From 4e9b5bb35e63de8f96ad3a8db07a57e4dbb6bc10 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Wed, 21 Jun 2023 23:14:38 +1000 Subject: [PATCH 01/24] Add Deneb builder flow types with generics --- beacon_node/beacon_chain/src/beacon_chain.rs | 4 +- beacon_node/beacon_chain/src/blob_cache.rs | 17 ++-- beacon_node/beacon_chain/src/builder.rs | 1 + .../http_api/src/build_block_contents.rs | 38 +++++++- beacon_node/http_api/src/lib.rs | 9 +- common/eth2/src/lib.rs | 17 +++- common/eth2/src/types.rs | 91 +++++++++++++++---- consensus/types/src/deneb_types.rs | 65 +++++++++++++ consensus/types/src/lib.rs | 1 + validator_client/src/block_service.rs | 13 ++- 10 files changed, 214 insertions(+), 42 deletions(-) create mode 100644 consensus/types/src/deneb_types.rs diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index d410e860db0..c31d49a258e 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -119,6 +119,7 @@ use types::beacon_block_body::KzgCommitments; use types::beacon_state::CloneConfig; use types::blob_sidecar::{BlobSidecarList, Blobs}; use types::consts::deneb::MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS; +use types::deneb_types::BlindedBlobSidecar; use types::*; pub type ForkChoiceError = fork_choice::Error; @@ -465,7 +466,8 @@ pub struct BeaconChain { pub validator_monitor: RwLock>, /// The slot at which blocks are downloaded back to. pub genesis_backfill_slot: Slot, - pub proposal_blob_cache: BlobCache, + pub proposal_blob_cache: BlobCache>, + pub proposal_blinded_blob_cache: BlobCache, pub data_availability_checker: Arc>, pub kzg: Option>, } diff --git a/beacon_node/beacon_chain/src/blob_cache.rs b/beacon_node/beacon_chain/src/blob_cache.rs index 64f113c285c..4b072e599fa 100644 --- a/beacon_node/beacon_chain/src/blob_cache.rs +++ b/beacon_node/beacon_chain/src/blob_cache.rs @@ -1,18 +1,19 @@ use lru::LruCache; use parking_lot::Mutex; -use types::{BlobSidecarList, EthSpec, Hash256}; +use types::deneb_types::{AbstractSidecar, SidecarList}; +use types::{EthSpec, Hash256}; pub const DEFAULT_BLOB_CACHE_SIZE: usize = 10; /// A cache blobs by beacon block root. -pub struct BlobCache { - blobs: Mutex>>, +pub struct BlobCache> { + blobs: Mutex>>, } #[derive(Hash, PartialEq, Eq)] struct BlobCacheId(Hash256); -impl Default for BlobCache { +impl> Default for BlobCache { fn default() -> Self { BlobCache { blobs: Mutex::new(LruCache::new(DEFAULT_BLOB_CACHE_SIZE)), @@ -20,16 +21,16 @@ impl Default for BlobCache { } } -impl BlobCache { +impl> BlobCache { pub fn put( &self, beacon_block: Hash256, - blobs: BlobSidecarList, - ) -> Option> { + blobs: SidecarList, + ) -> Option> { self.blobs.lock().put(BlobCacheId(beacon_block), blobs) } - pub fn pop(&self, root: &Hash256) -> Option> { + pub fn pop(&self, root: &Hash256) -> Option> { self.blobs.lock().pop(&BlobCacheId(*root)) } } diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index a35e7d615d8..4de0c0b4bfe 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -891,6 +891,7 @@ where .map_err(|e| format!("Error initializing DataAvailabiltyChecker: {:?}", e))?, ), proposal_blob_cache: BlobCache::default(), + proposal_blinded_blob_cache: BlobCache::default(), kzg, }; diff --git a/beacon_node/http_api/src/build_block_contents.rs b/beacon_node/http_api/src/build_block_contents.rs index d6c5ada0071..12199773f48 100644 --- a/beacon_node/http_api/src/build_block_contents.rs +++ b/beacon_node/http_api/src/build_block_contents.rs @@ -1,15 +1,16 @@ use beacon_chain::{BeaconChain, BeaconChainTypes, BlockProductionError}; -use eth2::types::{BeaconBlockAndBlobSidecars, BlockContents}; +use eth2::types::{BeaconBlockAndBlobSidecars, BlindedBeaconBlockAndBlobSidecars, BlockContents}; use std::sync::Arc; -use types::{AbstractExecPayload, BeaconBlock, ForkName}; +use types::deneb_types::BlindedBlobSidecar; +use types::{BeaconBlock, BlindedPayload, BlobSidecar, ForkName, FullPayload}; type Error = warp::reject::Rejection; -pub fn build_block_contents>( +pub fn build_block_contents( fork_name: ForkName, chain: Arc>, - block: BeaconBlock, -) -> Result, Error> { + block: BeaconBlock>, +) -> Result, BlobSidecar>, Error> { match fork_name { ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { Ok(BlockContents::Block(block)) @@ -31,3 +32,30 @@ pub fn build_block_contents( + fork_name: ForkName, + chain: Arc>, + block: BeaconBlock>, +) -> Result, BlindedBlobSidecar>, Error> { + match fork_name { + ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { + Ok(BlockContents::Block(block)) + } + ForkName::Deneb => { + let block_root = &block.canonical_root(); + if let Some(blob_sidecars) = chain.proposal_blinded_blob_cache.pop(block_root) { + let block_and_blobs = BlindedBeaconBlockAndBlobSidecars { + blinded_block: block, + blinded_blob_sidecars: blob_sidecars, + }; + + Ok(BlockContents::BlindedBlockAndBlobSidecars(block_and_blobs)) + } else { + Err(warp_utils::reject::block_production_error( + BlockProductionError::NoBlobsCached, + )) + } + } + } +} diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 3edc6aa3012..900d6c118c4 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -2634,9 +2634,9 @@ pub fn serve( .map_err(inconsistent_fork_rejection)?; let block_contents = - build_block_contents::build_block_contents(fork_name, chain, block); + build_block_contents::build_block_contents(fork_name, chain, block)?; - fork_versioned_response(endpoint_version, fork_name, block_contents?) + fork_versioned_response(endpoint_version, fork_name, block_contents) .map(|response| warp::reply::json(&response).into_response()) }, ); @@ -2692,8 +2692,11 @@ pub fn serve( .fork_name(&chain.spec) .map_err(inconsistent_fork_rejection)?; + let block_contents = + build_block_contents::build_blinded_block_contents(fork_name, chain, block)?; + // Pose as a V2 endpoint so we return the fork `version`. - fork_versioned_response(V2, fork_name, block) + fork_versioned_response(V2, fork_name, block_contents) .map(|response| warp::reply::json(&response).into_response()) }, ); diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 83cdeddd80d..5be15464c7b 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -16,6 +16,7 @@ pub mod types; use self::mixin::{RequestAccept, ResponseOptional}; use self::types::{Error as ResponseError, *}; +use ::types::deneb_types::AbstractSidecar; use futures::Stream; use futures_util::StreamExt; use lighthouse_network::PeerId; @@ -1384,24 +1385,32 @@ impl BeaconNodeHttpClient { } /// `GET v2/validator/blocks/{slot}` - pub async fn get_validator_blocks>( + pub async fn get_validator_blocks< + T: EthSpec, + Payload: AbstractExecPayload, + Sidecar: AbstractSidecar, + >( &self, slot: Slot, randao_reveal: &SignatureBytes, graffiti: Option<&Graffiti>, - ) -> Result>, Error> { + ) -> Result>, Error> { self.get_validator_blocks_modular(slot, randao_reveal, graffiti, SkipRandaoVerification::No) .await } /// `GET v2/validator/blocks/{slot}` - pub async fn get_validator_blocks_modular>( + pub async fn get_validator_blocks_modular< + T: EthSpec, + Payload: AbstractExecPayload, + Sidecar: AbstractSidecar, + >( &self, slot: Slot, randao_reveal: &SignatureBytes, graffiti: Option<&Graffiti>, skip_randao_verification: SkipRandaoVerification, - ) -> Result>, Error> { + ) -> Result>, Error> { let mut path = self.eth_path(V2)?; path.path_segments_mut() diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index bee9b6f1398..8e9df0b8ce8 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -11,6 +11,7 @@ use std::convert::TryFrom; use std::fmt; use std::str::{from_utf8, FromStr}; use std::time::Duration; +use types::deneb_types::{AbstractSidecar, SidecarList}; pub use types::*; #[cfg(feature = "lighthouse")] @@ -1277,32 +1278,43 @@ mod tests { #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(untagged)] #[serde(bound = "T: EthSpec")] -pub enum BlockContents> { - BlockAndBlobSidecars(BeaconBlockAndBlobSidecars), +// TODO(jimmy): we should be able to remove the `Sidecar: AbstractSidecar> generic type parameter +pub enum BlockContents, Sidecar: AbstractSidecar> { + BlockAndBlobSidecars(BeaconBlockAndBlobSidecars), + BlindedBlockAndBlobSidecars(BlindedBeaconBlockAndBlobSidecars), Block(BeaconBlock), } -impl> BlockContents { +impl, Sidecar: AbstractSidecar> + BlockContents +{ pub fn block(&self) -> &BeaconBlock { match self { BlockContents::BlockAndBlobSidecars(block_and_sidecars) => &block_and_sidecars.block, + BlockContents::BlindedBlockAndBlobSidecars(block_and_sidecars) => { + &block_and_sidecars.blinded_block + } BlockContents::Block(block) => block, } } - pub fn deconstruct(self) -> (BeaconBlock, Option>) { + pub fn deconstruct(self) -> (BeaconBlock, Option>) { match self { BlockContents::BlockAndBlobSidecars(block_and_sidecars) => ( block_and_sidecars.block, Some(block_and_sidecars.blob_sidecars), ), + BlockContents::BlindedBlockAndBlobSidecars(block_and_sidecars) => ( + block_and_sidecars.blinded_block, + Some(block_and_sidecars.blinded_blob_sidecars), + ), BlockContents::Block(block) => (block, None), } } } -impl> ForkVersionDeserialize - for BlockContents +impl, Sidecar: AbstractSidecar> + ForkVersionDeserialize for BlockContents { fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( value: serde_json::value::Value, @@ -1322,12 +1334,15 @@ impl> ForkVersionDeserialize } } -impl> Into> - for BlockContents +impl, Sidecar: AbstractSidecar> + Into> for BlockContents { fn into(self) -> BeaconBlock { match self { Self::BlockAndBlobSidecars(block_and_sidecars) => block_and_sidecars.block, + Self::BlindedBlockAndBlobSidecars(block_and_sidecars) => { + block_and_sidecars.blinded_block + } Self::Block(block) => block, } } @@ -1407,26 +1422,31 @@ pub struct SignedBeaconBlockAndBlobSidecars> { +#[serde(bound = "T: EthSpec, Payload: AbstractExecPayload, Sidecar: AbstractSidecar")] +pub struct BeaconBlockAndBlobSidecars< + T: EthSpec, + Payload: AbstractExecPayload, + Sidecar: AbstractSidecar, +> { pub block: BeaconBlock, - pub blob_sidecars: BlobSidecarList, + pub blob_sidecars: SidecarList, } -impl> ForkVersionDeserialize - for BeaconBlockAndBlobSidecars +impl, Sidecar: AbstractSidecar> + ForkVersionDeserialize for BeaconBlockAndBlobSidecars { fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( value: serde_json::value::Value, fork_name: ForkName, ) -> Result { #[derive(Deserialize)] - #[serde(bound = "T: EthSpec")] - struct Helper { + #[serde(bound = "T: EthSpec, Sidecar: AbstractSidecar")] + struct Helper> { block: serde_json::Value, - blob_sidecars: BlobSidecarList, + blob_sidecars: SidecarList, } - let helper: Helper = serde_json::from_value(value).map_err(serde::de::Error::custom)?; + let helper: Helper = + serde_json::from_value(value).map_err(serde::de::Error::custom)?; Ok(Self { block: BeaconBlock::deserialize_by_fork::<'de, D>(helper.block, fork_name)?, @@ -1434,3 +1454,40 @@ impl> ForkVersionDeserialize }) } } + +#[derive(Debug, Clone, Serialize, Deserialize, Encode)] +#[serde(bound = "T: EthSpec")] +pub struct BlindedBeaconBlockAndBlobSidecars< + T: EthSpec, + Payload: AbstractExecPayload, + Sidecar: AbstractSidecar, +> { + pub blinded_block: BeaconBlock, + pub blinded_blob_sidecars: SidecarList, +} + +impl, Sidecar: AbstractSidecar> + ForkVersionDeserialize for BlindedBeaconBlockAndBlobSidecars +{ + fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( + value: serde_json::value::Value, + fork_name: ForkName, + ) -> Result { + #[derive(Deserialize)] + #[serde(bound = "T: EthSpec, Sidecar: AbstractSidecar,")] + struct Helper> { + blinded_block: serde_json::Value, + blinded_blob_sidecars: SidecarList, + } + let helper: Helper = + serde_json::from_value(value).map_err(serde::de::Error::custom)?; + + Ok(Self { + blinded_block: BeaconBlock::deserialize_by_fork::<'de, D>( + helper.blinded_block, + fork_name, + )?, + blinded_blob_sidecars: helper.blinded_blob_sidecars, + }) + } +} diff --git a/consensus/types/src/deneb_types.rs b/consensus/types/src/deneb_types.rs new file mode 100644 index 00000000000..8902b85d9b2 --- /dev/null +++ b/consensus/types/src/deneb_types.rs @@ -0,0 +1,65 @@ +use derivative::Derivative; +use serde::de::DeserializeOwned; +use serde_derive::{Deserialize, Serialize}; +use ssz::{Decode, Encode}; +use ssz_derive::{Decode, Encode}; +use ssz_types::VariableList; +use std::sync::Arc; +use tree_hash_derive::TreeHash; + +use kzg::{KzgCommitment, KzgProof}; +use test_random_derive::TestRandom; + +use crate::test_utils::TestRandom; +use crate::{BeaconBlock, BlindedPayload, BlobSidecar, EthSpec, Hash256, SignedBeaconBlock, Slot}; + +// TODO(jimmy): to be moved to respective type files. + +#[derive( + Debug, + Clone, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + Default, + TestRandom, + Derivative, + arbitrary::Arbitrary, +)] +#[derivative(PartialEq, Eq, Hash)] +pub struct BlindedBlobSidecar { + pub block_root: Hash256, + #[serde(with = "serde_utils::quoted_u64")] + pub index: u64, + pub slot: Slot, + pub block_parent_root: Hash256, + #[serde(with = "serde_utils::quoted_u64")] + pub proposer_index: u64, + pub blob_root: Hash256, + pub kzg_commitment: KzgCommitment, + pub kzg_proof: KzgProof, +} + +type BlindedBlobSidecarList = VariableList::MaxBlobsPerBlock>; + +struct BlindedBlockContents { + blinded_block: BeaconBlock>, + blinded_blob_sidecars: BlindedBlobSidecarList, +} + +struct SignedBlindedBlockContents { + signed_blinded_block: SignedBeaconBlock>, + signed_blinded_blob_sidecars: BlindedBlobSidecarList, +} + +pub trait AbstractSidecar: + serde::Serialize + DeserializeOwned + Encode + Decode +{ +} + +impl AbstractSidecar for BlobSidecar {} +impl AbstractSidecar for BlindedBlobSidecar {} + +pub type SidecarList = VariableList, ::MaxBlobsPerBlock>; diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 46c5c2a4ce8..288d8a79c96 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -98,6 +98,7 @@ pub mod slot_data; pub mod sqlite; pub mod blob_sidecar; +pub mod deneb_types; pub mod signed_blob; pub mod transaction; diff --git a/validator_client/src/block_service.rs b/validator_client/src/block_service.rs index b5ae2f19218..14aa6c48553 100644 --- a/validator_client/src/block_service.rs +++ b/validator_client/src/block_service.rs @@ -19,6 +19,7 @@ use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc; use tokio::time::sleep; +use types::deneb_types::AbstractSidecar; use types::{ AbstractExecPayload, BlindedPayload, BlockType, EthSpec, FullPayload, Graffiti, PublicKeyBytes, Slot, @@ -581,22 +582,26 @@ impl BlockService { Ok::<_, BlockError>(()) } - async fn get_validator_block>( + async fn get_validator_block, Sidecar: AbstractSidecar>( beacon_node: &BeaconNodeHttpClient, slot: Slot, randao_reveal_ref: &SignatureBytes, graffiti: Option, proposer_index: Option, log: &Logger, - ) -> Result, BlockError> { - let block_contents: BlockContents = match Payload::block_type() { + ) -> Result, BlockError> { + let block_contents: BlockContents = match Payload::block_type() { BlockType::Full => { let _get_timer = metrics::start_timer_vec( &metrics::BLOCK_SERVICE_TIMES, &[metrics::BEACON_BLOCK_HTTP_GET], ); beacon_node - .get_validator_blocks::(slot, randao_reveal_ref, graffiti.as_ref()) + .get_validator_blocks::( + slot, + randao_reveal_ref, + graffiti.as_ref(), + ) .await .map_err(|e| { BlockError::Recoverable(format!( From d772adaea9a0ad342e7e98579e9b824c97b59ab5 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Thu, 22 Jun 2023 14:11:19 +1000 Subject: [PATCH 02/24] Update validator client `get_blinded_blocks` call to support Deneb --- common/eth2/src/lib.rs | 11 +++++++--- common/eth2/src/types.rs | 1 - validator_client/src/block_service.rs | 31 ++++++++++----------------- 3 files changed, 19 insertions(+), 24 deletions(-) diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 5be15464c7b..05938f914d2 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -1436,12 +1436,16 @@ impl BeaconNodeHttpClient { } /// `GET v2/validator/blinded_blocks/{slot}` - pub async fn get_validator_blinded_blocks>( + pub async fn get_validator_blinded_blocks< + T: EthSpec, + Payload: AbstractExecPayload, + Sidecar: AbstractSidecar, + >( &self, slot: Slot, randao_reveal: &SignatureBytes, graffiti: Option<&Graffiti>, - ) -> Result>, Error> { + ) -> Result>, Error> { self.get_validator_blinded_blocks_modular( slot, randao_reveal, @@ -1455,13 +1459,14 @@ impl BeaconNodeHttpClient { pub async fn get_validator_blinded_blocks_modular< T: EthSpec, Payload: AbstractExecPayload, + Sidecar: AbstractSidecar, >( &self, slot: Slot, randao_reveal: &SignatureBytes, graffiti: Option<&Graffiti>, skip_randao_verification: SkipRandaoVerification, - ) -> Result>, Error> { + ) -> Result>, Error> { let mut path = self.eth_path(V1)?; path.path_segments_mut() diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 8e9df0b8ce8..0079f68e09c 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -1278,7 +1278,6 @@ mod tests { #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(untagged)] #[serde(bound = "T: EthSpec")] -// TODO(jimmy): we should be able to remove the `Sidecar: AbstractSidecar> generic type parameter pub enum BlockContents, Sidecar: AbstractSidecar> { BlockAndBlobSidecars(BeaconBlockAndBlobSidecars), BlindedBlockAndBlobSidecars(BlindedBeaconBlockAndBlobSidecars), diff --git a/validator_client/src/block_service.rs b/validator_client/src/block_service.rs index 14aa6c48553..13632c17c3f 100644 --- a/validator_client/src/block_service.rs +++ b/validator_client/src/block_service.rs @@ -597,11 +597,7 @@ impl BlockService { &[metrics::BEACON_BLOCK_HTTP_GET], ); beacon_node - .get_validator_blocks::( - slot, - randao_reveal_ref, - graffiti.as_ref(), - ) + .get_validator_blocks(slot, randao_reveal_ref, graffiti.as_ref()) .await .map_err(|e| { BlockError::Recoverable(format!( @@ -616,21 +612,16 @@ impl BlockService { &metrics::BLOCK_SERVICE_TIMES, &[metrics::BLINDED_BEACON_BLOCK_HTTP_GET], ); - todo!("implement blinded flow for blobs"); - // beacon_node - // .get_validator_blinded_blocks::( - // slot, - // randao_reveal_ref, - // graffiti.as_ref(), - // ) - // .await - // .map_err(|e| { - // BlockError::Recoverable(format!( - // "Error from beacon node when producing block: {:?}", - // e - // )) - // })? - // .data + beacon_node + .get_validator_blinded_blocks(slot, randao_reveal_ref, graffiti.as_ref()) + .await + .map_err(|e| { + BlockError::Recoverable(format!( + "Error from beacon node when producing block: {:?}", + e + )) + })? + .data } }; From 1d4c55af630a57f1dd7afc1e649c1c93d51140d9 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Thu, 22 Jun 2023 17:20:19 +1000 Subject: [PATCH 03/24] `produceBlindedBlock` endpoint updates: - Handle new Deneb BuilderBid response from builder endpoint (new BlindedBlobsBundle type) - Build BlockContents response (containing kzg_commitments, proof and blinded_blob_sidecars) --- beacon_node/beacon_chain/src/beacon_chain.rs | 522 +++++++++++-------- beacon_node/beacon_chain/src/errors.rs | 1 + beacon_node/execution_layer/src/lib.rs | 142 ++--- consensus/types/src/blob_sidecar.rs | 1 + consensus/types/src/builder_bid.rs | 88 ++-- consensus/types/src/deneb_types.rs | 25 + 6 files changed, 466 insertions(+), 313 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index c31d49a258e..2891ceb76f9 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -117,9 +117,9 @@ use tokio_stream::Stream; use tree_hash::TreeHash; use types::beacon_block_body::KzgCommitments; use types::beacon_state::CloneConfig; -use types::blob_sidecar::{BlobSidecarList, Blobs}; +use types::blob_sidecar::{BlobRoots, BlobSidecarList, Blobs}; use types::consts::deneb::MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS; -use types::deneb_types::BlindedBlobSidecar; +use types::deneb_types::{BlindedBlobSidecar, BlobsOrBlobRoots, SidecarList}; use types::*; pub type ForkChoiceError = fork_choice::Error; @@ -4716,167 +4716,8 @@ impl BeaconChain { block_contents: Option>, verification: ProduceBlockVerification, ) -> Result, BlockProductionError> { - let PartialBeaconBlock { - mut state, - slot, - proposer_index, - parent_root, - randao_reveal, - eth1_data, - graffiti, - proposer_slashings, - attester_slashings, - attestations, - deposits, - voluntary_exits, - sync_aggregate, - // We don't need the prepare payload handle since the `execution_payload` is passed into - // this function. We can assume that the handle has already been consumed in order to - // produce said `execution_payload`. - prepare_payload_handle: _, - bls_to_execution_changes, - } = partial_beacon_block; - - let (inner_block, blobs_opt, proofs_opt) = match &state { - BeaconState::Base(_) => ( - BeaconBlock::Base(BeaconBlockBase { - slot, - proposer_index, - parent_root, - state_root: Hash256::zero(), - body: BeaconBlockBodyBase { - randao_reveal, - eth1_data, - graffiti, - proposer_slashings: proposer_slashings.into(), - attester_slashings: attester_slashings.into(), - attestations: attestations.into(), - deposits: deposits.into(), - voluntary_exits: voluntary_exits.into(), - _phantom: PhantomData, - }, - }), - None, - None, - ), - BeaconState::Altair(_) => ( - BeaconBlock::Altair(BeaconBlockAltair { - slot, - proposer_index, - parent_root, - state_root: Hash256::zero(), - body: BeaconBlockBodyAltair { - randao_reveal, - eth1_data, - graffiti, - proposer_slashings: proposer_slashings.into(), - attester_slashings: attester_slashings.into(), - attestations: attestations.into(), - deposits: deposits.into(), - voluntary_exits: voluntary_exits.into(), - sync_aggregate: sync_aggregate - .ok_or(BlockProductionError::MissingSyncAggregate)?, - _phantom: PhantomData, - }, - }), - None, - None, - ), - BeaconState::Merge(_) => { - let (payload, _, _, _) = block_contents - .ok_or(BlockProductionError::MissingExecutionPayload)? - .deconstruct(); - ( - BeaconBlock::Merge(BeaconBlockMerge { - slot, - proposer_index, - parent_root, - state_root: Hash256::zero(), - body: BeaconBlockBodyMerge { - randao_reveal, - eth1_data, - graffiti, - proposer_slashings: proposer_slashings.into(), - attester_slashings: attester_slashings.into(), - attestations: attestations.into(), - deposits: deposits.into(), - voluntary_exits: voluntary_exits.into(), - sync_aggregate: sync_aggregate - .ok_or(BlockProductionError::MissingSyncAggregate)?, - execution_payload: payload - .try_into() - .map_err(|_| BlockProductionError::InvalidPayloadFork)?, - }, - }), - None, - None, - ) - } - BeaconState::Capella(_) => { - let (payload, _, _, _) = block_contents - .ok_or(BlockProductionError::MissingExecutionPayload)? - .deconstruct(); - ( - BeaconBlock::Capella(BeaconBlockCapella { - slot, - proposer_index, - parent_root, - state_root: Hash256::zero(), - body: BeaconBlockBodyCapella { - randao_reveal, - eth1_data, - graffiti, - proposer_slashings: proposer_slashings.into(), - attester_slashings: attester_slashings.into(), - attestations: attestations.into(), - deposits: deposits.into(), - voluntary_exits: voluntary_exits.into(), - sync_aggregate: sync_aggregate - .ok_or(BlockProductionError::MissingSyncAggregate)?, - execution_payload: payload - .try_into() - .map_err(|_| BlockProductionError::InvalidPayloadFork)?, - bls_to_execution_changes: bls_to_execution_changes.into(), - }, - }), - None, - None, - ) - } - BeaconState::Deneb(_) => { - let (payload, kzg_commitments, blobs, proofs) = block_contents - .ok_or(BlockProductionError::MissingExecutionPayload)? - .deconstruct(); - ( - BeaconBlock::Deneb(BeaconBlockDeneb { - slot, - proposer_index, - parent_root, - state_root: Hash256::zero(), - body: BeaconBlockBodyDeneb { - randao_reveal, - eth1_data, - graffiti, - proposer_slashings: proposer_slashings.into(), - attester_slashings: attester_slashings.into(), - attestations: attestations.into(), - deposits: deposits.into(), - voluntary_exits: voluntary_exits.into(), - sync_aggregate: sync_aggregate - .ok_or(BlockProductionError::MissingSyncAggregate)?, - execution_payload: payload - .try_into() - .map_err(|_| BlockProductionError::InvalidPayloadFork)?, - bls_to_execution_changes: bls_to_execution_changes.into(), - blob_kzg_commitments: kzg_commitments - .ok_or(BlockProductionError::InvalidPayloadFork)?, - }, - }), - blobs, - proofs, - ) - } - }; + let (mut state, inner_block, blobs_opt, proofs_opt) = + Self::merge_block_contents_into_beacon_block(partial_beacon_block, block_contents)?; let block = SignedBeaconBlock::from_block( inner_block, @@ -4926,70 +4767,44 @@ impl BeaconChain { //FIXME(sean) // - add a new timer for processing here - if let Some(blobs) = blobs_opt { - let kzg = self - .kzg - .as_ref() - .ok_or(BlockProductionError::TrustedSetupNotInitialized)?; - let beacon_block_root = block.canonical_root(); + if let Some(blobs_or_blobs_roots) = blobs_opt { let expected_kzg_commitments = block.body().blob_kzg_commitments().map_err(|_| { BlockProductionError::InvalidBlockVariant( "DENEB block does not contain kzg commitments".to_string(), ) })?; - if expected_kzg_commitments.len() != blobs.len() { + if expected_kzg_commitments.len() != blobs_or_blobs_roots.len() { return Err(BlockProductionError::MissingKzgCommitment(format!( "Missing KZG commitment for slot {}. Expected {}, got: {}", - slot, - blobs.len(), + block.slot(), + blobs_or_blobs_roots.len(), expected_kzg_commitments.len() ))); } - let kzg_proofs = if let Some(proofs) = proofs_opt { - Vec::from(proofs) - } else { - Self::compute_blob_kzg_proofs(kzg, &blobs, expected_kzg_commitments, slot)? - }; - - kzg_utils::validate_blobs::( - kzg, - expected_kzg_commitments, - &blobs, - &kzg_proofs, - ) - .map_err(BlockProductionError::KzgError)?; - - let blob_sidecars = BlobSidecarList::from( - blobs - .into_iter() - .enumerate() - .map(|(blob_index, blob)| { - let kzg_commitment = expected_kzg_commitments - .get(blob_index) - .expect("KZG commitment should exist for blob"); - - let kzg_proof = kzg_proofs - .get(blob_index) - .expect("KZG proof should exist for blob"); - - Ok(Arc::new(BlobSidecar { - block_root: beacon_block_root, - index: blob_index as u64, - slot, - block_parent_root: block.parent_root(), - proposer_index, - blob, - kzg_commitment: *kzg_commitment, - kzg_proof: *kzg_proof, - })) - }) - .collect::, BlockProductionError>>()?, - ); + let kzg_proofs = proofs_opt + .ok_or(BlockProductionError::MissingKzgProof(format!( + "Missing KZG proofs for slot {}", + block.slot() + )))? + .to_vec(); - self.proposal_blob_cache - .put(beacon_block_root, blob_sidecars); + match blobs_or_blobs_roots { + BlobsOrBlobRoots::Blobs(blobs) => self.build_and_cache_blob_sidecars( + &block, + blobs, + expected_kzg_commitments, + kzg_proofs, + )?, + BlobsOrBlobRoots::BlobRoots(blob_roots) => self + .build_and_cache_blinded_blob_sidecars( + &block, + blob_roots, + expected_kzg_commitments, + kzg_proofs, + )?, + } } metrics::inc_counter(&metrics::BLOCK_PRODUCTION_SUCCESSES); @@ -6289,6 +6104,285 @@ impl BeaconChain { .map(|fork_epoch| fork_epoch <= current_epoch) .unwrap_or(false)) } + + fn build_and_cache_blob_sidecars>( + &self, + block: &BeaconBlock, + blobs: Blobs, + expected_kzg_commitments: &KzgCommitments, + kzg_proofs: Vec, + ) -> Result<(), BlockProductionError> { + let kzg = self + .kzg + .as_ref() + .ok_or(BlockProductionError::TrustedSetupNotInitialized)?; + let beacon_block_root = block.canonical_root(); + let slot = block.slot(); + + kzg_utils::validate_blobs::(kzg, expected_kzg_commitments, &blobs, &kzg_proofs) + .map_err(BlockProductionError::KzgError)?; + + let blob_sidecars = BlobSidecarList::from( + blobs + .into_iter() + .enumerate() + .map(|(blob_index, blob)| { + let kzg_commitment = expected_kzg_commitments + .get(blob_index) + .expect("KZG commitment should exist for blob"); + + let kzg_proof = kzg_proofs + .get(blob_index) + .expect("KZG proof should exist for blob"); + + Ok(Arc::new(BlobSidecar { + block_root: beacon_block_root, + index: blob_index as u64, + slot, + block_parent_root: block.parent_root(), + proposer_index: block.proposer_index(), + blob, + kzg_commitment: *kzg_commitment, + kzg_proof: *kzg_proof, + })) + }) + .collect::, BlockProductionError>>()?, + ); + + self.proposal_blob_cache + .put(beacon_block_root, blob_sidecars); + + Ok(()) + } + + fn build_and_cache_blinded_blob_sidecars>( + &self, + block: &BeaconBlock, + blob_roots: BlobRoots, + expected_kzg_commitments: &KzgCommitments, + kzg_proofs: Vec, + ) -> Result<(), BlockProductionError> { + let beacon_block_root = block.canonical_root(); + let slot = block.slot(); + + let blob_sidecars = SidecarList::::from( + blob_roots + .into_iter() + .enumerate() + .map(|(blob_index, blob_root)| { + let kzg_commitment = expected_kzg_commitments + .get(blob_index) + .expect("KZG commitment should exist for blob"); + + let kzg_proof = + kzg_proofs + .get(blob_index) + .ok_or(BlockProductionError::MissingKzgProof(format!( + "Missing KZG proof for slot {} blob index: {}", + slot, blob_index + )))?; + + Ok(Arc::new(BlindedBlobSidecar { + block_root: beacon_block_root, + index: blob_index as u64, + slot, + block_parent_root: block.parent_root(), + proposer_index: block.proposer_index(), + blob_root, + kzg_commitment: *kzg_commitment, + kzg_proof: *kzg_proof, + })) + }) + .collect::, BlockProductionError>>()?, + ); + + self.proposal_blinded_blob_cache + .put(beacon_block_root, blob_sidecars); + + Ok(()) + } + + fn merge_block_contents_into_beacon_block>( + partial_beacon_block: PartialBeaconBlock, + block_contents: Option>, + ) -> Result< + ( + BeaconState, + BeaconBlock, + Option>, + Option>, + ), + BlockProductionError, + > { + let PartialBeaconBlock { + state, + slot, + proposer_index, + parent_root, + randao_reveal, + eth1_data, + graffiti, + proposer_slashings, + attester_slashings, + attestations, + deposits, + voluntary_exits, + sync_aggregate, + // We don't need the prepare payload handle since the `execution_payload` is passed into + // this function. We can assume that the handle has already been consumed in order to + // produce said `execution_payload`. + prepare_payload_handle: _, + bls_to_execution_changes, + } = partial_beacon_block; + + let result = match &state { + BeaconState::Base(_) => ( + state, + BeaconBlock::Base(BeaconBlockBase { + slot, + proposer_index, + parent_root, + state_root: Hash256::zero(), + body: BeaconBlockBodyBase { + randao_reveal, + eth1_data, + graffiti, + proposer_slashings: proposer_slashings.into(), + attester_slashings: attester_slashings.into(), + attestations: attestations.into(), + deposits: deposits.into(), + voluntary_exits: voluntary_exits.into(), + _phantom: PhantomData, + }, + }), + None, + None, + ), + BeaconState::Altair(_) => ( + state, + BeaconBlock::Altair(BeaconBlockAltair { + slot, + proposer_index, + parent_root, + state_root: Hash256::zero(), + body: BeaconBlockBodyAltair { + randao_reveal, + eth1_data, + graffiti, + proposer_slashings: proposer_slashings.into(), + attester_slashings: attester_slashings.into(), + attestations: attestations.into(), + deposits: deposits.into(), + voluntary_exits: voluntary_exits.into(), + sync_aggregate: sync_aggregate + .ok_or(BlockProductionError::MissingSyncAggregate)?, + _phantom: PhantomData, + }, + }), + None, + None, + ), + BeaconState::Merge(_) => { + let (payload, _, _, _) = block_contents + .ok_or(BlockProductionError::MissingExecutionPayload)? + .deconstruct(); + ( + state, + BeaconBlock::Merge(BeaconBlockMerge { + slot, + proposer_index, + parent_root, + state_root: Hash256::zero(), + body: BeaconBlockBodyMerge { + randao_reveal, + eth1_data, + graffiti, + proposer_slashings: proposer_slashings.into(), + attester_slashings: attester_slashings.into(), + attestations: attestations.into(), + deposits: deposits.into(), + voluntary_exits: voluntary_exits.into(), + sync_aggregate: sync_aggregate + .ok_or(BlockProductionError::MissingSyncAggregate)?, + execution_payload: payload + .try_into() + .map_err(|_| BlockProductionError::InvalidPayloadFork)?, + }, + }), + None, + None, + ) + } + BeaconState::Capella(_) => { + let (payload, _, _, _) = block_contents + .ok_or(BlockProductionError::MissingExecutionPayload)? + .deconstruct(); + ( + state, + BeaconBlock::Capella(BeaconBlockCapella { + slot, + proposer_index, + parent_root, + state_root: Hash256::zero(), + body: BeaconBlockBodyCapella { + randao_reveal, + eth1_data, + graffiti, + proposer_slashings: proposer_slashings.into(), + attester_slashings: attester_slashings.into(), + attestations: attestations.into(), + deposits: deposits.into(), + voluntary_exits: voluntary_exits.into(), + sync_aggregate: sync_aggregate + .ok_or(BlockProductionError::MissingSyncAggregate)?, + execution_payload: payload + .try_into() + .map_err(|_| BlockProductionError::InvalidPayloadFork)?, + bls_to_execution_changes: bls_to_execution_changes.into(), + }, + }), + None, + None, + ) + } + BeaconState::Deneb(_) => { + let (payload, kzg_commitments, blobs_or_blob_roots, proofs) = block_contents + .ok_or(BlockProductionError::MissingExecutionPayload)? + .deconstruct(); + ( + state, + BeaconBlock::Deneb(BeaconBlockDeneb { + slot, + proposer_index, + parent_root, + state_root: Hash256::zero(), + body: BeaconBlockBodyDeneb { + randao_reveal, + eth1_data, + graffiti, + proposer_slashings: proposer_slashings.into(), + attester_slashings: attester_slashings.into(), + attestations: attestations.into(), + deposits: deposits.into(), + voluntary_exits: voluntary_exits.into(), + sync_aggregate: sync_aggregate + .ok_or(BlockProductionError::MissingSyncAggregate)?, + execution_payload: payload + .try_into() + .map_err(|_| BlockProductionError::InvalidPayloadFork)?, + bls_to_execution_changes: bls_to_execution_changes.into(), + blob_kzg_commitments: kzg_commitments + .ok_or(BlockProductionError::InvalidPayloadFork)?, + }, + }), + blobs_or_blob_roots, + proofs, + ) + } + }; + + Ok(result) + } } impl Drop for BeaconChain { diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs index e4c4ff2517c..4b24fe32d8d 100644 --- a/beacon_node/beacon_chain/src/errors.rs +++ b/beacon_node/beacon_chain/src/errors.rs @@ -280,6 +280,7 @@ pub enum BlockProductionError { MissingSyncAggregate, MissingExecutionPayload, MissingKzgCommitment(String), + MissingKzgProof(String), TokioJoin(tokio::task::JoinError), BeaconChain(BeaconChainError), InvalidPayloadFork, diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index fcedecdf223..6616e8996df 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -43,7 +43,9 @@ use tokio_stream::wrappers::WatchStream; use tree_hash::TreeHash; use types::beacon_block_body::KzgCommitments; use types::blob_sidecar::Blobs; +use types::builder_bid::BuilderBid; use types::consts::deneb::BLOB_TX_TYPE; +use types::deneb_types::BlobsOrBlobRoots; use types::transaction::{AccessTuple, BlobTransaction, EcdsaSignature, SignedBlobTransaction}; use types::{AbstractExecPayload, BeaconStateError, ExecPayload, VersionedHash}; use types::{ @@ -95,6 +97,31 @@ pub enum ProvenancedPayload

{ Builder(P), } +impl> From> + for ProvenancedPayload> +{ + fn from(value: BuilderBid) -> Self { + let block_proposal_contents = match value { + BuilderBid::Merge(builder_bid) => BlockProposalContents::Payload { + payload: builder_bid.header, + block_value: builder_bid.value, + }, + BuilderBid::Capella(builder_bid) => BlockProposalContents::Payload { + payload: builder_bid.header, + block_value: builder_bid.value, + }, + BuilderBid::Deneb(builder_bid) => BlockProposalContents::PayloadAndBlindedBlobs { + payload: builder_bid.header, + block_value: builder_bid.value, + kzg_commitments: builder_bid.blinded_blobs_bundle.commitments, + blob_roots: builder_bid.blinded_blobs_bundle.blob_roots, + proofs: builder_bid.blinded_blobs_bundle.proofs, + }, + }; + ProvenancedPayload::Builder(block_proposal_contents) + } +} + #[derive(Debug)] pub enum Error { NoEngine, @@ -143,6 +170,13 @@ pub enum BlockProposalContents> { blobs: Blobs, proofs: KzgProofs, }, + PayloadAndBlindedBlobs { + payload: Payload, + block_value: Uint256, + kzg_commitments: KzgCommitments, + blob_roots: VariableList, + proofs: KzgProofs, + }, } impl> From> @@ -173,7 +207,7 @@ impl> BlockProposalContents ( Payload, Option>, - Option>, + Option>, Option>, ) { match self { @@ -187,53 +221,46 @@ impl> BlockProposalContents (payload, Some(kzg_commitments), Some(blobs), Some(proofs)), + } => ( + payload, + Some(kzg_commitments), + Some(BlobsOrBlobRoots::Blobs(blobs)), + Some(proofs), + ), + Self::PayloadAndBlindedBlobs { + payload, + block_value: _, + kzg_commitments, + blob_roots, + proofs, + } => ( + payload, + Some(kzg_commitments), + Some(BlobsOrBlobRoots::BlobRoots(blob_roots)), + Some(proofs), + ), } } pub fn payload(&self) -> &Payload { match self { - Self::Payload { - payload, - block_value: _, - } => payload, - Self::PayloadAndBlobs { - payload, - block_value: _, - kzg_commitments: _, - blobs: _, - proofs: _, - } => payload, + Self::Payload { payload, .. } => payload, + Self::PayloadAndBlobs { payload, .. } => payload, + Self::PayloadAndBlindedBlobs { payload, .. } => payload, } } pub fn to_payload(self) -> Payload { match self { - Self::Payload { - payload, - block_value: _, - } => payload, - Self::PayloadAndBlobs { - payload, - block_value: _, - kzg_commitments: _, - blobs: _, - proofs: _, - } => payload, + Self::Payload { payload, .. } => payload, + Self::PayloadAndBlobs { payload, .. } => payload, + Self::PayloadAndBlindedBlobs { payload, .. } => payload, } } pub fn block_value(&self) -> &Uint256 { match self { - Self::Payload { - payload: _, - block_value, - } => block_value, - Self::PayloadAndBlobs { - payload: _, - block_value, - kzg_commitments: _, - blobs: _, - proofs: _, - } => block_value, + Self::Payload { block_value, .. } => block_value, + Self::PayloadAndBlobs { block_value, .. } => block_value, + Self::PayloadAndBlindedBlobs { block_value, .. } => block_value, } } pub fn default_at_fork(fork_name: ForkName) -> Result { @@ -863,7 +890,7 @@ impl ExecutionLayer { self.log(), "Requested blinded execution payload"; "relay_fee_recipient" => match &relay_result { - Ok(Some(r)) => format!("{:?}", r.data.message.header.fee_recipient()), + Ok(Some(r)) => format!("{:?}", r.data.message.header().fee_recipient()), Ok(None) => "empty response".to_string(), Err(_) => "request failed".to_string(), }, @@ -899,7 +926,7 @@ impl ExecutionLayer { Ok(ProvenancedPayload::Local(local)) } (Ok(Some(relay)), Ok(local)) => { - let header = &relay.data.message.header; + let header = &relay.data.message.header(); info!( self.log(), @@ -909,10 +936,10 @@ impl ExecutionLayer { "parent_hash" => ?parent_hash, ); - let relay_value = relay.data.message.value; + let relay_value = relay.data.message.value(); let local_value = *local.block_value(); if !self.inner.always_prefer_builder_payload { - if local_value >= relay_value { + if local_value >= *relay_value { info!( self.log(), "Local block is more profitable than relay block"; @@ -939,12 +966,7 @@ impl ExecutionLayer { current_fork, spec, ) { - Ok(()) => Ok(ProvenancedPayload::Builder( - BlockProposalContents::Payload { - payload: relay.data.message.header, - block_value: relay.data.message.value, - }, - )), + Ok(()) => Ok(ProvenancedPayload::from(relay.data.message)), Err(reason) if !reason.payload_invalid() => { info!( self.log(), @@ -974,7 +996,7 @@ impl ExecutionLayer { } } (Ok(Some(relay)), Err(local_error)) => { - let header = &relay.data.message.header; + let header = &relay.data.message.header(); info!( self.log(), @@ -993,20 +1015,12 @@ impl ExecutionLayer { current_fork, spec, ) { - Ok(()) => Ok(ProvenancedPayload::Builder( - BlockProposalContents::Payload { - payload: relay.data.message.header, - block_value: relay.data.message.value, - }, - )), + Ok(()) => Ok(ProvenancedPayload::from(relay.data.message)), // If the payload is valid then use it. The local EE failed // to produce a payload so we have no alternative. - Err(e) if !e.payload_invalid() => Ok(ProvenancedPayload::Builder( - BlockProposalContents::Payload { - payload: relay.data.message.header, - block_value: relay.data.message.value, - }, - )), + Err(e) if !e.payload_invalid() => { + Ok(ProvenancedPayload::from(relay.data.message)) + } Err(reason) => { metrics::inc_counter_vec( &metrics::EXECUTION_LAYER_GET_PAYLOAD_BUILDER_REJECTIONS, @@ -2050,11 +2064,11 @@ fn verify_builder_bid>( spec: &ChainSpec, ) -> Result<(), Box> { let is_signature_valid = bid.data.verify_signature(spec); - let header = &bid.data.message.header; - let payload_value = bid.data.message.value; + let header = &bid.data.message.header(); + let payload_value = bid.data.message.value(); // Avoid logging values that we can't represent with our Prometheus library. - let payload_value_gwei = bid.data.message.value / 1_000_000_000; + let payload_value_gwei = bid.data.message.value() / 1_000_000_000; if payload_value_gwei <= Uint256::from(i64::max_value()) { metrics::set_gauge_vec( &metrics::EXECUTION_LAYER_PAYLOAD_BIDS, @@ -2070,10 +2084,10 @@ fn verify_builder_bid>( .map(|withdrawals| Withdrawals::::from(withdrawals).tree_hash_root()); let payload_withdrawals_root = header.withdrawals_root().ok(); - if payload_value < profit_threshold { + if *payload_value < profit_threshold { Err(Box::new(InvalidBuilderPayload::LowValue { profit_threshold, - payload_value, + payload_value: *payload_value, })) } else if header.parent_hash() != parent_hash { Err(Box::new(InvalidBuilderPayload::ParentHash { @@ -2103,7 +2117,7 @@ fn verify_builder_bid>( } else if !is_signature_valid { Err(Box::new(InvalidBuilderPayload::Signature { signature: bid.data.signature.clone(), - pubkey: bid.data.message.pubkey, + pubkey: *bid.data.message.pubkey(), })) } else if payload_withdrawals_root != expected_withdrawals_root { Err(Box::new(InvalidBuilderPayload::WithdrawalsRoot { diff --git a/consensus/types/src/blob_sidecar.rs b/consensus/types/src/blob_sidecar.rs index 2d25de4032b..f4b7582f566 100644 --- a/consensus/types/src/blob_sidecar.rs +++ b/consensus/types/src/blob_sidecar.rs @@ -78,6 +78,7 @@ pub type BlobSidecarList = VariableList>, :: pub type FixedBlobSidecarList = FixedVector>>, ::MaxBlobsPerBlock>; pub type Blobs = VariableList, ::MaxBlobsPerBlock>; +pub type BlobRoots = VariableList::MaxBlobsPerBlock>; impl SignedRoot for BlobSidecar {} diff --git a/consensus/types/src/builder_bid.rs b/consensus/types/src/builder_bid.rs index 8723c2afed9..652d275151a 100644 --- a/consensus/types/src/builder_bid.rs +++ b/consensus/types/src/builder_bid.rs @@ -1,21 +1,48 @@ -use crate::{ - AbstractExecPayload, ChainSpec, EthSpec, ExecPayload, ExecutionPayloadHeader, ForkName, - ForkVersionDeserialize, SignedRoot, Uint256, -}; -use bls::PublicKeyBytes; -use bls::Signature; +use std::marker::PhantomData; + use serde::{Deserialize as De, Deserializer, Serialize as Ser, Serializer}; use serde_derive::{Deserialize, Serialize}; -use serde_with::{serde_as, DeserializeAs, SerializeAs}; -use std::marker::PhantomData; +use serde_with::{As, DeserializeAs, SerializeAs}; +use ssz_types::VariableList; +use superstruct::superstruct; use tree_hash_derive::TreeHash; -#[serde_as] +use bls::PublicKeyBytes; +use bls::Signature; + +use crate::beacon_block_body::KzgCommitments; +use crate::{ + AbstractExecPayload, ChainSpec, EthSpec, ExecPayload, ExecutionPayloadHeader, ForkName, + ForkVersionDeserialize, Hash256, KzgProofs, SignedRoot, Uint256, +}; + #[derive(PartialEq, Debug, Serialize, Deserialize, TreeHash, Clone)] -#[serde(bound = "E: EthSpec, Payload: ExecPayload")] +#[serde(bound = "E: EthSpec")] +pub struct BlindedBlobsBundle { + pub commitments: KzgCommitments, + pub proofs: KzgProofs, + pub blob_roots: VariableList, +} + +#[superstruct( + variants(Merge, Capella, Deneb), + variant_attributes( + derive(PartialEq, Debug, Serialize, Deserialize, TreeHash, Clone), + serde(bound = "E: EthSpec, Payload: ExecPayload", deny_unknown_fields) + ) +)] +#[derive(PartialEq, Debug, Serialize, Deserialize, TreeHash, Clone)] +#[serde( + bound = "E: EthSpec, Payload: ExecPayload", + deny_unknown_fields, + untagged +)] +#[tree_hash(enum_behaviour = "transparent")] pub struct BuilderBid> { - #[serde_as(as = "BlindedPayloadAsHeader")] + #[serde(with = "As::>")] pub header: Payload, + #[superstruct(only(Deneb))] + pub blinded_blobs_bundle: BlindedBlobsBundle, #[serde(with = "serde_utils::quoted_u256")] pub value: Uint256, pub pubkey: PublicKeyBytes, @@ -37,32 +64,23 @@ pub struct SignedBuilderBid> { impl> ForkVersionDeserialize for BuilderBid { - fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( + fn deserialize_by_fork<'de, D: Deserializer<'de>>( value: serde_json::value::Value, fork_name: ForkName, ) -> Result { - let convert_err = |_| { - serde::de::Error::custom( - "BuilderBid failed to deserialize: unable to convert payload header to payload", - ) - }; + let convert_err = + |e| serde::de::Error::custom(format!("BuilderBid failed to deserialize: {:?}", e)); - #[derive(Deserialize)] - struct Helper { - header: serde_json::Value, - #[serde(with = "serde_utils::quoted_u256")] - value: Uint256, - pubkey: PublicKeyBytes, - } - let helper: Helper = serde_json::from_value(value).map_err(serde::de::Error::custom)?; - let payload_header = - ExecutionPayloadHeader::deserialize_by_fork::<'de, D>(helper.header, fork_name)?; - - Ok(Self { - header: Payload::try_from(payload_header).map_err(convert_err)?, - value: helper.value, - pubkey: helper.pubkey, - _phantom_data: Default::default(), + Ok(match fork_name { + ForkName::Merge => Self::Merge(serde_json::from_value(value).map_err(convert_err)?), + ForkName::Capella => Self::Capella(serde_json::from_value(value).map_err(convert_err)?), + ForkName::Deneb => Self::Deneb(serde_json::from_value(value).map_err(convert_err)?), + ForkName::Base | ForkName::Altair => { + return Err(serde::de::Error::custom(format!( + "BuilderBid failed to deserialize: unsupported fork '{}'", + fork_name + ))); + } }) } } @@ -70,7 +88,7 @@ impl> ForkVersionDeserialize impl> ForkVersionDeserialize for SignedBuilderBid { - fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( + fn deserialize_by_fork<'de, D: Deserializer<'de>>( value: serde_json::value::Value, fork_name: ForkName, ) -> Result { @@ -115,7 +133,7 @@ impl<'de, E: EthSpec, Payload: AbstractExecPayload> DeserializeAs<'de, Payloa impl> SignedBuilderBid { pub fn verify_signature(&self, spec: &ChainSpec) -> bool { self.message - .pubkey + .pubkey() .decompress() .map(|pubkey| { let domain = spec.get_builder_domain(); diff --git a/consensus/types/src/deneb_types.rs b/consensus/types/src/deneb_types.rs index 8902b85d9b2..31b29fd89aa 100644 --- a/consensus/types/src/deneb_types.rs +++ b/consensus/types/src/deneb_types.rs @@ -10,6 +10,7 @@ use tree_hash_derive::TreeHash; use kzg::{KzgCommitment, KzgProof}; use test_random_derive::TestRandom; +use crate::blob_sidecar::{BlobRoots, Blobs}; use crate::test_utils::TestRandom; use crate::{BeaconBlock, BlindedPayload, BlobSidecar, EthSpec, Hash256, SignedBeaconBlock, Slot}; @@ -63,3 +64,27 @@ impl AbstractSidecar for BlobSidecar {} impl AbstractSidecar for BlindedBlobSidecar {} pub type SidecarList = VariableList, ::MaxBlobsPerBlock>; + +#[derive(Debug, Serialize, Deserialize)] +#[serde(untagged)] +#[serde(bound = "T: EthSpec")] +pub enum BlobsOrBlobRoots { + Blobs(Blobs), + BlobRoots(BlobRoots), +} + +impl BlobsOrBlobRoots { + pub fn len(&self) -> usize { + match self { + Self::Blobs(blobs) => blobs.len(), + Self::BlobRoots(blob_roots) => blob_roots.len(), + } + } + + pub fn is_empty(&self) -> bool { + match self { + Self::Blobs(blobs) => blobs.is_empty(), + Self::BlobRoots(blob_roots) => blob_roots.is_empty(), + } + } +} From fdbba309dfe2868c6ba8a2b02f91b2fdea85cb3f Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Thu, 22 Jun 2023 17:25:21 +1000 Subject: [PATCH 04/24] Appease Clippy lint --- consensus/types/src/deneb_types.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/consensus/types/src/deneb_types.rs b/consensus/types/src/deneb_types.rs index 31b29fd89aa..40cb6af1727 100644 --- a/consensus/types/src/deneb_types.rs +++ b/consensus/types/src/deneb_types.rs @@ -15,6 +15,7 @@ use crate::test_utils::TestRandom; use crate::{BeaconBlock, BlindedPayload, BlobSidecar, EthSpec, Hash256, SignedBeaconBlock, Slot}; // TODO(jimmy): to be moved to respective type files. +// TODO(jimmy): cleanup unused types. #[derive( Debug, @@ -43,13 +44,16 @@ pub struct BlindedBlobSidecar { pub kzg_proof: KzgProof, } +#[allow(unused)] type BlindedBlobSidecarList = VariableList::MaxBlobsPerBlock>; +#[allow(unused)] struct BlindedBlockContents { blinded_block: BeaconBlock>, blinded_blob_sidecars: BlindedBlobSidecarList, } +#[allow(unused)] struct SignedBlindedBlockContents { signed_blinded_block: SignedBeaconBlock>, signed_blinded_blob_sidecars: BlindedBlobSidecarList, From 59fb1af612dd9aed1eb3d615e2463bd03092ef16 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Fri, 23 Jun 2023 00:10:07 +1000 Subject: [PATCH 05/24] Partial implementation of submit blinded block & blobs. Refactor existing `BlobSidecar` related types to support blinded blobs. --- beacon_node/beacon_chain/src/beacon_chain.rs | 31 +----- beacon_node/beacon_chain/src/blob_cache.rs | 3 +- .../beacon_chain/src/blob_verification.rs | 4 +- beacon_node/beacon_chain/src/test_utils.rs | 64 ++++++----- beacon_node/execution_layer/src/lib.rs | 6 +- .../http_api/src/build_block_contents.rs | 9 +- beacon_node/http_api/src/lib.rs | 17 +-- beacon_node/http_api/src/publish_blocks.rs | 21 ++-- common/eth2/src/lib.rs | 1 - common/eth2/src/types.rs | 72 +++++++++--- consensus/types/src/blob_sidecar.rs | 105 +++++++++++++++--- consensus/types/src/deneb_types.rs | 94 ---------------- consensus/types/src/lib.rs | 6 +- consensus/types/src/signed_blob.rs | 28 +++-- validator_client/src/block_service.rs | 5 +- validator_client/src/validator_store.rs | 17 +-- 16 files changed, 254 insertions(+), 229 deletions(-) delete mode 100644 consensus/types/src/deneb_types.rs diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 2891ceb76f9..8073ac4c6ba 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -117,9 +117,10 @@ use tokio_stream::Stream; use tree_hash::TreeHash; use types::beacon_block_body::KzgCommitments; use types::beacon_state::CloneConfig; -use types::blob_sidecar::{BlobRoots, BlobSidecarList, Blobs}; +use types::blob_sidecar::{ + BlindedBlobSidecar, BlindedBlobSidecarList, BlobRoots, BlobSidecarList, Blobs, BlobsOrBlobRoots, +}; use types::consts::deneb::MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS; -use types::deneb_types::{BlindedBlobSidecar, BlobsOrBlobRoots, SidecarList}; use types::*; pub type ForkChoiceError = fork_choice::Error; @@ -4820,29 +4821,6 @@ impl BeaconChain { Ok((block, state)) } - fn compute_blob_kzg_proofs( - kzg: &Arc, - blobs: &Blobs, - expected_kzg_commitments: &KzgCommitments, - slot: Slot, - ) -> Result, BlockProductionError> { - blobs - .iter() - .enumerate() - .map(|(blob_index, blob)| { - let kzg_commitment = expected_kzg_commitments.get(blob_index).ok_or( - BlockProductionError::MissingKzgCommitment(format!( - "Missing KZG commitment for slot {} blob index {}", - slot, blob_index - )), - )?; - - kzg_utils::compute_blob_kzg_proof::(kzg, blob, *kzg_commitment) - .map_err(BlockProductionError::KzgError) - }) - .collect::, BlockProductionError>>() - } - /// This method must be called whenever an execution engine indicates that a payload is /// invalid. /// @@ -6165,7 +6143,7 @@ impl BeaconChain { let beacon_block_root = block.canonical_root(); let slot = block.slot(); - let blob_sidecars = SidecarList::::from( + let blob_sidecars = BlindedBlobSidecarList::::from( blob_roots .into_iter() .enumerate() @@ -6202,6 +6180,7 @@ impl BeaconChain { Ok(()) } + #[allow(clippy::type_complexity)] //TODO(jimmy): fix type complexity fn merge_block_contents_into_beacon_block>( partial_beacon_block: PartialBeaconBlock, block_contents: Option>, diff --git a/beacon_node/beacon_chain/src/blob_cache.rs b/beacon_node/beacon_chain/src/blob_cache.rs index 4b072e599fa..307cb7476c0 100644 --- a/beacon_node/beacon_chain/src/blob_cache.rs +++ b/beacon_node/beacon_chain/src/blob_cache.rs @@ -1,7 +1,6 @@ use lru::LruCache; use parking_lot::Mutex; -use types::deneb_types::{AbstractSidecar, SidecarList}; -use types::{EthSpec, Hash256}; +use types::{AbstractSidecar, EthSpec, Hash256, SidecarList}; pub const DEFAULT_BLOB_CACHE_SIZE: usize = 10; diff --git a/beacon_node/beacon_chain/src/blob_verification.rs b/beacon_node/beacon_chain/src/blob_verification.rs index d9abd3343df..fb0fd453290 100644 --- a/beacon_node/beacon_chain/src/blob_verification.rs +++ b/beacon_node/beacon_chain/src/blob_verification.rs @@ -681,8 +681,8 @@ impl From> for BlockWrapper { } } -impl From>> for BlockWrapper { - fn from(value: BlockContentsTuple>) -> Self { +impl From, BlobSidecar>> for BlockWrapper { + fn from(value: BlockContentsTuple, BlobSidecar>) -> Self { match value.1 { Some(variable_list) => { let mut blobs = Vec::with_capacity(E::max_blobs_per_block()); diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index dbf88def398..ce7c80baf0b 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -774,7 +774,10 @@ where &self, mut state: BeaconState, slot: Slot, - ) -> (BlockContentsTuple>, BeaconState) { + ) -> ( + BlockContentsTuple, BlobSidecar>, + BeaconState, + ) { assert_ne!(slot, 0, "can't produce a block at slot 0"); assert!(slot >= state.slot()); @@ -814,35 +817,36 @@ where &self.spec, ); - let block_contents: BlockContentsTuple> = match &signed_block { - SignedBeaconBlock::Base(_) - | SignedBeaconBlock::Altair(_) - | SignedBeaconBlock::Merge(_) - | SignedBeaconBlock::Capella(_) => (signed_block, None), - SignedBeaconBlock::Deneb(_) => { - if let Some(blobs) = self - .chain - .proposal_blob_cache - .pop(&signed_block.canonical_root()) - { - let signed_blobs = Vec::from(blobs) - .into_iter() - .map(|blob| { - blob.sign( - &self.validator_keypairs[proposer_index].sk, - &state.fork(), - state.genesis_validators_root(), - &self.spec, - ) - }) - .collect::>() - .into(); - (signed_block, Some(signed_blobs)) - } else { - (signed_block, None) + let block_contents: BlockContentsTuple, BlobSidecar> = + match &signed_block { + SignedBeaconBlock::Base(_) + | SignedBeaconBlock::Altair(_) + | SignedBeaconBlock::Merge(_) + | SignedBeaconBlock::Capella(_) => (signed_block, None), + SignedBeaconBlock::Deneb(_) => { + if let Some(blobs) = self + .chain + .proposal_blob_cache + .pop(&signed_block.canonical_root()) + { + let signed_blobs = Vec::from(blobs) + .into_iter() + .map(|blob| { + blob.sign( + &self.validator_keypairs[proposer_index].sk, + &state.fork(), + state.genesis_validators_root(), + &self.spec, + ) + }) + .collect::>() + .into(); + (signed_block, Some(signed_blobs)) + } else { + (signed_block, None) + } } - } - }; + }; (block_contents, state) } @@ -1849,7 +1853,7 @@ where ) -> Result< ( SignedBeaconBlockHash, - BlockContentsTuple>, + BlockContentsTuple, BlobSidecar>, BeaconState, ), BlockError, diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 6616e8996df..6fa37823f99 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -42,14 +42,13 @@ use tokio::{ use tokio_stream::wrappers::WatchStream; use tree_hash::TreeHash; use types::beacon_block_body::KzgCommitments; -use types::blob_sidecar::Blobs; +use types::blob_sidecar::BlobsOrBlobRoots; use types::builder_bid::BuilderBid; use types::consts::deneb::BLOB_TX_TYPE; -use types::deneb_types::BlobsOrBlobRoots; use types::transaction::{AccessTuple, BlobTransaction, EcdsaSignature, SignedBlobTransaction}; use types::{AbstractExecPayload, BeaconStateError, ExecPayload, VersionedHash}; use types::{ - BlindedPayload, BlockType, ChainSpec, Epoch, ExecutionBlockHash, ExecutionPayload, + BlindedPayload, Blobs, BlockType, ChainSpec, Epoch, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadMerge, ForkName, }; use types::{KzgProofs, Withdrawals}; @@ -1187,6 +1186,7 @@ impl ExecutionLayer { } }; + // TODO(jimmy): cache blobs bundle let payload_response = async { debug!( self.log(), diff --git a/beacon_node/http_api/src/build_block_contents.rs b/beacon_node/http_api/src/build_block_contents.rs index 12199773f48..254b7356590 100644 --- a/beacon_node/http_api/src/build_block_contents.rs +++ b/beacon_node/http_api/src/build_block_contents.rs @@ -1,16 +1,17 @@ use beacon_chain::{BeaconChain, BeaconChainTypes, BlockProductionError}; use eth2::types::{BeaconBlockAndBlobSidecars, BlindedBeaconBlockAndBlobSidecars, BlockContents}; use std::sync::Arc; -use types::deneb_types::BlindedBlobSidecar; -use types::{BeaconBlock, BlindedPayload, BlobSidecar, ForkName, FullPayload}; +use types::{BeaconBlock, BlindedBlobSidecar, BlindedPayload, BlobSidecar, ForkName, FullPayload}; type Error = warp::reject::Rejection; +type FullBlockContents = BlockContents, BlobSidecar>; +type BlindedBlockContents = BlockContents, BlindedBlobSidecar>; pub fn build_block_contents( fork_name: ForkName, chain: Arc>, block: BeaconBlock>, -) -> Result, BlobSidecar>, Error> { +) -> Result, Error> { match fork_name { ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { Ok(BlockContents::Block(block)) @@ -37,7 +38,7 @@ pub fn build_blinded_block_contents( fork_name: ForkName, chain: Arc>, block: BeaconBlock>, -) -> Result, BlindedBlobSidecar>, Error> { +) -> Result, Error> { match fork_name { ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { Ok(BlockContents::Block(block)) diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 900d6c118c4..fc1123d69c6 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -59,11 +59,10 @@ use tokio::sync::mpsc::{Sender, UnboundedSender}; use tokio_stream::{wrappers::BroadcastStream, StreamExt}; use types::{ Attestation, AttestationData, AttestationShufflingId, AttesterSlashing, BeaconStateError, - BlindedPayload, CommitteeCache, ConfigAndPreset, Epoch, EthSpec, ForkName, FullPayload, - ProposerPreparationData, ProposerSlashing, RelativeEpoch, SignedAggregateAndProof, - SignedBeaconBlock, SignedBlsToExecutionChange, SignedContributionAndProof, - SignedValidatorRegistrationData, SignedVoluntaryExit, Slot, SyncCommitteeMessage, - SyncContributionData, + BlindedBlobSidecar, BlindedPayload, CommitteeCache, ConfigAndPreset, Epoch, EthSpec, ForkName, + FullPayload, ProposerPreparationData, ProposerSlashing, RelativeEpoch, SignedAggregateAndProof, + SignedBlsToExecutionChange, SignedContributionAndProof, SignedValidatorRegistrationData, + SignedVoluntaryExit, Slot, SyncCommitteeMessage, SyncContributionData, }; use version::{ add_consensus_version_header, execution_optimistic_finalized_fork_versioned_response, @@ -1243,11 +1242,15 @@ pub fn serve( .and(network_tx_filter.clone()) .and(log_filter.clone()) .and_then( - |block: SignedBeaconBlock>, + |block_contents: SignedBlockContents< + T::EthSpec, + BlindedPayload<_>, + BlindedBlobSidecar, + >, chain: Arc>, network_tx: UnboundedSender>, log: Logger| async move { - publish_blocks::publish_blinded_block(block, chain, &network_tx, log) + publish_blocks::publish_blinded_block(block_contents, chain, &network_tx, log) .await .map(|()| warp::reply().into_response()) }, diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index dcbe8097249..8fa992e00a7 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -16,8 +16,8 @@ use store::FixedVector; use tokio::sync::mpsc::UnboundedSender; use tree_hash::TreeHash; use types::{ - AbstractExecPayload, BeaconBlockRef, BlindedPayload, EthSpec, ExecPayload, ExecutionBlockHash, - FullPayload, Hash256, SignedBeaconBlock, + AbstractExecPayload, BeaconBlockRef, BlindedBlobSidecar, BlindedPayload, EthSpec, ExecPayload, + ExecutionBlockHash, FullPayload, Hash256, SignedBeaconBlock, }; use warp::Rejection; @@ -185,13 +185,14 @@ pub async fn publish_block( /// Handles a request from the HTTP API for blinded blocks. This converts blinded blocks into full /// blocks before publishing. pub async fn publish_blinded_block( - block: SignedBeaconBlock>, + block_contents: SignedBlockContents, BlindedBlobSidecar>, chain: Arc>, network_tx: &UnboundedSender>, log: Logger, ) -> Result<(), Rejection> { - let block_root = block.canonical_root(); - let full_block = reconstruct_block(chain.clone(), block_root, block, log.clone()).await?; + let block_root = block_contents.signed_block().canonical_root(); + let full_block = + reconstruct_block(chain.clone(), block_root, block_contents, log.clone()).await?; publish_block::(Some(block_root), full_block, chain, network_tx, log).await } @@ -201,9 +202,10 @@ pub async fn publish_blinded_block( async fn reconstruct_block( chain: Arc>, block_root: Hash256, - block: SignedBeaconBlock>, + block_contents: SignedBlockContents, BlindedBlobSidecar>, log: Logger, ) -> Result, Rejection> { + let block = block_contents.signed_block(); let full_payload_opt = if let Ok(payload_header) = block.message().body().execution_payload() { let el = chain.execution_layer.as_ref().ok_or_else(|| { warp_utils::reject::custom_server_error("Missing execution layer".to_string()) @@ -224,6 +226,7 @@ async fn reconstruct_block( .into(); ProvenancedPayload::Local(payload) // If we already have an execution payload with this transactions root cached, use it. + // TODO(jimmy) get cached blobs } else if let Some(cached_payload) = el.get_payload_by_root(&payload_header.tree_hash_root()) { @@ -245,8 +248,9 @@ async fn reconstruct_block( &log, ); + // TODO(jimmy): handle blinded blobs bundle response let full_payload = el - .propose_blinded_beacon_block(block_root, &block) + .propose_blinded_beacon_block(block_root, block) .await .map_err(|e| { warp_utils::reject::custom_server_error(format!( @@ -263,6 +267,9 @@ async fn reconstruct_block( None }; + // TODO(jimmy) convert full `SignedBlockSidecars` + let (block, _maybe_blobs) = block_contents.deconstruct(); + match full_payload_opt { // A block without a payload is pre-merge and we consider it locally // built. diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 05938f914d2..fb438dfa5bd 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -16,7 +16,6 @@ pub mod types; use self::mixin::{RequestAccept, ResponseOptional}; use self::types::{Error as ResponseError, *}; -use ::types::deneb_types::AbstractSidecar; use futures::Stream; use futures_util::StreamExt; use lighthouse_network::PeerId; diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 0079f68e09c..236c6c4fdd5 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -11,7 +11,6 @@ use std::convert::TryFrom; use std::fmt; use std::str::{from_utf8, FromStr}; use std::time::Duration; -use types::deneb_types::{AbstractSidecar, SidecarList}; pub use types::*; #[cfg(feature = "lighthouse")] @@ -1347,36 +1346,50 @@ impl, Sidecar: AbstractSidecar> } } -pub type BlockContentsTuple = ( +pub type BlockContentsTuple = ( SignedBeaconBlock, - Option>, + Option>, ); /// A wrapper over a [`SignedBeaconBlock`] or a [`SignedBeaconBlockAndBlobSidecars`]. #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(untagged)] #[serde(bound = "T: EthSpec")] -pub enum SignedBlockContents = FullPayload> { - BlockAndBlobSidecars(SignedBeaconBlockAndBlobSidecars), +pub enum SignedBlockContents< + T: EthSpec, + Payload: AbstractExecPayload = FullPayload, + Sidecar: AbstractSidecar = BlobSidecar, +> { + BlockAndBlobSidecars(SignedBeaconBlockAndBlobSidecars), + BlindedBlockAndBlobSidecars(SignedBlindedBeaconBlockAndBlobSidecars), Block(SignedBeaconBlock), } -impl> SignedBlockContents { +impl, Sidecar: AbstractSidecar> + SignedBlockContents +{ pub fn signed_block(&self) -> &SignedBeaconBlock { match self { SignedBlockContents::BlockAndBlobSidecars(block_and_sidecars) => { &block_and_sidecars.signed_block } + SignedBlockContents::BlindedBlockAndBlobSidecars(block_and_sidecars) => { + &block_and_sidecars.signed_blinded_block + } SignedBlockContents::Block(block) => block, } } - pub fn deconstruct(self) -> BlockContentsTuple { + pub fn deconstruct(self) -> BlockContentsTuple { match self { SignedBlockContents::BlockAndBlobSidecars(block_and_sidecars) => ( block_and_sidecars.signed_block, Some(block_and_sidecars.signed_blob_sidecars), ), + SignedBlockContents::BlindedBlockAndBlobSidecars(block_and_sidecars) => ( + block_and_sidecars.signed_blinded_block, + Some(block_and_sidecars.signed_blinded_blob_sidecars), + ), SignedBlockContents::Block(block) => (block, None), } } @@ -1397,27 +1410,39 @@ impl> From> From> - for SignedBlockContents +impl, Sidecar: AbstractSidecar> + From> for SignedBlockContents { - fn from(block_contents_tuple: BlockContentsTuple) -> Self { + fn from(block_contents_tuple: BlockContentsTuple) -> Self { match block_contents_tuple { (signed_block, None) => SignedBlockContents::Block(signed_block), - (signed_block, Some(signed_blob_sidecars)) => { - SignedBlockContents::BlockAndBlobSidecars(SignedBeaconBlockAndBlobSidecars { - signed_block, - signed_blob_sidecars, - }) - } + (signed_block, Some(signed_blob_sidecars)) => match Payload::block_type() { + BlockType::Blinded => SignedBlockContents::BlindedBlockAndBlobSidecars( + SignedBlindedBeaconBlockAndBlobSidecars { + signed_blinded_block: signed_block, + signed_blinded_blob_sidecars: signed_blob_sidecars, + }, + ), + BlockType::Full => { + SignedBlockContents::BlockAndBlobSidecars(SignedBeaconBlockAndBlobSidecars { + signed_block, + signed_blob_sidecars, + }) + } + }, } } } #[derive(Debug, Clone, Serialize, Deserialize, Encode)] #[serde(bound = "T: EthSpec")] -pub struct SignedBeaconBlockAndBlobSidecars> { +pub struct SignedBeaconBlockAndBlobSidecars< + T: EthSpec, + Payload: AbstractExecPayload, + Sidecar: AbstractSidecar, +> { pub signed_block: SignedBeaconBlock, - pub signed_blob_sidecars: SignedBlobSidecarList, + pub signed_blob_sidecars: SignedSidecarList, } #[derive(Debug, Clone, Serialize, Deserialize, Encode)] @@ -1454,6 +1479,17 @@ impl, Sidecar: AbstractSidecar> } } +#[derive(Debug, Clone, Serialize, Deserialize, Encode)] +#[serde(bound = "T: EthSpec")] +pub struct SignedBlindedBeaconBlockAndBlobSidecars< + T: EthSpec, + Payload: AbstractExecPayload, + Sidecar: AbstractSidecar, +> { + pub signed_blinded_block: SignedBeaconBlock, + pub signed_blinded_blob_sidecars: SignedSidecarList, +} + #[derive(Debug, Clone, Serialize, Deserialize, Encode)] #[serde(bound = "T: EthSpec")] pub struct BlindedBeaconBlockAndBlobSidecars< diff --git a/consensus/types/src/blob_sidecar.rs b/consensus/types/src/blob_sidecar.rs index f4b7582f566..ed9f10f208d 100644 --- a/consensus/types/src/blob_sidecar.rs +++ b/consensus/types/src/blob_sidecar.rs @@ -1,16 +1,35 @@ -use crate::test_utils::TestRandom; -use crate::{Blob, ChainSpec, Domain, EthSpec, Fork, Hash256, SignedBlobSidecar, SignedRoot, Slot}; -use bls::SecretKey; +use std::hash::Hash; +use std::marker::PhantomData; +use std::sync::Arc; + use derivative::Derivative; -use kzg::{KzgCommitment, KzgProof}; +use serde::de::DeserializeOwned; use serde_derive::{Deserialize, Serialize}; -use ssz::Encode; +use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; use ssz_types::{FixedVector, VariableList}; -use std::sync::Arc; -use test_random_derive::TestRandom; +use tree_hash::TreeHash; use tree_hash_derive::TreeHash; +use bls::SecretKey; +use kzg::{KzgCommitment, KzgProof}; +use test_random_derive::TestRandom; + +use crate::test_utils::TestRandom; +use crate::{Blob, ChainSpec, Domain, EthSpec, Fork, Hash256, SignedBlobSidecar, SignedRoot, Slot}; + +pub trait AbstractSidecar: + serde::Serialize + + DeserializeOwned + + Encode + + Decode + + Hash + + TreeHash + + TestRandom + + for<'a> arbitrary::Arbitrary<'a> +{ +} + /// Container of the data that identifies an individual blob. #[derive( Serialize, Deserialize, Encode, Decode, TreeHash, Copy, Clone, Debug, PartialEq, Eq, Hash, @@ -62,6 +81,8 @@ pub struct BlobSidecar { pub kzg_proof: KzgProof, } +impl AbstractSidecar for BlobSidecar {} + impl PartialOrd for BlobSidecar { fn partial_cmp(&self, other: &Self) -> Option { self.index.partial_cmp(&other.index) @@ -74,12 +95,6 @@ impl Ord for BlobSidecar { } } -pub type BlobSidecarList = VariableList>, ::MaxBlobsPerBlock>; -pub type FixedBlobSidecarList = - FixedVector>>, ::MaxBlobsPerBlock>; -pub type Blobs = VariableList, ::MaxBlobsPerBlock>; -pub type BlobRoots = VariableList::MaxBlobsPerBlock>; - impl SignedRoot for BlobSidecar {} impl BlobSidecar { @@ -121,6 +136,70 @@ impl BlobSidecar { SignedBlobSidecar { message: self, signature, + _phantom: PhantomData, + } + } +} + +#[derive( + Debug, + Clone, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + Default, + TestRandom, + Derivative, + arbitrary::Arbitrary, +)] +#[derivative(PartialEq, Eq, Hash)] +pub struct BlindedBlobSidecar { + pub block_root: Hash256, + #[serde(with = "serde_utils::quoted_u64")] + pub index: u64, + pub slot: Slot, + pub block_parent_root: Hash256, + #[serde(with = "serde_utils::quoted_u64")] + pub proposer_index: u64, + pub blob_root: Hash256, + pub kzg_commitment: KzgCommitment, + pub kzg_proof: KzgProof, +} + +impl AbstractSidecar for BlindedBlobSidecar {} + +pub type SidecarList = VariableList, ::MaxBlobsPerBlock>; +pub type BlobSidecarList = SidecarList>; +pub type BlindedBlobSidecarList = SidecarList; + +pub type FixedBlobSidecarList = + FixedVector>>, ::MaxBlobsPerBlock>; + +pub type Blobs = VariableList, ::MaxBlobsPerBlock>; +pub type BlobRoots = VariableList::MaxBlobsPerBlock>; + +#[derive(Debug, Serialize, Deserialize)] +#[serde(untagged)] +#[serde(bound = "T: EthSpec")] +pub enum BlobsOrBlobRoots { + Blobs(Blobs), + BlobRoots(BlobRoots), +} + +impl BlobsOrBlobRoots { + pub fn len(&self) -> usize { + match self { + Self::Blobs(blobs) => blobs.len(), + Self::BlobRoots(blob_roots) => blob_roots.len(), + } + } + + pub fn is_empty(&self) -> bool { + match self { + Self::Blobs(blobs) => blobs.is_empty(), + Self::BlobRoots(blob_roots) => blob_roots.is_empty(), } } } diff --git a/consensus/types/src/deneb_types.rs b/consensus/types/src/deneb_types.rs deleted file mode 100644 index 40cb6af1727..00000000000 --- a/consensus/types/src/deneb_types.rs +++ /dev/null @@ -1,94 +0,0 @@ -use derivative::Derivative; -use serde::de::DeserializeOwned; -use serde_derive::{Deserialize, Serialize}; -use ssz::{Decode, Encode}; -use ssz_derive::{Decode, Encode}; -use ssz_types::VariableList; -use std::sync::Arc; -use tree_hash_derive::TreeHash; - -use kzg::{KzgCommitment, KzgProof}; -use test_random_derive::TestRandom; - -use crate::blob_sidecar::{BlobRoots, Blobs}; -use crate::test_utils::TestRandom; -use crate::{BeaconBlock, BlindedPayload, BlobSidecar, EthSpec, Hash256, SignedBeaconBlock, Slot}; - -// TODO(jimmy): to be moved to respective type files. -// TODO(jimmy): cleanup unused types. - -#[derive( - Debug, - Clone, - Serialize, - Deserialize, - Encode, - Decode, - TreeHash, - Default, - TestRandom, - Derivative, - arbitrary::Arbitrary, -)] -#[derivative(PartialEq, Eq, Hash)] -pub struct BlindedBlobSidecar { - pub block_root: Hash256, - #[serde(with = "serde_utils::quoted_u64")] - pub index: u64, - pub slot: Slot, - pub block_parent_root: Hash256, - #[serde(with = "serde_utils::quoted_u64")] - pub proposer_index: u64, - pub blob_root: Hash256, - pub kzg_commitment: KzgCommitment, - pub kzg_proof: KzgProof, -} - -#[allow(unused)] -type BlindedBlobSidecarList = VariableList::MaxBlobsPerBlock>; - -#[allow(unused)] -struct BlindedBlockContents { - blinded_block: BeaconBlock>, - blinded_blob_sidecars: BlindedBlobSidecarList, -} - -#[allow(unused)] -struct SignedBlindedBlockContents { - signed_blinded_block: SignedBeaconBlock>, - signed_blinded_blob_sidecars: BlindedBlobSidecarList, -} - -pub trait AbstractSidecar: - serde::Serialize + DeserializeOwned + Encode + Decode -{ -} - -impl AbstractSidecar for BlobSidecar {} -impl AbstractSidecar for BlindedBlobSidecar {} - -pub type SidecarList = VariableList, ::MaxBlobsPerBlock>; - -#[derive(Debug, Serialize, Deserialize)] -#[serde(untagged)] -#[serde(bound = "T: EthSpec")] -pub enum BlobsOrBlobRoots { - Blobs(Blobs), - BlobRoots(BlobRoots), -} - -impl BlobsOrBlobRoots { - pub fn len(&self) -> usize { - match self { - Self::Blobs(blobs) => blobs.len(), - Self::BlobRoots(blob_roots) => blob_roots.len(), - } - } - - pub fn is_empty(&self) -> bool { - match self { - Self::Blobs(blobs) => blobs.is_empty(), - Self::BlobRoots(blob_roots) => blob_roots.is_empty(), - } - } -} diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 288d8a79c96..d547cefbfad 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -98,7 +98,6 @@ pub mod slot_data; pub mod sqlite; pub mod blob_sidecar; -pub mod deneb_types; pub mod signed_blob; pub mod transaction; @@ -120,7 +119,10 @@ pub use crate::beacon_block_body::{ pub use crate::beacon_block_header::BeaconBlockHeader; pub use crate::beacon_committee::{BeaconCommittee, OwnedBeaconCommittee}; pub use crate::beacon_state::{BeaconTreeHashCache, Error as BeaconStateError, *}; -pub use crate::blob_sidecar::{BlobSidecar, BlobSidecarList}; +pub use crate::blob_sidecar::{ + AbstractSidecar, BlindedBlobSidecar, BlindedBlobSidecarList, BlobRoots, BlobSidecar, + BlobSidecarList, Blobs, SidecarList, +}; pub use crate::bls_to_execution_change::BlsToExecutionChange; pub use crate::chain_spec::{ChainSpec, Config, Domain}; pub use crate::checkpoint::Checkpoint; diff --git a/consensus/types/src/signed_blob.rs b/consensus/types/src/signed_blob.rs index aaab02ca783..16252bd025a 100644 --- a/consensus/types/src/signed_blob.rs +++ b/consensus/types/src/signed_blob.rs @@ -1,12 +1,13 @@ use crate::{ - test_utils::TestRandom, BlobSidecar, ChainSpec, Domain, EthSpec, Fork, Hash256, Signature, - SignedRoot, SigningData, + test_utils::TestRandom, AbstractSidecar, BlobSidecar, ChainSpec, Domain, EthSpec, Fork, + Hash256, Signature, SignedRoot, SigningData, }; use bls::PublicKey; use derivative::Derivative; use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::VariableList; +use std::marker::PhantomData; use std::sync::Arc; use test_random_derive::TestRandom; use tree_hash::TreeHash; @@ -25,17 +26,26 @@ use tree_hash_derive::TreeHash; Derivative, arbitrary::Arbitrary, )] -#[serde(bound = "T: EthSpec")] -#[arbitrary(bound = "T: EthSpec")] -#[derivative(Hash(bound = "T: EthSpec"))] -pub struct SignedBlobSidecar { - pub message: Arc>, +#[serde(bound = "T: EthSpec, Sidecar: AbstractSidecar")] +#[arbitrary(bound = "T: EthSpec, Sidecar: AbstractSidecar")] +#[derivative(Hash(bound = "T: EthSpec, Sidecar: AbstractSidecar"))] +pub struct SignedSidecar> { + pub message: Arc, pub signature: Signature, + #[ssz(skip_serializing, skip_deserializing)] + #[tree_hash(skip_hashing)] + #[serde(skip)] + #[arbitrary(default)] + pub _phantom: PhantomData, } -pub type SignedBlobSidecarList = - VariableList, ::MaxBlobsPerBlock>; +/// List of Signed Sidecars that implements `AbstractSidecar`. +pub type SignedSidecarList = + VariableList, ::MaxBlobsPerBlock>; +pub type SignedBlobSidecar = SignedSidecar>; + +// TODO(jimmy): impl on SignedSidecar instead? impl SignedBlobSidecar { /// Verify `self.signature`. /// diff --git a/validator_client/src/block_service.rs b/validator_client/src/block_service.rs index 13632c17c3f..bbf44f85c2b 100644 --- a/validator_client/src/block_service.rs +++ b/validator_client/src/block_service.rs @@ -19,10 +19,9 @@ use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc; use tokio::time::sleep; -use types::deneb_types::AbstractSidecar; use types::{ - AbstractExecPayload, BlindedPayload, BlockType, EthSpec, FullPayload, Graffiti, PublicKeyBytes, - Slot, + AbstractExecPayload, AbstractSidecar, BlindedPayload, BlockType, EthSpec, FullPayload, + Graffiti, PublicKeyBytes, Slot, }; #[derive(Debug)] diff --git a/validator_client/src/validator_store.rs b/validator_client/src/validator_store.rs index 811d7bb16b1..9033267a485 100644 --- a/validator_client/src/validator_store.rs +++ b/validator_client/src/validator_store.rs @@ -20,13 +20,13 @@ use std::sync::Arc; use task_executor::TaskExecutor; use types::{ attestation::Error as AttestationError, graffiti::GraffitiString, AbstractExecPayload, Address, - AggregateAndProof, Attestation, BeaconBlock, BlindedPayload, BlobSidecarList, ChainSpec, - ContributionAndProof, Domain, Epoch, EthSpec, Fork, Graffiti, Hash256, Keypair, PublicKeyBytes, - SelectionProof, Signature, SignedAggregateAndProof, SignedBeaconBlock, SignedBlobSidecar, - SignedBlobSidecarList, SignedContributionAndProof, SignedRoot, SignedValidatorRegistrationData, - SignedVoluntaryExit, Slot, SyncAggregatorSelectionData, SyncCommitteeContribution, - SyncCommitteeMessage, SyncSelectionProof, SyncSubnetId, ValidatorRegistrationData, - VoluntaryExit, + AggregateAndProof, Attestation, BeaconBlock, BlindedPayload, BlobSidecar, BlobSidecarList, + ChainSpec, ContributionAndProof, Domain, Epoch, EthSpec, Fork, Graffiti, Hash256, Keypair, + PublicKeyBytes, SelectionProof, Signature, SignedAggregateAndProof, SignedBeaconBlock, + SignedBlobSidecar, SignedContributionAndProof, SignedRoot, SignedSidecarList, + SignedValidatorRegistrationData, SignedVoluntaryExit, Slot, SyncAggregatorSelectionData, + SyncCommitteeContribution, SyncCommitteeMessage, SyncSelectionProof, SyncSubnetId, + ValidatorRegistrationData, VoluntaryExit, }; use validator_dir::ValidatorDir; @@ -546,7 +546,7 @@ impl ValidatorStore { &self, validator_pubkey: PublicKeyBytes, blob_sidecars: BlobSidecarList, - ) -> Result, Error> { + ) -> Result>, Error> { let mut signed_blob_sidecars = Vec::new(); for blob_sidecar in blob_sidecars.into_iter() { @@ -569,6 +569,7 @@ impl ValidatorStore { signed_blob_sidecars.push(SignedBlobSidecar { message: blob_sidecar, signature, + _phantom: PhantomData, }); } From 361664fee322d562a07a07cae67f02ef7be5282d Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Sat, 1 Jul 2023 01:38:42 +1000 Subject: [PATCH 06/24] Add associated types for BlockProposal --- .../beacon_chain/src/blob_verification.rs | 10 +- beacon_node/beacon_chain/src/test_utils.rs | 63 ++++---- .../http_api/src/build_block_contents.rs | 11 +- beacon_node/http_api/src/lib.rs | 16 +- beacon_node/http_api/src/publish_blocks.rs | 14 +- common/eth2/src/lib.rs | 36 ++--- common/eth2/src/types.rs | 144 +++++++++--------- consensus/types/src/blob_sidecar.rs | 18 ++- consensus/types/src/signed_blob.rs | 10 ++ validator_client/src/block_service.rs | 44 +++--- validator_client/src/signing_method.rs | 17 ++- validator_client/src/validator_store.rs | 62 ++++---- 12 files changed, 219 insertions(+), 226 deletions(-) diff --git a/beacon_node/beacon_chain/src/blob_verification.rs b/beacon_node/beacon_chain/src/blob_verification.rs index fb0fd453290..b2af386195d 100644 --- a/beacon_node/beacon_chain/src/blob_verification.rs +++ b/beacon_node/beacon_chain/src/blob_verification.rs @@ -12,7 +12,7 @@ use crate::data_availability_checker::{ }; use crate::kzg_utils::{validate_blob, validate_blobs}; use crate::BeaconChainError; -use eth2::types::BlockContentsTuple; +use eth2::types::{FullBlockProposal, SignedBlockContentsTuple}; use kzg::Kzg; use slog::{debug, warn}; use ssz_derive::{Decode, Encode}; @@ -21,8 +21,8 @@ use std::borrow::Cow; use types::blob_sidecar::{BlobIdentifier, FixedBlobSidecarList}; use types::{ BeaconBlockRef, BeaconState, BeaconStateError, BlobSidecar, ChainSpec, CloneConfig, Epoch, - EthSpec, FullPayload, Hash256, KzgCommitment, RelativeEpoch, SignedBeaconBlock, - SignedBeaconBlockHeader, SignedBlobSidecar, Slot, + EthSpec, Hash256, KzgCommitment, RelativeEpoch, SignedBeaconBlock, SignedBeaconBlockHeader, + SignedBlobSidecar, Slot, }; #[derive(Debug)] @@ -681,8 +681,8 @@ impl From> for BlockWrapper { } } -impl From, BlobSidecar>> for BlockWrapper { - fn from(value: BlockContentsTuple, BlobSidecar>) -> Self { +impl From> for BlockWrapper { + fn from(value: SignedBlockContentsTuple) -> Self { match value.1 { Some(variable_list) => { let mut blobs = Vec::with_capacity(E::max_blobs_per_block()); diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index ce7c80baf0b..87109f054f4 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -15,7 +15,7 @@ use crate::{ StateSkipConfig, }; use bls::get_withdrawal_credentials; -use eth2::types::BlockContentsTuple; +use eth2::types::{FullBlockProposal, SignedBlockContentsTuple}; use execution_layer::{ auth::JwtKey, test_utils::{ @@ -775,7 +775,7 @@ where mut state: BeaconState, slot: Slot, ) -> ( - BlockContentsTuple, BlobSidecar>, + SignedBlockContentsTuple, BeaconState, ) { assert_ne!(slot, 0, "can't produce a block at slot 0"); @@ -817,36 +817,35 @@ where &self.spec, ); - let block_contents: BlockContentsTuple, BlobSidecar> = - match &signed_block { - SignedBeaconBlock::Base(_) - | SignedBeaconBlock::Altair(_) - | SignedBeaconBlock::Merge(_) - | SignedBeaconBlock::Capella(_) => (signed_block, None), - SignedBeaconBlock::Deneb(_) => { - if let Some(blobs) = self - .chain - .proposal_blob_cache - .pop(&signed_block.canonical_root()) - { - let signed_blobs = Vec::from(blobs) - .into_iter() - .map(|blob| { - blob.sign( - &self.validator_keypairs[proposer_index].sk, - &state.fork(), - state.genesis_validators_root(), - &self.spec, - ) - }) - .collect::>() - .into(); - (signed_block, Some(signed_blobs)) - } else { - (signed_block, None) - } + let block_contents: SignedBlockContentsTuple = match &signed_block { + SignedBeaconBlock::Base(_) + | SignedBeaconBlock::Altair(_) + | SignedBeaconBlock::Merge(_) + | SignedBeaconBlock::Capella(_) => (signed_block, None), + SignedBeaconBlock::Deneb(_) => { + if let Some(blobs) = self + .chain + .proposal_blob_cache + .pop(&signed_block.canonical_root()) + { + let signed_blobs = Vec::from(blobs) + .into_iter() + .map(|blob| { + blob.sign( + &self.validator_keypairs[proposer_index].sk, + &state.fork(), + state.genesis_validators_root(), + &self.spec, + ) + }) + .collect::>() + .into(); + (signed_block, Some(signed_blobs)) + } else { + (signed_block, None) } - }; + } + }; (block_contents, state) } @@ -1853,7 +1852,7 @@ where ) -> Result< ( SignedBeaconBlockHash, - BlockContentsTuple, BlobSidecar>, + SignedBlockContentsTuple, BeaconState, ), BlockError, diff --git a/beacon_node/http_api/src/build_block_contents.rs b/beacon_node/http_api/src/build_block_contents.rs index 254b7356590..176c1b3515b 100644 --- a/beacon_node/http_api/src/build_block_contents.rs +++ b/beacon_node/http_api/src/build_block_contents.rs @@ -1,11 +1,14 @@ use beacon_chain::{BeaconChain, BeaconChainTypes, BlockProductionError}; -use eth2::types::{BeaconBlockAndBlobSidecars, BlindedBeaconBlockAndBlobSidecars, BlockContents}; +use eth2::types::{ + BeaconBlockAndBlobSidecars, BlindedBeaconBlockAndBlobSidecars, BlindedBlockProposal, + BlockContents, FullBlockProposal, +}; use std::sync::Arc; -use types::{BeaconBlock, BlindedBlobSidecar, BlindedPayload, BlobSidecar, ForkName, FullPayload}; +use types::{BeaconBlock, BlindedPayload, ForkName, FullPayload}; type Error = warp::reject::Rejection; -type FullBlockContents = BlockContents, BlobSidecar>; -type BlindedBlockContents = BlockContents, BlindedBlobSidecar>; +type FullBlockContents = BlockContents; +type BlindedBlockContents = BlockContents; pub fn build_block_contents( fork_name: ForkName, diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index fc1123d69c6..43373e7bc6c 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -32,8 +32,8 @@ use beacon_chain::{ pub use block_id::BlockId; use directory::DEFAULT_ROOT_DIR; use eth2::types::{ - self as api_types, EndpointVersion, ForkChoice, ForkChoiceNode, SignedBlockContents, - SkipRandaoVerification, ValidatorId, ValidatorStatus, + self as api_types, BlindedBlockProposal, EndpointVersion, ForkChoice, ForkChoiceNode, + FullBlockProposal, SignedBlockContents, SkipRandaoVerification, ValidatorId, ValidatorStatus, }; use lighthouse_network::{types::SyncState, EnrExt, NetworkGlobals, PeerId, PubsubMessage}; use lighthouse_version::version_with_platform; @@ -59,8 +59,8 @@ use tokio::sync::mpsc::{Sender, UnboundedSender}; use tokio_stream::{wrappers::BroadcastStream, StreamExt}; use types::{ Attestation, AttestationData, AttestationShufflingId, AttesterSlashing, BeaconStateError, - BlindedBlobSidecar, BlindedPayload, CommitteeCache, ConfigAndPreset, Epoch, EthSpec, ForkName, - FullPayload, ProposerPreparationData, ProposerSlashing, RelativeEpoch, SignedAggregateAndProof, + BlindedPayload, CommitteeCache, ConfigAndPreset, Epoch, EthSpec, ForkName, FullPayload, + ProposerPreparationData, ProposerSlashing, RelativeEpoch, SignedAggregateAndProof, SignedBlsToExecutionChange, SignedContributionAndProof, SignedValidatorRegistrationData, SignedVoluntaryExit, Slot, SyncCommitteeMessage, SyncContributionData, }; @@ -1216,7 +1216,7 @@ pub fn serve( .and(network_tx_filter.clone()) .and(log_filter.clone()) .and_then( - |block_contents: SignedBlockContents, + |block_contents: SignedBlockContents, chain: Arc>, network_tx: UnboundedSender>, log: Logger| async move { @@ -1242,11 +1242,7 @@ pub fn serve( .and(network_tx_filter.clone()) .and(log_filter.clone()) .and_then( - |block_contents: SignedBlockContents< - T::EthSpec, - BlindedPayload<_>, - BlindedBlobSidecar, - >, + |block_contents: SignedBlockContents, chain: Arc>, network_tx: UnboundedSender>, log: Logger| async move { diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 8fa992e00a7..1f3066e7fa4 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -4,7 +4,7 @@ use beacon_chain::blob_verification::{AsBlock, BlockWrapper}; use beacon_chain::validator_monitor::{get_block_delay_ms, timestamp_now}; use beacon_chain::{AvailabilityProcessingStatus, NotifyExecutionLayer}; use beacon_chain::{BeaconChain, BeaconChainTypes, BlockError, CountUnrealized}; -use eth2::types::SignedBlockContents; +use eth2::types::{BlindedBlockProposal, FullBlockProposal, SignedBlockContents}; use execution_layer::ProvenancedPayload; use lighthouse_network::PubsubMessage; use network::NetworkMessage; @@ -16,17 +16,17 @@ use store::FixedVector; use tokio::sync::mpsc::UnboundedSender; use tree_hash::TreeHash; use types::{ - AbstractExecPayload, BeaconBlockRef, BlindedBlobSidecar, BlindedPayload, EthSpec, ExecPayload, - ExecutionBlockHash, FullPayload, Hash256, SignedBeaconBlock, + AbstractExecPayload, BeaconBlockRef, EthSpec, ExecPayload, ExecutionBlockHash, FullPayload, + Hash256, SignedBeaconBlock, }; use warp::Rejection; pub enum ProvenancedBlock { /// The payload was built using a local EE. - Local(SignedBlockContents>), + Local(SignedBlockContents), /// The payload was build using a remote builder (e.g., via a mev-boost /// compatible relay). - Builder(SignedBlockContents>), + Builder(SignedBlockContents), } /// Handles a request from the HTTP API for full blocks. @@ -185,7 +185,7 @@ pub async fn publish_block( /// Handles a request from the HTTP API for blinded blocks. This converts blinded blocks into full /// blocks before publishing. pub async fn publish_blinded_block( - block_contents: SignedBlockContents, BlindedBlobSidecar>, + block_contents: SignedBlockContents, chain: Arc>, network_tx: &UnboundedSender>, log: Logger, @@ -202,7 +202,7 @@ pub async fn publish_blinded_block( async fn reconstruct_block( chain: Arc>, block_root: Hash256, - block_contents: SignedBlockContents, BlindedBlobSidecar>, + block_contents: SignedBlockContents, log: Logger, ) -> Result, Rejection> { let block = block_contents.signed_block(); diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index fb438dfa5bd..34f16425fa4 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -612,9 +612,9 @@ impl BeaconNodeHttpClient { /// `POST beacon/blocks` /// /// Returns `Ok(None)` on a 404 error. - pub async fn post_beacon_blocks>( + pub async fn post_beacon_blocks>( &self, - block_contents: &SignedBlockContents, + block_contents: &SignedBlockContents, ) -> Result<(), Error> { let mut path = self.eth_path(V1)?; @@ -1384,32 +1384,24 @@ impl BeaconNodeHttpClient { } /// `GET v2/validator/blocks/{slot}` - pub async fn get_validator_blocks< - T: EthSpec, - Payload: AbstractExecPayload, - Sidecar: AbstractSidecar, - >( + pub async fn get_validator_blocks>( &self, slot: Slot, randao_reveal: &SignatureBytes, graffiti: Option<&Graffiti>, - ) -> Result>, Error> { + ) -> Result>, Error> { self.get_validator_blocks_modular(slot, randao_reveal, graffiti, SkipRandaoVerification::No) .await } /// `GET v2/validator/blocks/{slot}` - pub async fn get_validator_blocks_modular< - T: EthSpec, - Payload: AbstractExecPayload, - Sidecar: AbstractSidecar, - >( + pub async fn get_validator_blocks_modular>( &self, slot: Slot, randao_reveal: &SignatureBytes, graffiti: Option<&Graffiti>, skip_randao_verification: SkipRandaoVerification, - ) -> Result>, Error> { + ) -> Result>, Error> { let mut path = self.eth_path(V2)?; path.path_segments_mut() @@ -1435,16 +1427,12 @@ impl BeaconNodeHttpClient { } /// `GET v2/validator/blinded_blocks/{slot}` - pub async fn get_validator_blinded_blocks< - T: EthSpec, - Payload: AbstractExecPayload, - Sidecar: AbstractSidecar, - >( + pub async fn get_validator_blinded_blocks>( &self, slot: Slot, randao_reveal: &SignatureBytes, graffiti: Option<&Graffiti>, - ) -> Result>, Error> { + ) -> Result>, Error> { self.get_validator_blinded_blocks_modular( slot, randao_reveal, @@ -1455,17 +1443,13 @@ impl BeaconNodeHttpClient { } /// `GET v1/validator/blinded_blocks/{slot}` - pub async fn get_validator_blinded_blocks_modular< - T: EthSpec, - Payload: AbstractExecPayload, - Sidecar: AbstractSidecar, - >( + pub async fn get_validator_blinded_blocks_modular>( &self, slot: Slot, randao_reveal: &SignatureBytes, graffiti: Option<&Graffiti>, skip_randao_verification: SkipRandaoVerification, - ) -> Result>, Error> { + ) -> Result>, Error> { let mut path = self.eth_path(V1)?; path.path_segments_mut() diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 236c6c4fdd5..6b17063c961 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -1273,20 +1273,40 @@ mod tests { } } +pub trait BlockProposal { + type Payload: AbstractExecPayload; + type Sidecar: AbstractSidecar; +} + +pub struct FullBlockProposal {} +impl BlockProposal for FullBlockProposal { + type Payload = FullPayload; + type Sidecar = BlobSidecar; +} + +pub struct BlindedBlockProposal {} +impl BlockProposal for BlindedBlockProposal { + type Payload = BlindedPayload; + type Sidecar = BlindedBlobSidecar; +} + /// A wrapper over a [`BeaconBlock`] or a [`BeaconBlockAndBlobSidecars`]. -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize)] #[serde(untagged)] #[serde(bound = "T: EthSpec")] -pub enum BlockContents, Sidecar: AbstractSidecar> { - BlockAndBlobSidecars(BeaconBlockAndBlobSidecars), - BlindedBlockAndBlobSidecars(BlindedBeaconBlockAndBlobSidecars), - Block(BeaconBlock), +pub enum BlockContents> { + BlockAndBlobSidecars(BeaconBlockAndBlobSidecars), + BlindedBlockAndBlobSidecars(BlindedBeaconBlockAndBlobSidecars), + Block(BeaconBlock), } -impl, Sidecar: AbstractSidecar> - BlockContents -{ - pub fn block(&self) -> &BeaconBlock { +pub type BlockContentsTuple = ( + BeaconBlock>::Payload>, + Option>::Sidecar>>, +); + +impl> BlockContents { + pub fn block(&self) -> &BeaconBlock { match self { BlockContents::BlockAndBlobSidecars(block_and_sidecars) => &block_and_sidecars.block, BlockContents::BlindedBlockAndBlobSidecars(block_and_sidecars) => { @@ -1296,7 +1316,7 @@ impl, Sidecar: AbstractSidecar> } } - pub fn deconstruct(self) -> (BeaconBlock, Option>) { + pub fn deconstruct(self) -> BlockContentsTuple { match self { BlockContents::BlockAndBlobSidecars(block_and_sidecars) => ( block_and_sidecars.block, @@ -1311,9 +1331,7 @@ impl, Sidecar: AbstractSidecar> } } -impl, Sidecar: AbstractSidecar> - ForkVersionDeserialize for BlockContents -{ +impl> ForkVersionDeserialize for BlockContents { fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( value: serde_json::value::Value, fork_name: ForkName, @@ -1332,10 +1350,8 @@ impl, Sidecar: AbstractSidecar> } } -impl, Sidecar: AbstractSidecar> - Into> for BlockContents -{ - fn into(self) -> BeaconBlock { +impl> Into> for BlockContents { + fn into(self) -> BeaconBlock { match self { Self::BlockAndBlobSidecars(block_and_sidecars) => block_and_sidecars.block, Self::BlindedBlockAndBlobSidecars(block_and_sidecars) => { @@ -1346,29 +1362,23 @@ impl, Sidecar: AbstractSidecar> } } -pub type BlockContentsTuple = ( - SignedBeaconBlock, - Option>, +pub type SignedBlockContentsTuple = ( + SignedBeaconBlock>::Payload>, + Option>::Sidecar>>, ); /// A wrapper over a [`SignedBeaconBlock`] or a [`SignedBeaconBlockAndBlobSidecars`]. -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize)] #[serde(untagged)] #[serde(bound = "T: EthSpec")] -pub enum SignedBlockContents< - T: EthSpec, - Payload: AbstractExecPayload = FullPayload, - Sidecar: AbstractSidecar = BlobSidecar, -> { - BlockAndBlobSidecars(SignedBeaconBlockAndBlobSidecars), - BlindedBlockAndBlobSidecars(SignedBlindedBeaconBlockAndBlobSidecars), - Block(SignedBeaconBlock), +pub enum SignedBlockContents> { + BlockAndBlobSidecars(SignedBeaconBlockAndBlobSidecars), + BlindedBlockAndBlobSidecars(SignedBlindedBeaconBlockAndBlobSidecars), + Block(SignedBeaconBlock), } -impl, Sidecar: AbstractSidecar> - SignedBlockContents -{ - pub fn signed_block(&self) -> &SignedBeaconBlock { +impl> SignedBlockContents { + pub fn signed_block(&self) -> &SignedBeaconBlock { match self { SignedBlockContents::BlockAndBlobSidecars(block_and_sidecars) => { &block_and_sidecars.signed_block @@ -1380,7 +1390,7 @@ impl, Sidecar: AbstractSidecar> } } - pub fn deconstruct(self) -> BlockContentsTuple { + pub fn deconstruct(self) -> SignedBlockContentsTuple { match self { SignedBlockContents::BlockAndBlobSidecars(block_and_sidecars) => ( block_and_sidecars.signed_block, @@ -1395,10 +1405,10 @@ impl, Sidecar: AbstractSidecar> } } -impl> From> - for SignedBlockContents +impl> From> + for SignedBlockContents { - fn from(block: SignedBeaconBlock) -> Self { + fn from(block: SignedBeaconBlock) -> Self { match block { SignedBeaconBlock::Base(_) | SignedBeaconBlock::Altair(_) @@ -1410,13 +1420,13 @@ impl> From, Sidecar: AbstractSidecar> - From> for SignedBlockContents +impl> From> + for SignedBlockContents { - fn from(block_contents_tuple: BlockContentsTuple) -> Self { + fn from(block_contents_tuple: SignedBlockContentsTuple) -> Self { match block_contents_tuple { (signed_block, None) => SignedBlockContents::Block(signed_block), - (signed_block, Some(signed_blob_sidecars)) => match Payload::block_type() { + (signed_block, Some(signed_blob_sidecars)) => match B::Payload::block_type() { BlockType::Blinded => SignedBlockContents::BlindedBlockAndBlobSidecars( SignedBlindedBeaconBlockAndBlobSidecars { signed_blinded_block: signed_block, @@ -1436,29 +1446,19 @@ impl, Sidecar: AbstractSidecar> #[derive(Debug, Clone, Serialize, Deserialize, Encode)] #[serde(bound = "T: EthSpec")] -pub struct SignedBeaconBlockAndBlobSidecars< - T: EthSpec, - Payload: AbstractExecPayload, - Sidecar: AbstractSidecar, -> { - pub signed_block: SignedBeaconBlock, - pub signed_blob_sidecars: SignedSidecarList, +pub struct SignedBeaconBlockAndBlobSidecars> { + pub signed_block: SignedBeaconBlock, + pub signed_blob_sidecars: SignedSidecarList, } #[derive(Debug, Clone, Serialize, Deserialize, Encode)] -#[serde(bound = "T: EthSpec, Payload: AbstractExecPayload, Sidecar: AbstractSidecar")] -pub struct BeaconBlockAndBlobSidecars< - T: EthSpec, - Payload: AbstractExecPayload, - Sidecar: AbstractSidecar, -> { - pub block: BeaconBlock, - pub blob_sidecars: SidecarList, +#[serde(bound = "T: EthSpec, B: BlockProposal")] +pub struct BeaconBlockAndBlobSidecars> { + pub block: BeaconBlock, + pub blob_sidecars: SidecarList, } -impl, Sidecar: AbstractSidecar> - ForkVersionDeserialize for BeaconBlockAndBlobSidecars -{ +impl> ForkVersionDeserialize for BeaconBlockAndBlobSidecars { fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( value: serde_json::value::Value, fork_name: ForkName, @@ -1469,7 +1469,7 @@ impl, Sidecar: AbstractSidecar> block: serde_json::Value, blob_sidecars: SidecarList, } - let helper: Helper = + let helper: Helper = serde_json::from_value(value).map_err(serde::de::Error::custom)?; Ok(Self { @@ -1483,26 +1483,22 @@ impl, Sidecar: AbstractSidecar> #[serde(bound = "T: EthSpec")] pub struct SignedBlindedBeaconBlockAndBlobSidecars< T: EthSpec, - Payload: AbstractExecPayload, - Sidecar: AbstractSidecar, + B: BlockProposal = BlindedBlockProposal, > { - pub signed_blinded_block: SignedBeaconBlock, - pub signed_blinded_blob_sidecars: SignedSidecarList, + pub signed_blinded_block: SignedBeaconBlock, + pub signed_blinded_blob_sidecars: SignedSidecarList, } #[derive(Debug, Clone, Serialize, Deserialize, Encode)] #[serde(bound = "T: EthSpec")] -pub struct BlindedBeaconBlockAndBlobSidecars< - T: EthSpec, - Payload: AbstractExecPayload, - Sidecar: AbstractSidecar, -> { - pub blinded_block: BeaconBlock, - pub blinded_blob_sidecars: SidecarList, +pub struct BlindedBeaconBlockAndBlobSidecars = BlindedBlockProposal> +{ + pub blinded_block: BeaconBlock, + pub blinded_blob_sidecars: SidecarList, } -impl, Sidecar: AbstractSidecar> - ForkVersionDeserialize for BlindedBeaconBlockAndBlobSidecars +impl> ForkVersionDeserialize + for BlindedBeaconBlockAndBlobSidecars { fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( value: serde_json::value::Value, @@ -1514,7 +1510,7 @@ impl, Sidecar: AbstractSidecar> blinded_block: serde_json::Value, blinded_blob_sidecars: SidecarList, } - let helper: Helper = + let helper: Helper = serde_json::from_value(value).map_err(serde::de::Error::custom)?; Ok(Self { diff --git a/consensus/types/src/blob_sidecar.rs b/consensus/types/src/blob_sidecar.rs index ed9f10f208d..991e383f50e 100644 --- a/consensus/types/src/blob_sidecar.rs +++ b/consensus/types/src/blob_sidecar.rs @@ -1,3 +1,4 @@ +use std::fmt::Debug; use std::hash::Hash; use std::marker::PhantomData; use std::sync::Arc; @@ -26,8 +27,11 @@ pub trait AbstractSidecar: + Hash + TreeHash + TestRandom + + Debug + + SignedRoot + for<'a> arbitrary::Arbitrary<'a> { + fn slot(&self) -> Slot; } /// Container of the data that identifies an individual blob. @@ -81,7 +85,11 @@ pub struct BlobSidecar { pub kzg_proof: KzgProof, } -impl AbstractSidecar for BlobSidecar {} +impl AbstractSidecar for BlobSidecar { + fn slot(&self) -> Slot { + self.slot + } +} impl PartialOrd for BlobSidecar { fn partial_cmp(&self, other: &Self) -> Option { @@ -168,7 +176,13 @@ pub struct BlindedBlobSidecar { pub kzg_proof: KzgProof, } -impl AbstractSidecar for BlindedBlobSidecar {} +impl SignedRoot for BlindedBlobSidecar {} + +impl AbstractSidecar for BlindedBlobSidecar { + fn slot(&self) -> Slot { + self.slot + } +} pub type SidecarList = VariableList, ::MaxBlobsPerBlock>; pub type BlobSidecarList = SidecarList>; diff --git a/consensus/types/src/signed_blob.rs b/consensus/types/src/signed_blob.rs index 16252bd025a..dd521f99a61 100644 --- a/consensus/types/src/signed_blob.rs +++ b/consensus/types/src/signed_blob.rs @@ -39,6 +39,16 @@ pub struct SignedSidecar> { pub _phantom: PhantomData, } +impl> SignedSidecar { + pub fn new(message: Arc, signature: Signature) -> SignedSidecar { + Self { + message, + signature, + _phantom: Default::default(), + } + } +} + /// List of Signed Sidecars that implements `AbstractSidecar`. pub type SignedSidecarList = VariableList, ::MaxBlobsPerBlock>; diff --git a/validator_client/src/block_service.rs b/validator_client/src/block_service.rs index bbf44f85c2b..15bff9b8db0 100644 --- a/validator_client/src/block_service.rs +++ b/validator_client/src/block_service.rs @@ -8,7 +8,9 @@ use crate::{ use crate::{http_metrics::metrics, validator_store::ValidatorStore}; use bls::SignatureBytes; use environment::RuntimeContext; -use eth2::types::{BlockContents, SignedBlockContents}; +use eth2::types::{ + BlindedBlockProposal, BlockContents, BlockProposal, FullBlockProposal, SignedBlockContents, +}; use eth2::BeaconNodeHttpClient; use slog::{crit, debug, error, info, trace, warn, Logger}; use slot_clock::SlotClock; @@ -19,10 +21,7 @@ use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc; use tokio::time::sleep; -use types::{ - AbstractExecPayload, AbstractSidecar, BlindedPayload, BlockType, EthSpec, FullPayload, - Graffiti, PublicKeyBytes, Slot, -}; +use types::{BlockType, EthSpec, ExecPayload, Graffiti, PublicKeyBytes, Slot}; #[derive(Debug)] pub enum BlockError { @@ -343,7 +342,7 @@ impl BlockService { if builder_proposals { let result = service .clone() - .publish_block::>(slot, validator_pubkey) + .publish_block::(slot, validator_pubkey) .await; match result { Err(BlockError::Recoverable(e)) => { @@ -355,7 +354,7 @@ impl BlockService { "info" => "blinded proposal failed, attempting full block" ); if let Err(e) = service - .publish_block::>(slot, validator_pubkey) + .publish_block::(slot, validator_pubkey) .await { // Log a `crit` since a full block @@ -384,7 +383,7 @@ impl BlockService { Ok(_) => {} }; } else if let Err(e) = service - .publish_block::>(slot, validator_pubkey) + .publish_block::(slot, validator_pubkey) .await { // Log a `crit` since a full block (non-builder) @@ -406,7 +405,7 @@ impl BlockService { } /// Produce a block at the given slot for validator_pubkey - async fn publish_block>( + async fn publish_block>( self, slot: Slot, validator_pubkey: PublicKeyBytes, @@ -458,7 +457,7 @@ impl BlockService { // // Try the proposer nodes last, since it's likely that they don't have a // great view of attestations on the network. - let block_contents = proposer_fallback + let block_contents: BlockContents = proposer_fallback .first_success_try_proposers_last( RequireSynced::No, OfflineOnFailure::Yes, @@ -480,7 +479,7 @@ impl BlockService { let signed_block = self_ref .validator_store - .sign_block::(*validator_pubkey_ref, block, current_slot) + .sign_block::(*validator_pubkey_ref, block, current_slot) .await .map_err(|e| BlockError::Recoverable(format!("Unable to sign block: {:?}", e)))?; @@ -488,7 +487,7 @@ impl BlockService { Some(blob_sidecars) => Some( self_ref .validator_store - .sign_blobs(*validator_pubkey_ref, blob_sidecars) + .sign_blobs::(*validator_pubkey_ref, blob_sidecars) .await .map_err(|e| { BlockError::Recoverable(format!("Unable to sign blob: {:?}", e)) @@ -519,11 +518,8 @@ impl BlockService { RequireSynced::No, OfflineOnFailure::Yes, |beacon_node| async { - Self::publish_signed_block_contents::( - &signed_block_contents, - beacon_node, - ) - .await + Self::publish_signed_block_contents::(&signed_block_contents, beacon_node) + .await }, ) .await?; @@ -531,7 +527,7 @@ impl BlockService { info!( log, "Successfully published block"; - "block_type" => ?Payload::block_type(), + "block_type" => ?B::Payload::block_type(), "deposits" => signed_block_contents.signed_block().message().body().deposits().len(), "attestations" => signed_block_contents.signed_block().message().body().attestations().len(), "graffiti" => ?graffiti.map(|g| g.as_utf8_lossy()), @@ -541,11 +537,11 @@ impl BlockService { Ok(()) } - async fn publish_signed_block_contents>( - signed_block_contents: &SignedBlockContents, + async fn publish_signed_block_contents>( + signed_block_contents: &SignedBlockContents, beacon_node: &BeaconNodeHttpClient, ) -> Result<(), BlockError> { - match Payload::block_type() { + match B::Payload::block_type() { BlockType::Full => { let _post_timer = metrics::start_timer_vec( &metrics::BLOCK_SERVICE_TIMES, @@ -581,15 +577,15 @@ impl BlockService { Ok::<_, BlockError>(()) } - async fn get_validator_block, Sidecar: AbstractSidecar>( + async fn get_validator_block>( beacon_node: &BeaconNodeHttpClient, slot: Slot, randao_reveal_ref: &SignatureBytes, graffiti: Option, proposer_index: Option, log: &Logger, - ) -> Result, BlockError> { - let block_contents: BlockContents = match Payload::block_type() { + ) -> Result, BlockError> { + let block_contents: BlockContents = match B::Payload::block_type() { BlockType::Full => { let _get_timer = metrics::start_timer_vec( &metrics::BLOCK_SERVICE_TIMES, diff --git a/validator_client/src/signing_method.rs b/validator_client/src/signing_method.rs index 5291ad6ddc0..64a356224e4 100644 --- a/validator_client/src/signing_method.rs +++ b/validator_client/src/signing_method.rs @@ -4,6 +4,7 @@ //! - Via a remote signer (Web3Signer) use crate::http_metrics::metrics; +use eth2::types::{BlockProposal, FullBlockProposal}; use eth2_keystore::Keystore; use lockfile::Lockfile; use parking_lot::Mutex; @@ -34,10 +35,10 @@ pub enum Error { } /// Enumerates all messages that can be signed by a validator. -pub enum SignableMessage<'a, T: EthSpec, Payload: AbstractExecPayload = FullPayload> { +pub enum SignableMessage<'a, T: EthSpec, B: BlockProposal = FullBlockProposal> { RandaoReveal(Epoch), - BeaconBlock(&'a BeaconBlock), - BlobSidecar(&'a BlobSidecar), + BeaconBlock(&'a BeaconBlock), + BlobSidecar(&'a B::Sidecar), AttestationData(&'a AttestationData), SignedAggregateAndProof(&'a AggregateAndProof), SelectionProof(Slot), @@ -51,7 +52,7 @@ pub enum SignableMessage<'a, T: EthSpec, Payload: AbstractExecPayload = FullP VoluntaryExit(&'a VoluntaryExit), } -impl<'a, T: EthSpec, Payload: AbstractExecPayload> SignableMessage<'a, T, Payload> { +impl<'a, T: EthSpec, B: BlockProposal> SignableMessage<'a, T, B> { /// Returns the `SignedRoot` for the contained message. /// /// The actual `SignedRoot` trait is not used since it also requires a `TreeHash` impl, which is @@ -120,9 +121,9 @@ impl SigningContext { impl SigningMethod { /// Return the signature of `signable_message`, with respect to the `signing_context`. - pub async fn get_signature>( + pub async fn get_signature>( &self, - signable_message: SignableMessage<'_, T, Payload>, + signable_message: SignableMessage<'_, T, B>, signing_context: SigningContext, spec: &ChainSpec, executor: &TaskExecutor, @@ -145,9 +146,9 @@ impl SigningMethod { .await } - pub async fn get_signature_from_root>( + pub async fn get_signature_from_root>( &self, - signable_message: SignableMessage<'_, T, Payload>, + signable_message: SignableMessage<'_, T, B>, signing_root: Hash256, executor: &TaskExecutor, fork_info: Option, diff --git a/validator_client/src/validator_store.rs b/validator_client/src/validator_store.rs index 9033267a485..6429fedbae0 100644 --- a/validator_client/src/validator_store.rs +++ b/validator_client/src/validator_store.rs @@ -6,7 +6,7 @@ use crate::{ Config, }; use account_utils::{validator_definitions::ValidatorDefinition, ZeroizeString}; -use eth2::types::VariableList; +use eth2::types::{BlindedBlockProposal, BlockProposal, VariableList}; use parking_lot::{Mutex, RwLock}; use slashing_protection::{ interchange::Interchange, InterchangeError, NotSafe, Safe, SlashingDatabase, @@ -19,14 +19,13 @@ use std::path::Path; use std::sync::Arc; use task_executor::TaskExecutor; use types::{ - attestation::Error as AttestationError, graffiti::GraffitiString, AbstractExecPayload, Address, - AggregateAndProof, Attestation, BeaconBlock, BlindedPayload, BlobSidecar, BlobSidecarList, - ChainSpec, ContributionAndProof, Domain, Epoch, EthSpec, Fork, Graffiti, Hash256, Keypair, - PublicKeyBytes, SelectionProof, Signature, SignedAggregateAndProof, SignedBeaconBlock, - SignedBlobSidecar, SignedContributionAndProof, SignedRoot, SignedSidecarList, - SignedValidatorRegistrationData, SignedVoluntaryExit, Slot, SyncAggregatorSelectionData, - SyncCommitteeContribution, SyncCommitteeMessage, SyncSelectionProof, SyncSubnetId, - ValidatorRegistrationData, VoluntaryExit, + attestation::Error as AttestationError, graffiti::GraffitiString, AbstractSidecar, Address, + AggregateAndProof, Attestation, BeaconBlock, ChainSpec, ContributionAndProof, Domain, Epoch, + EthSpec, Fork, Graffiti, Hash256, Keypair, PublicKeyBytes, SelectionProof, SidecarList, + Signature, SignedAggregateAndProof, SignedBeaconBlock, SignedContributionAndProof, SignedRoot, + SignedSidecar, SignedSidecarList, SignedValidatorRegistrationData, SignedVoluntaryExit, Slot, + SyncAggregatorSelectionData, SyncCommitteeContribution, SyncCommitteeMessage, + SyncSelectionProof, SyncSubnetId, ValidatorRegistrationData, VoluntaryExit, }; use validator_dir::ValidatorDir; @@ -388,7 +387,7 @@ impl ValidatorStore { let signing_context = self.signing_context(Domain::Randao, signing_epoch); let signature = signing_method - .get_signature::>( + .get_signature::( SignableMessage::RandaoReveal(signing_epoch), signing_context, &self.spec, @@ -465,12 +464,12 @@ impl ValidatorStore { .unwrap_or(self.builder_proposals) } - pub async fn sign_block>( + pub async fn sign_block>( &self, validator_pubkey: PublicKeyBytes, - block: BeaconBlock, + block: BeaconBlock, current_slot: Slot, - ) -> Result, Error> { + ) -> Result, Error> { // Make sure the block slot is not higher than the current slot to avoid potential attacks. if block.slot() > current_slot { warn!( @@ -503,7 +502,7 @@ impl ValidatorStore { let signing_method = self.doppelganger_checked_signing_method(validator_pubkey)?; let signature = signing_method - .get_signature::( + .get_signature::( SignableMessage::BeaconBlock(&block), signing_context, &self.spec, @@ -542,22 +541,21 @@ impl ValidatorStore { } } - pub async fn sign_blobs( + pub async fn sign_blobs>( &self, validator_pubkey: PublicKeyBytes, - blob_sidecars: BlobSidecarList, - ) -> Result>, Error> { + blob_sidecars: SidecarList, + ) -> Result, Error> { let mut signed_blob_sidecars = Vec::new(); - for blob_sidecar in blob_sidecars.into_iter() { - let slot = blob_sidecar.slot; + let slot = blob_sidecar.slot(); let signing_epoch = slot.epoch(E::slots_per_epoch()); let signing_context = self.signing_context(Domain::BlobSidecar, signing_epoch); let signing_method = self.doppelganger_checked_signing_method(validator_pubkey)?; let signature = signing_method - .get_signature::>( - SignableMessage::BlobSidecar(&blob_sidecar), + .get_signature::( + SignableMessage::BlobSidecar(blob_sidecar.as_ref()), signing_context, &self.spec, &self.task_executor, @@ -566,11 +564,7 @@ impl ValidatorStore { metrics::inc_counter_vec(&metrics::SIGNED_BLOBS_TOTAL, &[metrics::SUCCESS]); - signed_blob_sidecars.push(SignedBlobSidecar { - message: blob_sidecar, - signature, - _phantom: PhantomData, - }); + signed_blob_sidecars.push(SignedSidecar::new(blob_sidecar, signature)); } Ok(VariableList::from(signed_blob_sidecars)) @@ -606,7 +600,7 @@ impl ValidatorStore { Ok(Safe::Valid) => { let signing_method = self.doppelganger_checked_signing_method(validator_pubkey)?; let signature = signing_method - .get_signature::>( + .get_signature::( SignableMessage::AttestationData(&attestation.data), signing_context, &self.spec, @@ -671,7 +665,7 @@ impl ValidatorStore { let signing_method = self.doppelganger_bypassed_signing_method(validator_pubkey)?; let signature = signing_method - .get_signature::>( + .get_signature::( SignableMessage::VoluntaryExit(&voluntary_exit), signing_context, &self.spec, @@ -697,7 +691,7 @@ impl ValidatorStore { let signing_method = self.doppelganger_bypassed_signing_method(validator_registration_data.pubkey)?; let signature = signing_method - .get_signature_from_root::>( + .get_signature_from_root::( SignableMessage::ValidatorRegistration(&validator_registration_data), signing_root, &self.task_executor, @@ -738,7 +732,7 @@ impl ValidatorStore { let signing_method = self.doppelganger_checked_signing_method(validator_pubkey)?; let signature = signing_method - .get_signature::>( + .get_signature::( SignableMessage::SignedAggregateAndProof(&message), signing_context, &self.spec, @@ -771,7 +765,7 @@ impl ValidatorStore { let signing_method = self.doppelganger_bypassed_signing_method(validator_pubkey)?; let signature = signing_method - .get_signature::>( + .get_signature::( SignableMessage::SelectionProof(slot), signing_context, &self.spec, @@ -810,7 +804,7 @@ impl ValidatorStore { }; let signature = signing_method - .get_signature::>( + .get_signature::( SignableMessage::SyncSelectionProof(&message), signing_context, &self.spec, @@ -836,7 +830,7 @@ impl ValidatorStore { let signing_method = self.doppelganger_bypassed_signing_method(*validator_pubkey)?; let signature = signing_method - .get_signature::>( + .get_signature::( SignableMessage::SyncCommitteeSignature { beacon_block_root, slot, @@ -881,7 +875,7 @@ impl ValidatorStore { }; let signature = signing_method - .get_signature::>( + .get_signature::( SignableMessage::SignedContributionAndProof(&message), signing_context, &self.spec, From b0d8d902c0fbd7c040514b7525862e00d0c0688b Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Tue, 4 Jul 2023 17:11:45 +1000 Subject: [PATCH 07/24] Rename `AbstractSidecar` to `Sidecar` --- beacon_node/beacon_chain/src/blob_cache.rs | 16 ++++++++-------- common/eth2/src/types.rs | 14 +++++++------- consensus/types/src/blob_sidecar.rs | 6 +++--- consensus/types/src/lib.rs | 4 ++-- consensus/types/src/signed_blob.rs | 20 ++++++++++---------- validator_client/src/validator_store.rs | 8 ++++---- 6 files changed, 34 insertions(+), 34 deletions(-) diff --git a/beacon_node/beacon_chain/src/blob_cache.rs b/beacon_node/beacon_chain/src/blob_cache.rs index 307cb7476c0..6144b109087 100644 --- a/beacon_node/beacon_chain/src/blob_cache.rs +++ b/beacon_node/beacon_chain/src/blob_cache.rs @@ -1,18 +1,18 @@ use lru::LruCache; use parking_lot::Mutex; -use types::{AbstractSidecar, EthSpec, Hash256, SidecarList}; +use types::{EthSpec, Hash256, Sidecar, SidecarList}; pub const DEFAULT_BLOB_CACHE_SIZE: usize = 10; /// A cache blobs by beacon block root. -pub struct BlobCache> { - blobs: Mutex>>, +pub struct BlobCache> { + blobs: Mutex>>, } #[derive(Hash, PartialEq, Eq)] struct BlobCacheId(Hash256); -impl> Default for BlobCache { +impl> Default for BlobCache { fn default() -> Self { BlobCache { blobs: Mutex::new(LruCache::new(DEFAULT_BLOB_CACHE_SIZE)), @@ -20,16 +20,16 @@ impl> Default for BlobCache } } -impl> BlobCache { +impl> BlobCache { pub fn put( &self, beacon_block: Hash256, - blobs: SidecarList, - ) -> Option> { + blobs: SidecarList, + ) -> Option> { self.blobs.lock().put(BlobCacheId(beacon_block), blobs) } - pub fn pop(&self, root: &Hash256) -> Option> { + pub fn pop(&self, root: &Hash256) -> Option> { self.blobs.lock().pop(&BlobCacheId(*root)) } } diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index f92b67c8d42..3b65f3797ee 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -1302,7 +1302,7 @@ mod tests { pub trait BlockProposal { type Payload: AbstractExecPayload; - type Sidecar: AbstractSidecar; + type Sidecar: Sidecar; } pub struct FullBlockProposal {} @@ -1491,10 +1491,10 @@ impl> ForkVersionDeserialize for BeaconBlockAndB fork_name: ForkName, ) -> Result { #[derive(Deserialize)] - #[serde(bound = "T: EthSpec, Sidecar: AbstractSidecar")] - struct Helper> { + #[serde(bound = "T: EthSpec, S: Sidecar")] + struct Helper> { block: serde_json::Value, - blob_sidecars: SidecarList, + blob_sidecars: SidecarList, } let helper: Helper = serde_json::from_value(value).map_err(serde::de::Error::custom)?; @@ -1532,10 +1532,10 @@ impl> ForkVersionDeserialize fork_name: ForkName, ) -> Result { #[derive(Deserialize)] - #[serde(bound = "T: EthSpec, Sidecar: AbstractSidecar,")] - struct Helper> { + #[serde(bound = "T: EthSpec, S: Sidecar")] + struct Helper> { blinded_block: serde_json::Value, - blinded_blob_sidecars: SidecarList, + blinded_blob_sidecars: SidecarList, } let helper: Helper = serde_json::from_value(value).map_err(serde::de::Error::custom)?; diff --git a/consensus/types/src/blob_sidecar.rs b/consensus/types/src/blob_sidecar.rs index 9822a660dca..a76af99270c 100644 --- a/consensus/types/src/blob_sidecar.rs +++ b/consensus/types/src/blob_sidecar.rs @@ -20,7 +20,7 @@ use test_random_derive::TestRandom; use crate::test_utils::TestRandom; use crate::{Blob, ChainSpec, Domain, EthSpec, Fork, Hash256, SignedBlobSidecar, SignedRoot, Slot}; -pub trait AbstractSidecar: +pub trait Sidecar: serde::Serialize + DeserializeOwned + Encode @@ -86,7 +86,7 @@ pub struct BlobSidecar { pub kzg_proof: KzgProof, } -impl AbstractSidecar for BlobSidecar { +impl Sidecar for BlobSidecar { fn slot(&self) -> Slot { self.slot } @@ -211,7 +211,7 @@ pub struct BlindedBlobSidecar { impl SignedRoot for BlindedBlobSidecar {} -impl AbstractSidecar for BlindedBlobSidecar { +impl Sidecar for BlindedBlobSidecar { fn slot(&self) -> Slot { self.slot } diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 0d2593bb187..f7e2903ab37 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -119,8 +119,8 @@ pub use crate::beacon_block_header::BeaconBlockHeader; pub use crate::beacon_committee::{BeaconCommittee, OwnedBeaconCommittee}; pub use crate::beacon_state::{BeaconTreeHashCache, Error as BeaconStateError, *}; pub use crate::blob_sidecar::{ - AbstractSidecar, BlindedBlobSidecar, BlindedBlobSidecarList, BlobRoots, BlobSidecar, - BlobSidecarList, Blobs, SidecarList, + BlindedBlobSidecar, BlindedBlobSidecarList, BlobRoots, BlobSidecar, BlobSidecarList, Blobs, + Sidecar, SidecarList, }; pub use crate::bls_to_execution_change::BlsToExecutionChange; pub use crate::chain_spec::{ChainSpec, Config, Domain}; diff --git a/consensus/types/src/signed_blob.rs b/consensus/types/src/signed_blob.rs index dd521f99a61..b5261c93c84 100644 --- a/consensus/types/src/signed_blob.rs +++ b/consensus/types/src/signed_blob.rs @@ -1,6 +1,6 @@ use crate::{ - test_utils::TestRandom, AbstractSidecar, BlobSidecar, ChainSpec, Domain, EthSpec, Fork, - Hash256, Signature, SignedRoot, SigningData, + test_utils::TestRandom, BlobSidecar, ChainSpec, Domain, EthSpec, Fork, Hash256, Sidecar, + Signature, SignedRoot, SigningData, }; use bls::PublicKey; use derivative::Derivative; @@ -26,11 +26,11 @@ use tree_hash_derive::TreeHash; Derivative, arbitrary::Arbitrary, )] -#[serde(bound = "T: EthSpec, Sidecar: AbstractSidecar")] -#[arbitrary(bound = "T: EthSpec, Sidecar: AbstractSidecar")] -#[derivative(Hash(bound = "T: EthSpec, Sidecar: AbstractSidecar"))] -pub struct SignedSidecar> { - pub message: Arc, +#[serde(bound = "T: EthSpec, S: Sidecar")] +#[arbitrary(bound = "T: EthSpec, S: Sidecar")] +#[derivative(Hash(bound = "T: EthSpec, S: Sidecar"))] +pub struct SignedSidecar> { + pub message: Arc, pub signature: Signature, #[ssz(skip_serializing, skip_deserializing)] #[tree_hash(skip_hashing)] @@ -39,8 +39,8 @@ pub struct SignedSidecar> { pub _phantom: PhantomData, } -impl> SignedSidecar { - pub fn new(message: Arc, signature: Signature) -> SignedSidecar { +impl> SignedSidecar { + pub fn new(message: Arc, signature: Signature) -> SignedSidecar { Self { message, signature, @@ -49,7 +49,7 @@ impl> SignedSidecar { } } -/// List of Signed Sidecars that implements `AbstractSidecar`. +/// List of Signed Sidecars that implements `Sidecar`. pub type SignedSidecarList = VariableList, ::MaxBlobsPerBlock>; diff --git a/validator_client/src/validator_store.rs b/validator_client/src/validator_store.rs index 6429fedbae0..bfa77c7df3d 100644 --- a/validator_client/src/validator_store.rs +++ b/validator_client/src/validator_store.rs @@ -19,10 +19,10 @@ use std::path::Path; use std::sync::Arc; use task_executor::TaskExecutor; use types::{ - attestation::Error as AttestationError, graffiti::GraffitiString, AbstractSidecar, Address, - AggregateAndProof, Attestation, BeaconBlock, ChainSpec, ContributionAndProof, Domain, Epoch, - EthSpec, Fork, Graffiti, Hash256, Keypair, PublicKeyBytes, SelectionProof, SidecarList, - Signature, SignedAggregateAndProof, SignedBeaconBlock, SignedContributionAndProof, SignedRoot, + attestation::Error as AttestationError, graffiti::GraffitiString, Address, AggregateAndProof, + Attestation, BeaconBlock, ChainSpec, ContributionAndProof, Domain, Epoch, EthSpec, Fork, + Graffiti, Hash256, Keypair, PublicKeyBytes, SelectionProof, Sidecar, SidecarList, Signature, + SignedAggregateAndProof, SignedBeaconBlock, SignedContributionAndProof, SignedRoot, SignedSidecar, SignedSidecarList, SignedValidatorRegistrationData, SignedVoluntaryExit, Slot, SyncAggregatorSelectionData, SyncCommitteeContribution, SyncCommitteeMessage, SyncSelectionProof, SyncSubnetId, ValidatorRegistrationData, VoluntaryExit, From 33ac5924c3150af1755f5e65f7f0a92feaa13f8d Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Thu, 27 Jul 2023 00:56:59 +1000 Subject: [PATCH 08/24] Remove blob cache as it's no longer necessary --- beacon_node/beacon_chain/src/beacon_chain.rs | 133 +++++++++--------- beacon_node/beacon_chain/src/blob_cache.rs | 35 ----- beacon_node/beacon_chain/src/builder.rs | 3 - beacon_node/beacon_chain/src/lib.rs | 1 - beacon_node/beacon_chain/src/test_utils.rs | 20 +-- .../http_api/src/build_block_contents.rs | 17 +-- beacon_node/http_api/src/lib.rs | 20 +-- common/eth2/src/types.rs | 2 +- consensus/types/src/blob_sidecar.rs | 5 + consensus/types/src/lib.rs | 5 +- 10 files changed, 97 insertions(+), 144 deletions(-) delete mode 100644 beacon_node/beacon_chain/src/blob_cache.rs diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 5684094d74c..22da1b23952 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -7,7 +7,6 @@ use crate::attester_cache::{AttesterCache, AttesterCacheKey}; use crate::beacon_block_streamer::{BeaconBlockStreamer, CheckEarlyAttesterCache}; use crate::beacon_proposer_cache::compute_proposer_duties_from_head; use crate::beacon_proposer_cache::BeaconProposerCache; -use crate::blob_cache::BlobCache; use crate::blob_verification::{self, BlobError, GossipVerifiedBlob}; use crate::block_times_cache::BlockTimesCache; use crate::block_verification::POS_PANDA_BANNER; @@ -69,7 +68,7 @@ use crate::validator_monitor::{ }; use crate::validator_pubkey_cache::ValidatorPubkeyCache; use crate::{metrics, BeaconChainError, BeaconForkChoiceStore, BeaconSnapshot, CachedHead}; -use eth2::types::{EventKind, SseBlock, SseExtendedPayloadAttributes, SyncDuty}; +use eth2::types::{BlockProposal, EventKind, SseBlock, SseExtendedPayloadAttributes, SyncDuty}; use execution_layer::{ BlockProposalContents, BuilderParams, ChainHealth, ExecutionLayer, FailedCondition, PayloadAttributes, PayloadStatus, @@ -119,9 +118,6 @@ use tokio_stream::Stream; use tree_hash::TreeHash; use types::beacon_block_body::KzgCommitments; use types::beacon_state::CloneConfig; -use types::blob_sidecar::{ - BlindedBlobSidecar, BlindedBlobSidecarList, BlobRoots, BlobSidecarList, Blobs, BlobsOrBlobRoots, -}; use types::consts::deneb::MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS; use types::*; @@ -480,13 +476,15 @@ pub struct BeaconChain { pub validator_monitor: RwLock>, /// The slot at which blocks are downloaded back to. pub genesis_backfill_slot: Slot, - pub proposal_blob_cache: BlobCache>, - pub proposal_blinded_blob_cache: BlobCache, pub data_availability_checker: Arc>, pub kzg: Option::Kzg>>>, } -type BeaconBlockAndState = (BeaconBlock, BeaconState); +type BeaconBlockAndState = ( + BeaconBlock>::Payload>, + BeaconState, + Option>, +); impl FinalizationAndCanonicity { pub fn is_finalized(self) -> bool { @@ -3819,13 +3817,13 @@ impl BeaconChain { /// /// The produced block will not be inherently valid, it must be signed by a block producer. /// Block signing is out of the scope of this function and should be done by a separate program. - pub async fn produce_block + 'static>( + pub async fn produce_block + 'static>( self: &Arc, randao_reveal: Signature, slot: Slot, validator_graffiti: Option, - ) -> Result, BlockProductionError> { - self.produce_block_with_verification( + ) -> Result, BlockProductionError> { + self.produce_block_with_verification::( randao_reveal, slot, validator_graffiti, @@ -3835,15 +3833,13 @@ impl BeaconChain { } /// Same as `produce_block` but allowing for configuration of RANDAO-verification. - pub async fn produce_block_with_verification< - Payload: AbstractExecPayload + 'static, - >( + pub async fn produce_block_with_verification + 'static>( self: &Arc, randao_reveal: Signature, slot: Slot, validator_graffiti: Option, verification: ProduceBlockVerification, - ) -> Result, BlockProductionError> { + ) -> Result, BlockProductionError> { // Part 1/2 (blocking) // // Load the parent state from disk. @@ -3861,7 +3857,7 @@ impl BeaconChain { // Part 2/2 (async, with some blocking components) // // Produce the block upon the state - self.produce_block_on_state::( + self.produce_block_on_state::( state, state_root_opt, slot, @@ -4433,7 +4429,7 @@ impl BeaconChain { /// The provided `state_root_opt` should only ever be set to `Some` if the contained value is /// equal to the root of `state`. Providing this value will serve as an optimization to avoid /// performing a tree hash in some scenarios. - pub async fn produce_block_on_state + 'static>( + pub async fn produce_block_on_state + 'static>( self: &Arc, state: BeaconState, state_root_opt: Option, @@ -4441,7 +4437,7 @@ impl BeaconChain { randao_reveal: Signature, validator_graffiti: Option, verification: ProduceBlockVerification, - ) -> Result, BlockProductionError> { + ) -> Result, BlockProductionError> { // Part 1/3 (blocking) // // Perform the state advance and block-packing functions. @@ -4450,7 +4446,7 @@ impl BeaconChain { .task_executor .spawn_blocking_handle( move || { - chain.produce_partial_beacon_block( + chain.produce_partial_beacon_block::( state, state_root_opt, produce_at_slot, @@ -4486,7 +4482,7 @@ impl BeaconChain { self.task_executor .spawn_blocking_handle( move || { - chain.complete_partial_beacon_block( + chain.complete_partial_beacon_block::( partial_beacon_block, block_contents, verification, @@ -4499,14 +4495,14 @@ impl BeaconChain { .map_err(BlockProductionError::TokioJoin)? } - fn produce_partial_beacon_block + 'static>( + fn produce_partial_beacon_block + 'static>( self: &Arc, mut state: BeaconState, state_root_opt: Option, produce_at_slot: Slot, randao_reveal: Signature, validator_graffiti: Option, - ) -> Result, BlockProductionError> { + ) -> Result, BlockProductionError> { let eth1_chain = self .eth1_chain .as_ref() @@ -4738,12 +4734,12 @@ impl BeaconChain { }) } - fn complete_partial_beacon_block>( + fn complete_partial_beacon_block>( &self, - partial_beacon_block: PartialBeaconBlock, - block_contents: Option>, + partial_beacon_block: PartialBeaconBlock, + block_contents: Option>, verification: ProduceBlockVerification, - ) -> Result, BlockProductionError> { + ) -> Result, BlockProductionError> { let (mut state, inner_block, blobs_opt, proofs_opt) = Self::merge_block_contents_into_beacon_block(partial_beacon_block, block_contents)?; @@ -4795,40 +4791,45 @@ impl BeaconChain { //FIXME(sean) // - add a new timer for processing here - if let (Some(blobs_or_blobs_roots), Some(proofs)) = (blobs_opt, proofs_opt) { - let expected_kzg_commitments = block.body().blob_kzg_commitments().map_err(|_| { - BlockProductionError::InvalidBlockVariant( - "DENEB block does not contain kzg commitments".to_string(), - ) - })?; + let maybe_sidecar_list = match (blobs_opt, proofs_opt) { + (Some(blobs_or_blobs_roots), Some(proofs)) => { + let expected_kzg_commitments = + block.body().blob_kzg_commitments().map_err(|_| { + BlockProductionError::InvalidBlockVariant( + "DENEB block does not contain kzg commitments".to_string(), + ) + })?; - if expected_kzg_commitments.len() != blobs_or_blobs_roots.len() { - return Err(BlockProductionError::MissingKzgCommitment(format!( - "Missing KZG commitment for slot {}. Expected {}, got: {}", - block.slot(), - blobs_or_blobs_roots.len(), - expected_kzg_commitments.len() - ))); - } + if expected_kzg_commitments.len() != blobs_or_blobs_roots.len() { + return Err(BlockProductionError::MissingKzgCommitment(format!( + "Missing KZG commitment for slot {}. Expected {}, got: {}", + block.slot(), + blobs_or_blobs_roots.len(), + expected_kzg_commitments.len() + ))); + } + + let kzg_proofs = Vec::from(proofs); - let kzg_proofs = Vec::from(proofs); - - match blobs_or_blobs_roots { - BlobsOrBlobRoots::Blobs(blobs) => self.build_and_cache_blob_sidecars( - &block, - blobs, - expected_kzg_commitments, - kzg_proofs, - )?, - BlobsOrBlobRoots::BlobRoots(blob_roots) => self - .build_and_cache_blinded_blob_sidecars( + match blobs_or_blobs_roots { + BlobsOrBlobRoots::Blobs(blobs) => Some(self.build_blob_sidecars::( &block, - blob_roots, + blobs, expected_kzg_commitments, kzg_proofs, - )?, + )?), + BlobsOrBlobRoots::BlobRoots(blob_roots) => { + Some(self.build_blinded_blob_sidecars::( + &block, + blob_roots, + expected_kzg_commitments, + kzg_proofs, + )?) + } + } } - } + _ => None, + }; metrics::inc_counter(&metrics::BLOCK_PRODUCTION_SUCCESSES); @@ -4840,7 +4841,7 @@ impl BeaconChain { "slot" => block.slot() ); - Ok((block, state)) + Ok((block, state, maybe_sidecar_list)) } /// This method must be called whenever an execution engine indicates that a payload is @@ -6097,13 +6098,13 @@ impl BeaconChain { .unwrap_or(false)) } - fn build_and_cache_blob_sidecars>( + fn build_blob_sidecars>( &self, - block: &BeaconBlock, + block: &BeaconBlock, blobs: Blobs, expected_kzg_commitments: &KzgCommitments, kzg_proofs: Vec, - ) -> Result<(), BlockProductionError> { + ) -> Result, BlockProductionError> { let kzg = self .kzg .as_ref() @@ -6141,19 +6142,16 @@ impl BeaconChain { .collect::, BlockProductionError>>()?, ); - self.proposal_blob_cache - .put(beacon_block_root, blob_sidecars); - - Ok(()) + Ok(SidecarListVariant::Full(blob_sidecars)) } - fn build_and_cache_blinded_blob_sidecars>( + fn build_blinded_blob_sidecars>( &self, - block: &BeaconBlock, + block: &BeaconBlock, blob_roots: BlobRoots, expected_kzg_commitments: &KzgCommitments, kzg_proofs: Vec, - ) -> Result<(), BlockProductionError> { + ) -> Result, BlockProductionError> { let beacon_block_root = block.canonical_root(); let slot = block.slot(); @@ -6188,10 +6186,7 @@ impl BeaconChain { .collect::, BlockProductionError>>()?, ); - self.proposal_blinded_blob_cache - .put(beacon_block_root, blob_sidecars); - - Ok(()) + Ok(SidecarListVariant::Blinded(blob_sidecars)) } #[allow(clippy::type_complexity)] //TODO(jimmy): fix type complexity diff --git a/beacon_node/beacon_chain/src/blob_cache.rs b/beacon_node/beacon_chain/src/blob_cache.rs deleted file mode 100644 index 6144b109087..00000000000 --- a/beacon_node/beacon_chain/src/blob_cache.rs +++ /dev/null @@ -1,35 +0,0 @@ -use lru::LruCache; -use parking_lot::Mutex; -use types::{EthSpec, Hash256, Sidecar, SidecarList}; - -pub const DEFAULT_BLOB_CACHE_SIZE: usize = 10; - -/// A cache blobs by beacon block root. -pub struct BlobCache> { - blobs: Mutex>>, -} - -#[derive(Hash, PartialEq, Eq)] -struct BlobCacheId(Hash256); - -impl> Default for BlobCache { - fn default() -> Self { - BlobCache { - blobs: Mutex::new(LruCache::new(DEFAULT_BLOB_CACHE_SIZE)), - } - } -} - -impl> BlobCache { - pub fn put( - &self, - beacon_block: Hash256, - blobs: SidecarList, - ) -> Option> { - self.blobs.lock().put(BlobCacheId(beacon_block), blobs) - } - - pub fn pop(&self, root: &Hash256) -> Option> { - self.blobs.lock().pop(&BlobCacheId(*root)) - } -} diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index 9c857f5ba06..45c4f42411d 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -1,5 +1,4 @@ use crate::beacon_chain::{CanonicalHead, BEACON_CHAIN_DB_KEY, ETH1_CACHE_DB_KEY, OP_POOL_DB_KEY}; -use crate::blob_cache::BlobCache; use crate::data_availability_checker::DataAvailabilityChecker; use crate::eth1_chain::{CachingEth1Backend, SszEth1}; use crate::eth1_finalization_cache::Eth1FinalizationCache; @@ -889,8 +888,6 @@ where DataAvailabilityChecker::new(slot_clock, kzg.clone(), store, self.spec) .map_err(|e| format!("Error initializing DataAvailabiltyChecker: {:?}", e))?, ), - proposal_blob_cache: BlobCache::default(), - proposal_blinded_blob_cache: BlobCache::default(), kzg, }; diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index 3410196c8c4..426670a2b6c 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -7,7 +7,6 @@ mod beacon_chain; mod beacon_fork_choice_store; pub mod beacon_proposer_cache; mod beacon_snapshot; -pub mod blob_cache; pub mod blob_verification; pub mod block_reward; mod block_times_cache; diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 4dbd2a4a0cc..b0d9868d4d8 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -863,9 +863,9 @@ where let randao_reveal = self.sign_randao_reveal(&state, proposer_index, slot); - let (block, state) = self + let (block, state, maybe_blob_sidecars) = self .chain - .produce_block_on_state( + .produce_block_on_state::( state, None, slot, @@ -889,11 +889,7 @@ where | SignedBeaconBlock::Merge(_) | SignedBeaconBlock::Capella(_) => (signed_block, None), SignedBeaconBlock::Deneb(_) => { - if let Some(blobs) = self - .chain - .proposal_blob_cache - .pop(&signed_block.canonical_root()) - { + if let Some(SidecarListVariant::Full(blobs)) = maybe_blob_sidecars { let signed_blobs: SignedSidecarList> = Vec::from(blobs) .into_iter() .map(|blob| { @@ -952,9 +948,9 @@ where let pre_state = state.clone(); - let (block, state) = self + let (block, state, maybe_blob_sidecars) = self .chain - .produce_block_on_state( + .produce_block_on_state::( state, None, slot, @@ -978,11 +974,7 @@ where | SignedBeaconBlock::Merge(_) | SignedBeaconBlock::Capella(_) => (signed_block, None), SignedBeaconBlock::Deneb(_) => { - if let Some(blobs) = self - .chain - .proposal_blob_cache - .pop(&signed_block.canonical_root()) - { + if let Some(SidecarListVariant::Full(blobs)) = maybe_blob_sidecars { let signed_blobs: SignedSidecarList> = Vec::from(blobs) .into_iter() .map(|blob| { diff --git a/beacon_node/http_api/src/build_block_contents.rs b/beacon_node/http_api/src/build_block_contents.rs index 176c1b3515b..ae709714a74 100644 --- a/beacon_node/http_api/src/build_block_contents.rs +++ b/beacon_node/http_api/src/build_block_contents.rs @@ -1,10 +1,9 @@ -use beacon_chain::{BeaconChain, BeaconChainTypes, BlockProductionError}; +use beacon_chain::{BeaconChainTypes, BlockProductionError}; use eth2::types::{ BeaconBlockAndBlobSidecars, BlindedBeaconBlockAndBlobSidecars, BlindedBlockProposal, BlockContents, FullBlockProposal, }; -use std::sync::Arc; -use types::{BeaconBlock, BlindedPayload, ForkName, FullPayload}; +use types::{BeaconBlock, BlindedPayload, ForkName, FullPayload, SidecarListVariant}; type Error = warp::reject::Rejection; type FullBlockContents = BlockContents; @@ -12,16 +11,15 @@ type BlindedBlockContents = BlockContents; pub fn build_block_contents( fork_name: ForkName, - chain: Arc>, block: BeaconBlock>, + maybe_blobs: Option>, ) -> Result, Error> { match fork_name { ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { Ok(BlockContents::Block(block)) } ForkName::Deneb => { - let block_root = &block.canonical_root(); - if let Some(blob_sidecars) = chain.proposal_blob_cache.pop(block_root) { + if let Some(SidecarListVariant::Full(blob_sidecars)) = maybe_blobs { let block_and_blobs = BeaconBlockAndBlobSidecars { block, blob_sidecars, @@ -39,19 +37,18 @@ pub fn build_block_contents( pub fn build_blinded_block_contents( fork_name: ForkName, - chain: Arc>, block: BeaconBlock>, + maybe_blobs: Option>, ) -> Result, Error> { match fork_name { ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { Ok(BlockContents::Block(block)) } ForkName::Deneb => { - let block_root = &block.canonical_root(); - if let Some(blob_sidecars) = chain.proposal_blinded_blob_cache.pop(block_root) { + if let Some(SidecarListVariant::Blinded(blinded_blob_sidecars)) = maybe_blobs { let block_and_blobs = BlindedBeaconBlockAndBlobSidecars { blinded_block: block, - blinded_blob_sidecars: blob_sidecars, + blinded_blob_sidecars, }; Ok(BlockContents::BlindedBlockAndBlobSidecars(block_and_blobs)) diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 9447341104c..1fc32ad0971 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -62,10 +62,10 @@ use tokio::sync::mpsc::{Sender, UnboundedSender}; use tokio_stream::{wrappers::BroadcastStream, StreamExt}; use types::{ Attestation, AttestationData, AttestationShufflingId, AttesterSlashing, BeaconStateError, - BlindedPayload, CommitteeCache, ConfigAndPreset, Epoch, EthSpec, ForkName, FullPayload, - ProposerPreparationData, ProposerSlashing, RelativeEpoch, SignedAggregateAndProof, - SignedBlsToExecutionChange, SignedContributionAndProof, SignedValidatorRegistrationData, - SignedVoluntaryExit, Slot, SyncCommitteeMessage, SyncContributionData, + CommitteeCache, ConfigAndPreset, Epoch, EthSpec, ForkName, ProposerPreparationData, + ProposerSlashing, RelativeEpoch, SignedAggregateAndProof, SignedBlsToExecutionChange, + SignedContributionAndProof, SignedValidatorRegistrationData, SignedVoluntaryExit, Slot, + SyncCommitteeMessage, SyncContributionData, }; use version::{ add_consensus_version_header, execution_optimistic_finalized_fork_versioned_response, @@ -2723,8 +2723,8 @@ pub fn serve( ProduceBlockVerification::VerifyRandao }; - let (block, _) = chain - .produce_block_with_verification::>( + let (block, _, maybe_blobs) = chain + .produce_block_with_verification::( randao_reveal, slot, query.graffiti.map(Into::into), @@ -2738,7 +2738,7 @@ pub fn serve( .map_err(inconsistent_fork_rejection)?; let block_contents = - build_block_contents::build_block_contents(fork_name, chain, block)?; + build_block_contents::build_block_contents::(fork_name, block, maybe_blobs)?; fork_versioned_response(endpoint_version, fork_name, block_contents) .map(|response| warp::reply::json(&response).into_response()) @@ -2782,8 +2782,8 @@ pub fn serve( ProduceBlockVerification::VerifyRandao }; - let (block, _) = chain - .produce_block_with_verification::>( + let (block, _, maybe_blobs) = chain + .produce_block_with_verification::( randao_reveal, slot, query.graffiti.map(Into::into), @@ -2797,7 +2797,7 @@ pub fn serve( .map_err(inconsistent_fork_rejection)?; let block_contents = - build_block_contents::build_blinded_block_contents(fork_name, chain, block)?; + build_block_contents::build_blinded_block_contents::(fork_name, block, maybe_blobs)?; // Pose as a V2 endpoint so we return the fork `version`. fork_versioned_response(V2, fork_name, block_contents) diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 8c6e13815e0..096012fa6d5 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -1351,7 +1351,7 @@ mod tests { } } -pub trait BlockProposal { +pub trait BlockProposal: Send { type Payload: AbstractExecPayload; type Sidecar: Sidecar; } diff --git a/consensus/types/src/blob_sidecar.rs b/consensus/types/src/blob_sidecar.rs index 5411b877744..76967ecec62 100644 --- a/consensus/types/src/blob_sidecar.rs +++ b/consensus/types/src/blob_sidecar.rs @@ -20,6 +20,11 @@ use test_random_derive::TestRandom; use crate::test_utils::TestRandom; use crate::{Blob, ChainSpec, Domain, EthSpec, Fork, Hash256, SignedBlobSidecar, SignedRoot, Slot}; +pub enum SidecarListVariant { + Full(BlobSidecarList), + Blinded(BlindedBlobSidecarList), +} + pub trait Sidecar: serde::Serialize + Clone diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 2ce37354d4c..41acf993f97 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -118,7 +118,10 @@ pub use crate::beacon_block_body::{ pub use crate::beacon_block_header::BeaconBlockHeader; pub use crate::beacon_committee::{BeaconCommittee, OwnedBeaconCommittee}; pub use crate::beacon_state::{BeaconTreeHashCache, Error as BeaconStateError, *}; -pub use crate::blob_sidecar::*; +pub use crate::blob_sidecar::{ + BlindedBlobSidecar, BlindedBlobSidecarList, BlobRoots, BlobSidecar, BlobSidecarList, Blobs, + BlobsOrBlobRoots, Sidecar, SidecarList, SidecarListVariant, +}; pub use crate::bls_to_execution_change::BlsToExecutionChange; pub use crate::chain_spec::{ChainSpec, Config, Domain}; pub use crate::checkpoint::Checkpoint; From df84f174ec9bbe048dc747b5f256733271239abd Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Thu, 27 Jul 2023 01:10:44 +1000 Subject: [PATCH 09/24] Remove unnecessary enum variant --- .../beacon_chain/src/block_verification.rs | 13 +++--- beacon_node/execution_layer/src/lib.rs | 41 ++++--------------- 2 files changed, 13 insertions(+), 41 deletions(-) diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index 3b67bb7d75d..d9691bc722a 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -98,11 +98,10 @@ use store::{Error as DBError, HotStateSummary, KeyValueStore, StoreOp}; use task_executor::JoinHandle; use tree_hash::TreeHash; use types::{ - BeaconBlockRef, BeaconState, BeaconStateError, BlobSidecar, ChainSpec, CloneConfig, Epoch, - EthSpec, ExecutionBlockHash, Hash256, InconsistentFork, PublicKey, PublicKeyBytes, - RelativeEpoch, SignedBeaconBlock, SignedBeaconBlockHeader, Slot, + BeaconBlockRef, BeaconState, BeaconStateError, ChainSpec, CloneConfig, Epoch, EthSpec, + ExecPayload, ExecutionBlockHash, Hash256, InconsistentFork, PublicKey, PublicKeyBytes, + RelativeEpoch, SignedBeaconBlock, SignedBeaconBlockHeader, SignedBlobSidecarList, Slot, }; -use types::{ExecPayload, SignedSidecarList}; pub const POS_PANDA_BANNER: &str = r#" ,,, ,,, ,,, ,,, @@ -664,7 +663,7 @@ pub trait IntoGossipVerifiedBlockContents: Sized { chain: &BeaconChain, ) -> Result, BlockError>; fn inner_block(&self) -> &SignedBeaconBlock; - fn inner_blobs(&self) -> Option>>; + fn inner_blobs(&self) -> Option>; } impl IntoGossipVerifiedBlockContents for GossipVerifiedBlockContents { @@ -677,7 +676,7 @@ impl IntoGossipVerifiedBlockContents for GossipVerifiedB fn inner_block(&self) -> &SignedBeaconBlock { self.0.block.as_block() } - fn inner_blobs(&self) -> Option>> { + fn inner_blobs(&self) -> Option> { self.1.as_ref().map(|blobs| { VariableList::from( blobs @@ -715,7 +714,7 @@ impl IntoGossipVerifiedBlockContents self.signed_block() } - fn inner_blobs(&self) -> Option>> { + fn inner_blobs(&self) -> Option> { self.blobs_cloned() } } diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index aa9ad654af5..0ec4b19a25f 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -45,7 +45,7 @@ use types::blob_sidecar::BlobsOrBlobRoots; use types::builder_bid::BuilderBid; use types::KzgProofs; use types::{ - AbstractExecPayload, BeaconStateError, Blobs, ExecPayload, ExecutionPayloadDeneb, VersionedHash, + AbstractExecPayload, BeaconStateError, ExecPayload, ExecutionPayloadDeneb, VersionedHash, }; use types::{ BlindedPayload, BlockType, ChainSpec, Epoch, ExecutionPayloadCapella, ExecutionPayloadMerge, @@ -104,11 +104,11 @@ impl> From> payload: builder_bid.header, block_value: builder_bid.value, }, - BuilderBid::Deneb(builder_bid) => BlockProposalContents::PayloadAndBlindedBlobs { + BuilderBid::Deneb(builder_bid) => BlockProposalContents::PayloadAndBlobs { payload: builder_bid.header, block_value: builder_bid.value, kzg_commitments: builder_bid.blinded_blobs_bundle.commitments, - blob_roots: builder_bid.blinded_blobs_bundle.blob_roots, + blobs: BlobsOrBlobRoots::BlobRoots(builder_bid.blinded_blobs_bundle.blob_roots), proofs: builder_bid.blinded_blobs_bundle.proofs, }, }; @@ -161,14 +161,7 @@ pub enum BlockProposalContents> { payload: Payload, block_value: Uint256, kzg_commitments: KzgCommitments, - blobs: Blobs, - proofs: KzgProofs, - }, - PayloadAndBlindedBlobs { - payload: Payload, - block_value: Uint256, - kzg_commitments: KzgCommitments, - blob_roots: VariableList, + blobs: BlobsOrBlobRoots, proofs: KzgProofs, }, } @@ -183,7 +176,7 @@ impl> From> payload: execution_payload.into(), block_value, kzg_commitments: bundle.commitments, - blobs: bundle.blobs, + blobs: BlobsOrBlobRoots::Blobs(bundle.blobs), proofs: bundle.proofs, }, None => Self::Payload { @@ -215,24 +208,7 @@ impl> BlockProposalContents ( - payload, - Some(kzg_commitments), - Some(BlobsOrBlobRoots::Blobs(blobs)), - Some(proofs), - ), - Self::PayloadAndBlindedBlobs { - payload, - block_value: _, - kzg_commitments, - blob_roots, - proofs, - } => ( - payload, - Some(kzg_commitments), - Some(BlobsOrBlobRoots::BlobRoots(blob_roots)), - Some(proofs), - ), + } => (payload, Some(kzg_commitments), Some(blobs), Some(proofs)), } } @@ -240,21 +216,18 @@ impl> BlockProposalContents payload, Self::PayloadAndBlobs { payload, .. } => payload, - Self::PayloadAndBlindedBlobs { payload, .. } => payload, } } pub fn to_payload(self) -> Payload { match self { Self::Payload { payload, .. } => payload, Self::PayloadAndBlobs { payload, .. } => payload, - Self::PayloadAndBlindedBlobs { payload, .. } => payload, } } pub fn block_value(&self) -> &Uint256 { match self { Self::Payload { block_value, .. } => block_value, Self::PayloadAndBlobs { block_value, .. } => block_value, - Self::PayloadAndBlindedBlobs { block_value, .. } => block_value, } } pub fn default_at_fork(fork_name: ForkName) -> Result { @@ -268,7 +241,7 @@ impl> BlockProposalContents BlockProposalContents::PayloadAndBlobs { payload: Payload::default_at_fork(fork_name)?, block_value: Uint256::zero(), - blobs: VariableList::default(), + blobs: BlobsOrBlobRoots::Blobs(VariableList::default()), kzg_commitments: VariableList::default(), proofs: VariableList::default(), }, From 1cfa6d2de04035e172fdf1d41393855ffb793f0b Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Thu, 27 Jul 2023 01:42:01 +1000 Subject: [PATCH 10/24] Clean up --- beacon_node/beacon_chain/src/beacon_chain.rs | 20 ++++---- beacon_node/beacon_chain/src/errors.rs | 2 +- beacon_node/beacon_chain/src/test_utils.rs | 9 +++- .../beacon_chain/tests/block_verification.rs | 1 + beacon_node/execution_layer/src/lib.rs | 6 +-- .../http_api/src/build_block_contents.rs | 4 +- beacon_node/http_api/src/lib.rs | 13 +++--- beacon_node/http_api/src/publish_blocks.rs | 13 ++---- .../http_api/tests/interactive_tests.rs | 8 ++-- beacon_node/http_api/tests/tests.rs | 46 +++++++++---------- .../src/network_beacon_processor/tests.rs | 6 +-- common/eth2/src/lib.rs | 4 +- common/eth2/src/types.rs | 25 ++-------- consensus/types/src/block_proposal.rs | 21 +++++++++ consensus/types/src/lib.rs | 2 + consensus/types/src/signed_blob.rs | 11 ----- testing/web3signer_tests/src/lib.rs | 14 ++++-- validator_client/src/block_service.rs | 9 ++-- validator_client/src/signing_method.rs | 1 - validator_client/src/validator_store.rs | 21 +++++---- 20 files changed, 119 insertions(+), 117 deletions(-) create mode 100644 consensus/types/src/block_proposal.rs diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 22da1b23952..3b0edaf6fa8 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -68,7 +68,7 @@ use crate::validator_monitor::{ }; use crate::validator_pubkey_cache::ValidatorPubkeyCache; use crate::{metrics, BeaconChainError, BeaconForkChoiceStore, BeaconSnapshot, CachedHead}; -use eth2::types::{BlockProposal, EventKind, SseBlock, SseExtendedPayloadAttributes, SyncDuty}; +use eth2::types::{EventKind, SseBlock, SseExtendedPayloadAttributes, SyncDuty}; use execution_layer::{ BlockProposalContents, BuilderParams, ChainHealth, ExecutionLayer, FailedCondition, PayloadAttributes, PayloadStatus, @@ -4446,7 +4446,7 @@ impl BeaconChain { .task_executor .spawn_blocking_handle( move || { - chain.produce_partial_beacon_block::( + chain.produce_partial_beacon_block::( state, state_root_opt, produce_at_slot, @@ -4495,14 +4495,14 @@ impl BeaconChain { .map_err(BlockProductionError::TokioJoin)? } - fn produce_partial_beacon_block + 'static>( + fn produce_partial_beacon_block + 'static>( self: &Arc, mut state: BeaconState, state_root_opt: Option, produce_at_slot: Slot, randao_reveal: Signature, validator_graffiti: Option, - ) -> Result, BlockProductionError> { + ) -> Result, BlockProductionError> { let eth1_chain = self .eth1_chain .as_ref() @@ -4812,14 +4812,14 @@ impl BeaconChain { let kzg_proofs = Vec::from(proofs); match blobs_or_blobs_roots { - BlobsOrBlobRoots::Blobs(blobs) => Some(self.build_blob_sidecars::( + BlobsOrBlobRoots::Blobs(blobs) => Some(self.build_blob_sidecars( &block, blobs, expected_kzg_commitments, kzg_proofs, )?), BlobsOrBlobRoots::BlobRoots(blob_roots) => { - Some(self.build_blinded_blob_sidecars::( + Some(self.build_blinded_blob_sidecars( &block, blob_roots, expected_kzg_commitments, @@ -6098,9 +6098,9 @@ impl BeaconChain { .unwrap_or(false)) } - fn build_blob_sidecars>( + fn build_blob_sidecars>( &self, - block: &BeaconBlock, + block: &BeaconBlock, blobs: Blobs, expected_kzg_commitments: &KzgCommitments, kzg_proofs: Vec, @@ -6145,9 +6145,9 @@ impl BeaconChain { Ok(SidecarListVariant::Full(blob_sidecars)) } - fn build_blinded_blob_sidecars>( + fn build_blinded_blob_sidecars>( &self, - block: &BeaconBlock, + block: &BeaconBlock, blob_roots: BlobRoots, expected_kzg_commitments: &KzgCommitments, kzg_proofs: Vec, diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs index 2e715655ddf..d0da91406b5 100644 --- a/beacon_node/beacon_chain/src/errors.rs +++ b/beacon_node/beacon_chain/src/errors.rs @@ -275,11 +275,11 @@ pub enum BlockProductionError { blob_block_hash: ExecutionBlockHash, payload_block_hash: ExecutionBlockHash, }, - NoBlobsCached, FailedToReadFinalizedBlock(store::Error), MissingFinalizedBlock(Hash256), BlockTooLarge(usize), ShuttingDown, + MissingBlobs, MissingSyncAggregate, MissingExecutionPayload, MissingKzgCommitment(String), diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index b0d9868d4d8..6d40ca19530 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -15,7 +15,7 @@ use crate::{ StateSkipConfig, }; use bls::get_withdrawal_credentials; -use eth2::types::{BlindedBlockProposal, FullBlockProposal, SignedBlockContentsTuple}; +use eth2::types::SignedBlockContentsTuple; use execution_layer::test_utils::generate_genesis_header; use execution_layer::{ auth::JwtKey, @@ -50,6 +50,7 @@ use state_processing::{ use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use std::fmt; +use std::marker::PhantomData; use std::str::FromStr; use std::sync::Arc; use std::time::Duration; @@ -828,7 +829,11 @@ where .into_iter() .map(|blob_sidecar| { let blinded_sidecar: BlindedBlobSidecar = blob_sidecar.message.into(); - SignedSidecar::new(Arc::new(blinded_sidecar), blob_sidecar.signature) + SignedSidecar { + message: Arc::new(blinded_sidecar), + signature: blob_sidecar.signature, + _phantom: PhantomData, + } }) .collect(), ) diff --git a/beacon_node/beacon_chain/tests/block_verification.rs b/beacon_node/beacon_chain/tests/block_verification.rs index 102707a3892..fb7715b4fad 100644 --- a/beacon_node/beacon_chain/tests/block_verification.rs +++ b/beacon_node/beacon_chain/tests/block_verification.rs @@ -128,6 +128,7 @@ async fn get_chain_segment_with_signed_blobs() -> ( .get(&BlobSignatureKey::new(block_root, blob_index)) .unwrap() .clone(), + _phantom: PhantomData, } }) .collect::>(); diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 0ec4b19a25f..ba1519ac7d5 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -13,8 +13,8 @@ pub use engine_api::*; pub use engine_api::{http, http::deposit_methods, http::HttpJsonRpc}; use engines::{Engine, EngineError}; pub use engines::{EngineState, ForkchoiceState}; +use eth2::types::SignedBlockContents; use eth2::types::{builder_bid::SignedBuilderBid, ForkVersionedResponse}; -use eth2::types::{BlindedBlockProposal, SignedBlockContents}; use ethers_core::abi::ethereum_types::FromStrRadixErr; use ethers_core::types::Transaction as EthersTransaction; use fork_choice::ForkchoiceUpdateParameters; @@ -48,7 +48,8 @@ use types::{ AbstractExecPayload, BeaconStateError, ExecPayload, ExecutionPayloadDeneb, VersionedHash, }; use types::{ - BlindedPayload, BlockType, ChainSpec, Epoch, ExecutionPayloadCapella, ExecutionPayloadMerge, + BlindedBlockProposal, BlindedPayload, BlockType, ChainSpec, Epoch, ExecutionPayloadCapella, + ExecutionPayloadMerge, }; use types::{ProposerPreparationData, PublicKeyBytes, Signature, Slot, Transaction}; @@ -1154,7 +1155,6 @@ impl ExecutionLayer { } }; - // TODO(jimmy): cache blobs bundle let payload_response = async { debug!( self.log(), diff --git a/beacon_node/http_api/src/build_block_contents.rs b/beacon_node/http_api/src/build_block_contents.rs index ae709714a74..5392692748b 100644 --- a/beacon_node/http_api/src/build_block_contents.rs +++ b/beacon_node/http_api/src/build_block_contents.rs @@ -28,7 +28,7 @@ pub fn build_block_contents( Ok(BlockContents::BlockAndBlobSidecars(block_and_blobs)) } else { Err(warp_utils::reject::block_production_error( - BlockProductionError::NoBlobsCached, + BlockProductionError::MissingBlobs, )) } } @@ -54,7 +54,7 @@ pub fn build_blinded_block_contents( Ok(BlockContents::BlindedBlockAndBlobSidecars(block_and_blobs)) } else { Err(warp_utils::reject::block_production_error( - BlockProductionError::NoBlobsCached, + BlockProductionError::MissingBlobs, )) } } diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 1fc32ad0971..6e55b855b92 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -32,9 +32,8 @@ use beacon_chain::{ pub use block_id::BlockId; use directory::DEFAULT_ROOT_DIR; use eth2::types::{ - self as api_types, BlindedBlockProposal, BroadcastValidation, EndpointVersion, ForkChoice, - ForkChoiceNode, FullBlockProposal, SignedBlockContents, SkipRandaoVerification, ValidatorId, - ValidatorStatus, + self as api_types, BroadcastValidation, EndpointVersion, ForkChoice, ForkChoiceNode, + FullBlockProposal, SignedBlockContents, SkipRandaoVerification, ValidatorId, ValidatorStatus, }; use lighthouse_network::{types::SyncState, EnrExt, NetworkGlobals, PeerId, PubsubMessage}; use lighthouse_version::version_with_platform; @@ -62,10 +61,10 @@ use tokio::sync::mpsc::{Sender, UnboundedSender}; use tokio_stream::{wrappers::BroadcastStream, StreamExt}; use types::{ Attestation, AttestationData, AttestationShufflingId, AttesterSlashing, BeaconStateError, - CommitteeCache, ConfigAndPreset, Epoch, EthSpec, ForkName, ProposerPreparationData, - ProposerSlashing, RelativeEpoch, SignedAggregateAndProof, SignedBlsToExecutionChange, - SignedContributionAndProof, SignedValidatorRegistrationData, SignedVoluntaryExit, Slot, - SyncCommitteeMessage, SyncContributionData, + BlindedBlockProposal, CommitteeCache, ConfigAndPreset, Epoch, EthSpec, ForkName, + ProposerPreparationData, ProposerSlashing, RelativeEpoch, SignedAggregateAndProof, + SignedBlsToExecutionChange, SignedContributionAndProof, SignedValidatorRegistrationData, + SignedVoluntaryExit, Slot, SyncCommitteeMessage, SyncContributionData, }; use version::{ add_consensus_version_header, execution_optimistic_finalized_fork_versioned_response, diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index b8c4ea9bbd5..b8dd4971e97 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -6,7 +6,7 @@ use beacon_chain::{ AvailabilityProcessingStatus, BeaconChain, BeaconChainError, BeaconChainTypes, BlockError, IntoGossipVerifiedBlockContents, NotifyExecutionLayer, }; -use eth2::types::{BlindedBlockProposal, SignedBlockContents}; +use eth2::types::SignedBlockContents; use eth2::types::{BroadcastValidation, FullBlockProposal}; use execution_layer::ProvenancedPayload; use lighthouse_network::PubsubMessage; @@ -19,8 +19,8 @@ use std::time::Duration; use tokio::sync::mpsc::UnboundedSender; use tree_hash::TreeHash; use types::{ - AbstractExecPayload, BeaconBlockRef, EthSpec, ExecPayload, ExecutionBlockHash, FullPayload, - Hash256, SignedBeaconBlock, SignedBlobSidecarList, + AbstractExecPayload, BeaconBlockRef, BlindedBlockProposal, EthSpec, ExecPayload, + ExecutionBlockHash, FullPayload, Hash256, SignedBeaconBlock, SignedBlobSidecarList, }; use warp::Rejection; @@ -317,7 +317,6 @@ pub async fn reconstruct_block( .into(); ProvenancedPayload::Local(payload) // If we already have an execution payload with this transactions root cached, use it. - // TODO(jimmy) get cached blobs } else if let Some(cached_payload) = el.get_payload_by_root(&payload_header.tree_hash_root()) { @@ -365,20 +364,14 @@ pub async fn reconstruct_block( // A block without a payload is pre-merge and we consider it locally // built. None => block - // .deconstruct() - // .0 .try_into_full_block(None) .map(SignedBlockContents::Block) .map(ProvenancedBlock::local), Some(ProvenancedPayload::Local(full_payload)) => block - // .deconstruct() - // .0 .try_into_full_block(Some(full_payload)) .map(SignedBlockContents::Block) .map(ProvenancedBlock::local), Some(ProvenancedPayload::Builder(full_payload)) => block - // .deconstruct() - // .0 .try_into_full_block(Some(full_payload)) .map(SignedBlockContents::Block) .map(ProvenancedBlock::builder), diff --git a/beacon_node/http_api/tests/interactive_tests.rs b/beacon_node/http_api/tests/interactive_tests.rs index f27a4d9519d..63f46471d7f 100644 --- a/beacon_node/http_api/tests/interactive_tests.rs +++ b/beacon_node/http_api/tests/interactive_tests.rs @@ -4,7 +4,7 @@ use beacon_chain::{ test_utils::{AttestationStrategy, BlockStrategy, SyncCommitteeStrategy}, ChainConfig, }; -use eth2::types::{DepositContractData, StateId}; +use eth2::types::{DepositContractData, FullBlockProposal, StateId}; use execution_layer::{ForkchoiceState, PayloadAttributes}; use http_api::test_utils::InteractiveTester; use parking_lot::Mutex; @@ -17,8 +17,8 @@ use std::sync::Arc; use std::time::Duration; use tree_hash::TreeHash; use types::{ - Address, Epoch, EthSpec, ExecPayload, ExecutionBlockHash, ForkName, FullPayload, - MainnetEthSpec, MinimalEthSpec, ProposerPreparationData, Slot, + Address, Epoch, EthSpec, ExecPayload, ExecutionBlockHash, ForkName, MainnetEthSpec, + MinimalEthSpec, ProposerPreparationData, Slot, }; type E = MainnetEthSpec; @@ -812,7 +812,7 @@ pub async fn fork_choice_before_proposal() { .into(); let block_d = tester .client - .get_validator_blocks::>(slot_d, &randao_reveal, None) + .get_validator_blocks::(slot_d, &randao_reveal, None) .await .unwrap() .data diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index f5a5d74e29b..7d31b5a3bf9 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -63,8 +63,8 @@ struct ApiTester { harness: Arc>>, chain: Arc>>, client: BeaconNodeHttpClient, - next_block: SignedBlockContents, - reorg_block: SignedBlockContents, + next_block: SignedBlockContents, + reorg_block: SignedBlockContents, attestations: Vec>, contribution_and_proofs: Vec>, attester_slashing: AttesterSlashing, @@ -2385,7 +2385,7 @@ impl ApiTester { let block = self .client - .get_validator_blocks::>(slot, &randao_reveal, None) + .get_validator_blocks::(slot, &randao_reveal, None) .await .unwrap() .data @@ -2414,7 +2414,7 @@ impl ApiTester { let block = self .client - .get_validator_blocks_modular::>( + .get_validator_blocks_modular::( slot, &Signature::infinity().unwrap().into(), None, @@ -2472,13 +2472,13 @@ impl ApiTester { // Check failure with no `skip_randao_verification` passed. self.client - .get_validator_blocks::>(slot, &bad_randao_reveal, None) + .get_validator_blocks::(slot, &bad_randao_reveal, None) .await .unwrap_err(); // Check failure with `skip_randao_verification` (requires infinity sig). self.client - .get_validator_blocks_modular::>( + .get_validator_blocks_modular::( slot, &bad_randao_reveal, None, @@ -2493,7 +2493,7 @@ impl ApiTester { self } - pub async fn test_blinded_block_production>(&self) { + pub async fn test_blinded_block_production>(&self) { let fork = self.chain.canonical_head.cached_head().head_fork(); let genesis_validators_root = self.chain.genesis_validators_root; @@ -2533,7 +2533,7 @@ impl ApiTester { let block = self .client - .get_validator_blinded_blocks::(slot, &randao_reveal, None) + .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() .data; @@ -2553,15 +2553,13 @@ impl ApiTester { } } - pub async fn test_blinded_block_production_no_verify_randao>( - self, - ) -> Self { + pub async fn test_blinded_block_production_no_verify_randao>(self) -> Self { for _ in 0..E::slots_per_epoch() { let slot = self.chain.slot().unwrap(); let block = self .client - .get_validator_blinded_blocks_modular::( + .get_validator_blinded_blocks_modular::( slot, &Signature::infinity().unwrap().into(), None, @@ -2577,9 +2575,7 @@ impl ApiTester { self } - pub async fn test_blinded_block_production_verify_randao_invalid< - Payload: AbstractExecPayload, - >( + pub async fn test_blinded_block_production_verify_randao_invalid>( self, ) -> Self { let fork = self.chain.canonical_head.cached_head().head_fork(); @@ -2621,13 +2617,13 @@ impl ApiTester { // Check failure with full randao verification enabled. self.client - .get_validator_blinded_blocks::(slot, &bad_randao_reveal, None) + .get_validator_blinded_blocks::(slot, &bad_randao_reveal, None) .await .unwrap_err(); // Check failure with `skip_randao_verification` (requires infinity sig). self.client - .get_validator_blinded_blocks_modular::( + .get_validator_blinded_blocks_modular::( slot, &bad_randao_reveal, None, @@ -4600,7 +4596,7 @@ async fn block_production_verify_randao_invalid() { async fn blinded_block_production_full_payload_premerge() { ApiTester::new() .await - .test_blinded_block_production::>() + .test_blinded_block_production::() .await; } @@ -4609,7 +4605,7 @@ async fn blinded_block_production_with_skip_slots_full_payload_premerge() { ApiTester::new() .await .skip_slots(E::slots_per_epoch() * 2) - .test_blinded_block_production::>() + .test_blinded_block_production::() .await; } @@ -4617,7 +4613,7 @@ async fn blinded_block_production_with_skip_slots_full_payload_premerge() { async fn blinded_block_production_no_verify_randao_full_payload_premerge() { ApiTester::new() .await - .test_blinded_block_production_no_verify_randao::>() + .test_blinded_block_production_no_verify_randao::() .await; } @@ -4625,7 +4621,7 @@ async fn blinded_block_production_no_verify_randao_full_payload_premerge() { async fn blinded_block_production_verify_randao_invalid_full_payload_premerge() { ApiTester::new() .await - .test_blinded_block_production_verify_randao_invalid::>() + .test_blinded_block_production_verify_randao_invalid::() .await; } @@ -4633,7 +4629,7 @@ async fn blinded_block_production_verify_randao_invalid_full_payload_premerge() async fn blinded_block_production_blinded_payload_premerge() { ApiTester::new() .await - .test_blinded_block_production::>() + .test_blinded_block_production::() .await; } @@ -4642,7 +4638,7 @@ async fn blinded_block_production_with_skip_slots_blinded_payload_premerge() { ApiTester::new() .await .skip_slots(E::slots_per_epoch() * 2) - .test_blinded_block_production::>() + .test_blinded_block_production::() .await; } @@ -4650,7 +4646,7 @@ async fn blinded_block_production_with_skip_slots_blinded_payload_premerge() { async fn blinded_block_production_no_verify_randao_blinded_payload_premerge() { ApiTester::new() .await - .test_blinded_block_production_no_verify_randao::>() + .test_blinded_block_production_no_verify_randao::() .await; } @@ -4658,7 +4654,7 @@ async fn blinded_block_production_no_verify_randao_blinded_payload_premerge() { async fn blinded_block_production_verify_randao_invalid_blinded_payload_premerge() { ApiTester::new() .await - .test_blinded_block_production_verify_randao_invalid::>() + .test_blinded_block_production_verify_randao_invalid::() .await; } diff --git a/beacon_node/network/src/network_beacon_processor/tests.rs b/beacon_node/network/src/network_beacon_processor/tests.rs index d23d0bacc03..2c37d177aab 100644 --- a/beacon_node/network/src/network_beacon_processor/tests.rs +++ b/beacon_node/network/src/network_beacon_processor/tests.rs @@ -29,8 +29,8 @@ use std::time::Duration; use tokio::sync::mpsc; use types::blob_sidecar::FixedBlobSidecarList; use types::{ - Attestation, AttesterSlashing, BlobSidecar, Epoch, Hash256, MainnetEthSpec, ProposerSlashing, - SignedAggregateAndProof, SignedBeaconBlock, SignedSidecarList, SignedVoluntaryExit, Slot, + Attestation, AttesterSlashing, Epoch, Hash256, MainnetEthSpec, ProposerSlashing, + SignedAggregateAndProof, SignedBeaconBlock, SignedBlobSidecarList, SignedVoluntaryExit, Slot, SubnetId, }; @@ -52,7 +52,7 @@ const STANDARD_TIMEOUT: Duration = Duration::from_secs(10); struct TestRig { chain: Arc>, next_block: Arc>, - next_blobs: Option>>, + next_blobs: Option>, attestations: Vec<(Attestation, SubnetId)>, next_block_attestations: Vec<(Attestation, SubnetId)>, next_block_aggregate_attestations: Vec>, diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index e862cca1ee5..9e211fd098c 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -711,9 +711,9 @@ impl BeaconNodeHttpClient { } /// `POST v2/beacon/blocks` - pub async fn post_beacon_blocks_v2( + pub async fn post_beacon_blocks_v2>( &self, - block_contents: &SignedBlockContents, + block_contents: &SignedBlockContents, validation_level: Option, ) -> Result<(), Error> { self.post_generic_with_consensus_version( diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 096012fa6d5..7bcea6343b5 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -1351,23 +1351,6 @@ mod tests { } } -pub trait BlockProposal: Send { - type Payload: AbstractExecPayload; - type Sidecar: Sidecar; -} - -pub struct FullBlockProposal {} -impl BlockProposal for FullBlockProposal { - type Payload = FullPayload; - type Sidecar = BlobSidecar; -} - -pub struct BlindedBlockProposal {} -impl BlockProposal for BlindedBlockProposal { - type Payload = BlindedPayload; - type Sidecar = BlindedBlobSidecar; -} - /// A wrapper over a [`BeaconBlock`] or a [`BeaconBlockAndBlobSidecars`]. #[derive(Debug, Serialize, Deserialize)] #[serde(untagged)] @@ -1446,7 +1429,7 @@ pub type SignedBlockContentsTuple = ( ); /// A wrapper over a [`SignedBeaconBlock`] or a [`SignedBeaconBlockAndBlobSidecars`]. -#[derive(Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(untagged)] #[serde(bound = "T: EthSpec")] pub enum SignedBlockContents> { @@ -1548,14 +1531,14 @@ impl> From> #[derive(Debug, Clone, Serialize, Deserialize, Encode)] #[serde(bound = "T: EthSpec")] -pub struct SignedBeaconBlockAndBlobSidecars> { +pub struct SignedBeaconBlockAndBlobSidecars = FullBlockProposal> { pub signed_block: SignedBeaconBlock, pub signed_blob_sidecars: SignedSidecarList, } #[derive(Debug, Clone, Serialize, Deserialize, Encode)] #[serde(bound = "T: EthSpec, B: BlockProposal")] -pub struct BeaconBlockAndBlobSidecars> { +pub struct BeaconBlockAndBlobSidecars = FullBlockProposal> { pub block: BeaconBlock, pub blob_sidecars: SidecarList, } @@ -1592,7 +1575,7 @@ pub struct SignedBlindedBeaconBlockAndBlobSidecars< } #[derive(Debug, Clone, Serialize, Deserialize, Encode)] -#[serde(bound = "T: EthSpec")] +#[serde(bound = "T: EthSpec, B: BlockProposal")] pub struct BlindedBeaconBlockAndBlobSidecars = BlindedBlockProposal> { pub blinded_block: BeaconBlock, diff --git a/consensus/types/src/block_proposal.rs b/consensus/types/src/block_proposal.rs new file mode 100644 index 00000000000..c6808c6d296 --- /dev/null +++ b/consensus/types/src/block_proposal.rs @@ -0,0 +1,21 @@ +use crate::{ + AbstractExecPayload, BlindedBlobSidecar, BlindedPayload, BlobSidecar, EthSpec, FullPayload, + Sidecar, +}; + +pub trait BlockProposal: Send { + type Payload: AbstractExecPayload; + type Sidecar: Sidecar; +} + +pub struct FullBlockProposal {} +impl BlockProposal for FullBlockProposal { + type Payload = FullPayload; + type Sidecar = BlobSidecar; +} + +pub struct BlindedBlockProposal {} +impl BlockProposal for BlindedBlockProposal { + type Payload = BlindedPayload; + type Sidecar = BlindedBlobSidecar; +} diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 41acf993f97..1b9e80db3d1 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -98,6 +98,7 @@ pub mod slot_data; pub mod sqlite; pub mod blob_sidecar; +pub mod block_proposal; pub mod signed_blob; use ethereum_types::{H160, H256}; @@ -122,6 +123,7 @@ pub use crate::blob_sidecar::{ BlindedBlobSidecar, BlindedBlobSidecarList, BlobRoots, BlobSidecar, BlobSidecarList, Blobs, BlobsOrBlobRoots, Sidecar, SidecarList, SidecarListVariant, }; +pub use crate::block_proposal::{BlindedBlockProposal, BlockProposal, FullBlockProposal}; pub use crate::bls_to_execution_change::BlsToExecutionChange; pub use crate::chain_spec::{ChainSpec, Config, Domain}; pub use crate::checkpoint::Checkpoint; diff --git a/consensus/types/src/signed_blob.rs b/consensus/types/src/signed_blob.rs index 5c504353d21..bc81897438a 100644 --- a/consensus/types/src/signed_blob.rs +++ b/consensus/types/src/signed_blob.rs @@ -39,16 +39,6 @@ pub struct SignedSidecar> { pub _phantom: PhantomData, } -impl> SignedSidecar { - pub fn new(message: Arc, signature: Signature) -> SignedSidecar { - Self { - message, - signature, - _phantom: Default::default(), - } - } -} - /// List of Signed Sidecars that implements `Sidecar`. pub type SignedSidecarList = VariableList, ::MaxBlobsPerBlock>; @@ -56,7 +46,6 @@ pub type SignedBlobSidecarList = SignedSidecarList>; pub type SignedBlobSidecar = SignedSidecar>; -// TODO(jimmy): impl on SignedSidecar instead? impl SignedBlobSidecar { /// Verify `self.signature`. /// diff --git a/testing/web3signer_tests/src/lib.rs b/testing/web3signer_tests/src/lib.rs index dd17ae23b15..b0bbc0d6897 100644 --- a/testing/web3signer_tests/src/lib.rs +++ b/testing/web3signer_tests/src/lib.rs @@ -512,7 +512,7 @@ mod tests { let block = BeaconBlock::Base(BeaconBlockBase::empty(spec)); let block_slot = block.slot(); validator_store - .sign_block(pubkey, block, block_slot) + .sign_block::(pubkey, block, block_slot) .await .unwrap() }) @@ -576,7 +576,11 @@ mod tests { let mut altair_block = BeaconBlockAltair::empty(spec); altair_block.slot = altair_fork_slot; validator_store - .sign_block(pubkey, BeaconBlock::Altair(altair_block), altair_fork_slot) + .sign_block::( + pubkey, + BeaconBlock::Altair(altair_block), + altair_fork_slot, + ) .await .unwrap() }, @@ -661,7 +665,11 @@ mod tests { let mut merge_block = BeaconBlockMerge::empty(spec); merge_block.slot = merge_fork_slot; validator_store - .sign_block(pubkey, BeaconBlock::Merge(merge_block), merge_fork_slot) + .sign_block::( + pubkey, + BeaconBlock::Merge(merge_block), + merge_fork_slot, + ) .await .unwrap() }) diff --git a/validator_client/src/block_service.rs b/validator_client/src/block_service.rs index 15bff9b8db0..6d878ad3240 100644 --- a/validator_client/src/block_service.rs +++ b/validator_client/src/block_service.rs @@ -8,9 +8,7 @@ use crate::{ use crate::{http_metrics::metrics, validator_store::ValidatorStore}; use bls::SignatureBytes; use environment::RuntimeContext; -use eth2::types::{ - BlindedBlockProposal, BlockContents, BlockProposal, FullBlockProposal, SignedBlockContents, -}; +use eth2::types::{BlockContents, SignedBlockContents}; use eth2::BeaconNodeHttpClient; use slog::{crit, debug, error, info, trace, warn, Logger}; use slot_clock::SlotClock; @@ -21,7 +19,10 @@ use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc; use tokio::time::sleep; -use types::{BlockType, EthSpec, ExecPayload, Graffiti, PublicKeyBytes, Slot}; +use types::{ + BlindedBlockProposal, BlockProposal, BlockType, EthSpec, ExecPayload, FullBlockProposal, + Graffiti, PublicKeyBytes, Slot, +}; #[derive(Debug)] pub enum BlockError { diff --git a/validator_client/src/signing_method.rs b/validator_client/src/signing_method.rs index 64a356224e4..660b6302657 100644 --- a/validator_client/src/signing_method.rs +++ b/validator_client/src/signing_method.rs @@ -4,7 +4,6 @@ //! - Via a remote signer (Web3Signer) use crate::http_metrics::metrics; -use eth2::types::{BlockProposal, FullBlockProposal}; use eth2_keystore::Keystore; use lockfile::Lockfile; use parking_lot::Mutex; diff --git a/validator_client/src/validator_store.rs b/validator_client/src/validator_store.rs index bfa77c7df3d..69a2a3f2d81 100644 --- a/validator_client/src/validator_store.rs +++ b/validator_client/src/validator_store.rs @@ -6,7 +6,7 @@ use crate::{ Config, }; use account_utils::{validator_definitions::ValidatorDefinition, ZeroizeString}; -use eth2::types::{BlindedBlockProposal, BlockProposal, VariableList}; +use eth2::types::VariableList; use parking_lot::{Mutex, RwLock}; use slashing_protection::{ interchange::Interchange, InterchangeError, NotSafe, Safe, SlashingDatabase, @@ -20,12 +20,13 @@ use std::sync::Arc; use task_executor::TaskExecutor; use types::{ attestation::Error as AttestationError, graffiti::GraffitiString, Address, AggregateAndProof, - Attestation, BeaconBlock, ChainSpec, ContributionAndProof, Domain, Epoch, EthSpec, Fork, - Graffiti, Hash256, Keypair, PublicKeyBytes, SelectionProof, Sidecar, SidecarList, Signature, - SignedAggregateAndProof, SignedBeaconBlock, SignedContributionAndProof, SignedRoot, - SignedSidecar, SignedSidecarList, SignedValidatorRegistrationData, SignedVoluntaryExit, Slot, - SyncAggregatorSelectionData, SyncCommitteeContribution, SyncCommitteeMessage, - SyncSelectionProof, SyncSubnetId, ValidatorRegistrationData, VoluntaryExit, + Attestation, BeaconBlock, BlindedBlockProposal, BlockProposal, ChainSpec, ContributionAndProof, + Domain, Epoch, EthSpec, Fork, Graffiti, Hash256, Keypair, PublicKeyBytes, SelectionProof, + Sidecar, SidecarList, Signature, SignedAggregateAndProof, SignedBeaconBlock, + SignedContributionAndProof, SignedRoot, SignedSidecar, SignedSidecarList, + SignedValidatorRegistrationData, SignedVoluntaryExit, Slot, SyncAggregatorSelectionData, + SyncCommitteeContribution, SyncCommitteeMessage, SyncSelectionProof, SyncSubnetId, + ValidatorRegistrationData, VoluntaryExit, }; use validator_dir::ValidatorDir; @@ -564,7 +565,11 @@ impl ValidatorStore { metrics::inc_counter_vec(&metrics::SIGNED_BLOBS_TOTAL, &[metrics::SUCCESS]); - signed_blob_sidecars.push(SignedSidecar::new(blob_sidecar, signature)); + signed_blob_sidecars.push(SignedSidecar { + message: blob_sidecar, + signature, + _phantom: PhantomData, + }); } Ok(VariableList::from(signed_blob_sidecars)) From 80d3b2dd2985f28102c9380684e4ba0c393c90e7 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Fri, 28 Jul 2023 05:59:02 +1000 Subject: [PATCH 11/24] Hanlde unblinded blobs and publish full block contents --- Cargo.lock | 1 + beacon_node/builder_client/src/lib.rs | 6 +- beacon_node/execution_layer/src/lib.rs | 9 +- beacon_node/http_api/src/publish_blocks.rs | 32 +++--- common/eth2/Cargo.toml | 1 + common/eth2/src/lib.rs | 4 +- common/eth2/src/types.rs | 122 ++++++++++++++++++++- consensus/types/src/signed_blob.rs | 24 +++- validator_client/src/block_service.rs | 19 ++-- 9 files changed, 177 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 667ae84e793..89e43dc860d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2452,6 +2452,7 @@ dependencies = [ "serde", "serde_json", "slashing_protection", + "ssz_types", "store", "tokio", "types", diff --git a/beacon_node/builder_client/src/lib.rs b/beacon_node/builder_client/src/lib.rs index 39d02ea20ad..fea5353f1b1 100644 --- a/beacon_node/builder_client/src/lib.rs +++ b/beacon_node/builder_client/src/lib.rs @@ -1,7 +1,7 @@ use eth2::types::builder_bid::SignedBuilderBid; use eth2::types::{ - AbstractExecPayload, BlindedBlockProposal, EthSpec, ExecutionBlockHash, ExecutionPayload, - ForkVersionedResponse, PublicKeyBytes, SignedBlockContents, SignedValidatorRegistrationData, + AbstractExecPayload, BlindedBlockProposal, EthSpec, ExecutionBlockHash, ForkVersionedResponse, + FullPayloadContents, PublicKeyBytes, SignedBlockContents, SignedValidatorRegistrationData, Slot, }; pub use eth2::Error; @@ -141,7 +141,7 @@ impl BuilderHttpClient { pub async fn post_builder_blinded_blocks( &self, blinded_block: &SignedBlockContents, - ) -> Result>, Error> { + ) -> Result>, Error> { let mut path = self.server.full.clone(); path.path_segments_mut() diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index ba1519ac7d5..1da8ba5fe6e 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -13,8 +13,8 @@ pub use engine_api::*; pub use engine_api::{http, http::deposit_methods, http::HttpJsonRpc}; use engines::{Engine, EngineError}; pub use engines::{EngineState, ForkchoiceState}; -use eth2::types::SignedBlockContents; use eth2::types::{builder_bid::SignedBuilderBid, ForkVersionedResponse}; +use eth2::types::{FullPayloadContents, SignedBlockContents}; use ethers_core::abi::ethereum_types::FromStrRadixErr; use ethers_core::types::Transaction as EthersTransaction; use fork_choice::ForkchoiceUpdateParameters; @@ -1090,7 +1090,7 @@ impl ExecutionLayer { payload_attributes, forkchoice_update_params, current_fork, - Self::cache_payload, + Self::cache_payload, // TODO(jimmy): cache blobs as well? ) .await } @@ -1863,7 +1863,7 @@ impl ExecutionLayer { &self, block_root: Hash256, block: &SignedBlockContents, - ) -> Result, Error> { + ) -> Result, Error> { debug!( self.log(), "Sending block to builder"; @@ -1882,11 +1882,12 @@ impl ExecutionLayer { .await; match &payload_result { - Ok(payload) => { + Ok(unblinded_response) => { metrics::inc_counter_vec( &metrics::EXECUTION_LAYER_BUILDER_REVEAL_PAYLOAD_OUTCOME, &[metrics::SUCCESS], ); + let payload = unblinded_response.payload_ref(); info!( self.log(), "Builder successfully revealed payload"; diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index b8dd4971e97..2e580b5a4b2 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -6,8 +6,8 @@ use beacon_chain::{ AvailabilityProcessingStatus, BeaconChain, BeaconChainError, BeaconChainTypes, BlockError, IntoGossipVerifiedBlockContents, NotifyExecutionLayer, }; -use eth2::types::SignedBlockContents; use eth2::types::{BroadcastValidation, FullBlockProposal}; +use eth2::types::{FullPayloadContents, SignedBlockContents}; use execution_layer::ProvenancedPayload; use lighthouse_network::PubsubMessage; use network::NetworkMessage; @@ -274,7 +274,7 @@ pub async fn publish_blinded_block( validation_level: BroadcastValidation, ) -> Result<(), Rejection> { let block_root = block_contents.signed_block().canonical_root(); - let full_block: ProvenancedBlock> = + let full_block = reconstruct_block(chain.clone(), block_root, block_contents, log.clone()).await?; publish_block::( Some(block_root), @@ -303,7 +303,7 @@ pub async fn reconstruct_block( })?; // If the execution block hash is zero, use an empty payload. - let full_payload = if payload_header.block_hash() == ExecutionBlockHash::zero() { + let full_payload_contents = if payload_header.block_hash() == ExecutionBlockHash::zero() { let payload = FullPayload::default_at_fork( chain .spec @@ -315,13 +315,14 @@ pub async fn reconstruct_block( )) })? .into(); - ProvenancedPayload::Local(payload) + ProvenancedPayload::Local(FullPayloadContents::Payload(payload)) // If we already have an execution payload with this transactions root cached, use it. + // TODO(jimmy): blobs should be cached as well } else if let Some(cached_payload) = el.get_payload_by_root(&payload_header.tree_hash_root()) { info!(log, "Reconstructing a full block using a local payload"; "block_hash" => ?cached_payload.block_hash()); - ProvenancedPayload::Local(cached_payload) + ProvenancedPayload::Local(FullPayloadContents::Payload(cached_payload)) // Otherwise, this means we are attempting a blind block proposal. } else { // Perform the logging for late blocks when we publish to the @@ -338,7 +339,6 @@ pub async fn reconstruct_block( &log, ); - // TODO(jimmy): handle blinded blobs bundle response let full_payload = el .propose_blinded_beacon_block(block_root, &block_contents) .await @@ -352,28 +352,22 @@ pub async fn reconstruct_block( ProvenancedPayload::Builder(full_payload) }; - Some(full_payload) + Some(full_payload_contents) } else { None }; - // TODO(jimmy) convert full `SignedBlockSidecars` - let (block, _maybe_blobs) = block_contents.deconstruct(); - match full_payload_opt { // A block without a payload is pre-merge and we consider it locally // built. - None => block - .try_into_full_block(None) - .map(SignedBlockContents::Block) + None => block_contents + .try_into_full_block_and_blobs(None) .map(ProvenancedBlock::local), - Some(ProvenancedPayload::Local(full_payload)) => block - .try_into_full_block(Some(full_payload)) - .map(SignedBlockContents::Block) + Some(ProvenancedPayload::Local(full_payload_contents)) => block_contents + .try_into_full_block_and_blobs(Some(full_payload_contents)) .map(ProvenancedBlock::local), - Some(ProvenancedPayload::Builder(full_payload)) => block - .try_into_full_block(Some(full_payload)) - .map(SignedBlockContents::Block) + Some(ProvenancedPayload::Builder(full_payload_contents)) => block_contents + .try_into_full_block_and_blobs(Some(full_payload_contents)) .map(ProvenancedBlock::builder), } .ok_or_else(|| { diff --git a/common/eth2/Cargo.toml b/common/eth2/Cargo.toml index d15c56d9b05..d0cb62da29c 100644 --- a/common/eth2/Cargo.toml +++ b/common/eth2/Cargo.toml @@ -9,6 +9,7 @@ edition = "2021" [dependencies] serde = { version = "1.0.116", features = ["derive"] } serde_json = "1.0.58" +ssz_types = "0.5.4" types = { path = "../../consensus/types" } reqwest = { version = "0.11.0", features = ["json", "stream"] } lighthouse_network = { path = "../../beacon_node/lighthouse_network" } diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 9e211fd098c..f214852818e 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -657,9 +657,9 @@ impl BeaconNodeHttpClient { /// `POST beacon/blinded_blocks` /// /// Returns `Ok(None)` on a 404 error. - pub async fn post_beacon_blinded_blocks>( + pub async fn post_beacon_blinded_blocks>( &self, - block: &SignedBeaconBlock, + block: &SignedBlockContents, ) -> Result<(), Error> { let mut path = self.eth_path(V1)?; diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 7bcea6343b5..5206fa89cec 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -4,12 +4,14 @@ use crate::Error as ServerError; use lighthouse_network::{ConnectionDirection, Enr, Multiaddr, PeerConnectionStatus}; use mediatype::{names, MediaType, MediaTypeList}; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; +use serde_json::Value; use ssz_derive::Encode; use std::convert::TryFrom; use std::fmt::{self, Display}; use std::str::{from_utf8, FromStr}; use std::time::Duration; +use types::beacon_block_body::KzgCommitments; pub use types::*; #[cfg(feature = "lighthouse")] @@ -1490,6 +1492,56 @@ impl> SignedBlockContents { } } +impl SignedBlockContents { + pub fn try_into_full_block_and_blobs( + self, + maybe_full_payload_contents: Option>, + ) -> Option> { + match self { + SignedBlockContents::BlindedBlockAndBlobSidecars(blinded_block_and_blob_sidecars) => { + maybe_full_payload_contents.and_then(|full_payload_contents| { + match full_payload_contents.deconstruct() { + (full_payload, Some(blobs_bundle)) => { + let maybe_full_block = blinded_block_and_blob_sidecars + .signed_blinded_block + .try_into_full_block(Some(full_payload)); + let full_blob_sidecars: Vec<_> = blinded_block_and_blob_sidecars + .signed_blinded_blob_sidecars + .into_iter() + .zip(blobs_bundle.blobs) + .map(|(blinded_blob_sidecar, blob)| { + blinded_blob_sidecar.into_full_blob_sidecars(blob) + }) + .collect(); + + maybe_full_block.map(|signed_block| { + SignedBlockContents::BlockAndBlobSidecars( + SignedBeaconBlockAndBlobSidecars { + signed_block, + signed_blob_sidecars: VariableList::from( + full_blob_sidecars, + ), + }, + ) + }) + } + // Can't build full block contents without full blobs + _ => None, + } + }) + } + SignedBlockContents::Block(blinded_block) => { + let full_payload_opt = maybe_full_payload_contents.map(|o| o.deconstruct().0); + blinded_block + .try_into_full_block(full_payload_opt) + .map(SignedBlockContents::Block) + } + // Unexpected scenario for blinded block proposal + SignedBlockContents::BlockAndBlobSidecars(_) => None, + } + } +} + impl> From> for SignedBlockContents { @@ -1607,3 +1659,71 @@ impl> ForkVersionDeserialize }) } } + +#[derive(Debug, PartialEq, Deserialize)] +#[serde(untagged)] +#[serde(bound = "E: EthSpec")] +pub enum FullPayloadContents { + Payload(ExecutionPayload), + PayloadAndBlobs(ExecutionPayloadAndBlobs), +} + +impl FullPayloadContents { + pub fn payload_ref(&self) -> &ExecutionPayload { + match self { + FullPayloadContents::Payload(payload) => payload, + FullPayloadContents::PayloadAndBlobs(payload_and_blobs) => { + &payload_and_blobs.execution_payload + } + } + } + + pub fn block_hash(&self) -> ExecutionBlockHash { + self.payload_ref().block_hash() + } + + pub fn deconstruct(self) -> (ExecutionPayload, Option>) { + match self { + FullPayloadContents::Payload(payload) => (payload, None), + FullPayloadContents::PayloadAndBlobs(payload_and_blobs) => ( + payload_and_blobs.execution_payload, + Some(payload_and_blobs.blobs_bundle), + ), + } + } +} + +impl ForkVersionDeserialize for FullPayloadContents { + fn deserialize_by_fork<'de, D: Deserializer<'de>>( + value: Value, + fork_name: ForkName, + ) -> Result { + match fork_name { + ForkName::Merge | ForkName::Capella => serde_json::from_value(value) + .map(Self::Payload) + .map_err(serde::de::Error::custom), + ForkName::Deneb => serde_json::from_value(value) + .map(Self::PayloadAndBlobs) + .map_err(serde::de::Error::custom), + ForkName::Base | ForkName::Altair => Err(serde::de::Error::custom(format!( + "FullPayloadContents deserialization for {fork_name} not implemented" + ))), + } + } +} + +#[derive(Debug, PartialEq, Deserialize)] +#[serde(bound = "E: EthSpec")] +pub struct ExecutionPayloadAndBlobs { + pub execution_payload: ExecutionPayload, + pub blobs_bundle: BlobsBundle, +} + +#[derive(Debug, PartialEq, Deserialize)] +#[serde(bound = "E: EthSpec")] +pub struct BlobsBundle { + pub commitments: KzgCommitments, + pub proofs: KzgProofs, + #[serde(with = "ssz_types::serde_utils::list_of_hex_fixed_vec")] + pub blobs: Blobs, +} diff --git a/consensus/types/src/signed_blob.rs b/consensus/types/src/signed_blob.rs index bc81897438a..7d1c553d923 100644 --- a/consensus/types/src/signed_blob.rs +++ b/consensus/types/src/signed_blob.rs @@ -1,6 +1,6 @@ use crate::{ - test_utils::TestRandom, BlobSidecar, ChainSpec, Domain, EthSpec, Fork, Hash256, Sidecar, - Signature, SignedRoot, SigningData, + test_utils::TestRandom, BlindedBlobSidecar, Blob, BlobSidecar, ChainSpec, Domain, EthSpec, + Fork, Hash256, Sidecar, Signature, SignedRoot, SigningData, }; use bls::PublicKey; use derivative::Derivative; @@ -39,6 +39,26 @@ pub struct SignedSidecar> { pub _phantom: PhantomData, } +impl SignedSidecar { + pub fn into_full_blob_sidecars(self, blob: Blob) -> SignedSidecar> { + let blinded_sidecar = self.message; + SignedSidecar { + message: Arc::new(BlobSidecar { + block_root: blinded_sidecar.block_root, + index: blinded_sidecar.index, + slot: blinded_sidecar.slot, + block_parent_root: blinded_sidecar.block_parent_root, + proposer_index: blinded_sidecar.proposer_index, + blob, + kzg_commitment: blinded_sidecar.kzg_commitment, + kzg_proof: blinded_sidecar.kzg_proof, + }), + signature: self.signature, + _phantom: PhantomData, + } + } +} + /// List of Signed Sidecars that implements `Sidecar`. pub type SignedSidecarList = VariableList, ::MaxBlobsPerBlock>; diff --git a/validator_client/src/block_service.rs b/validator_client/src/block_service.rs index 6d878ad3240..788275232fd 100644 --- a/validator_client/src/block_service.rs +++ b/validator_client/src/block_service.rs @@ -563,16 +563,15 @@ impl BlockService { &metrics::BLOCK_SERVICE_TIMES, &[metrics::BLINDED_BEACON_BLOCK_HTTP_POST], ); - todo!("need to be adjusted for blobs"); - // beacon_node - // .post_beacon_blinded_blocks(signed_block_contents.signed_block()) - // .await - // .map_err(|e| { - // BlockError::Irrecoverable(format!( - // "Error from beacon node when publishing block: {:?}", - // e - // )) - // })? + beacon_node + .post_beacon_blinded_blocks(signed_block_contents) + .await + .map_err(|e| { + BlockError::Irrecoverable(format!( + "Error from beacon node when publishing block: {:?}", + e + )) + })? } } Ok::<_, BlockError>(()) From b040d5b9ed58a4ca47faa8f9d3b5c2be37c2cd8b Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Fri, 28 Jul 2023 07:40:24 +1000 Subject: [PATCH 12/24] Fix tests --- .../tests/broadcast_validation_tests.rs | 44 ++++++--- .../http_api/tests/interactive_tests.rs | 2 +- beacon_node/http_api/tests/tests.rs | 92 +++++++++++++------ common/eth2/src/types.rs | 21 +++-- consensus/types/src/block_proposal.rs | 18 +++- 5 files changed, 127 insertions(+), 50 deletions(-) diff --git a/beacon_node/http_api/tests/broadcast_validation_tests.rs b/beacon_node/http_api/tests/broadcast_validation_tests.rs index b525c23a5f3..ccc8d32ca97 100644 --- a/beacon_node/http_api/tests/broadcast_validation_tests.rs +++ b/beacon_node/http_api/tests/broadcast_validation_tests.rs @@ -9,7 +9,7 @@ use http_api::test_utils::InteractiveTester; use http_api::{publish_blinded_block, publish_block, reconstruct_block, ProvenancedBlock}; use std::sync::Arc; use tree_hash::TreeHash; -use types::{Hash256, MainnetEthSpec, Slot}; +use types::{BlindedBlockProposal, FullBlockProposal, Hash256, MainnetEthSpec, Slot}; use warp::Rejection; use warp_utils::reject::CustomBadRequest; @@ -76,7 +76,10 @@ pub async fn gossip_invalid() { let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2(&SignedBlockContents::new(block, blobs), validation_level) + .post_beacon_blocks_v2( + &SignedBlockContents::::new(block, blobs), + validation_level, + ) .await; assert!(response.is_err()); @@ -127,7 +130,10 @@ pub async fn gossip_partial_pass() { let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2(&SignedBlockContents::new(block, blobs), validation_level) + .post_beacon_blocks_v2( + &SignedBlockContents::::new(block, blobs), + validation_level, + ) .await; assert!(response.is_err()); @@ -170,7 +176,7 @@ pub async fn gossip_full_pass() { let response: Result<(), eth2::Error> = tester .client .post_beacon_blocks_v2( - &SignedBlockContents::new(block.clone(), blobs), + &SignedBlockContents::::new(block.clone(), blobs), validation_level, ) .await; @@ -219,7 +225,10 @@ pub async fn consensus_invalid() { let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2(&SignedBlockContents::new(block, blobs), validation_level) + .post_beacon_blocks_v2( + &SignedBlockContents::::new(block, blobs), + validation_level, + ) .await; assert!(response.is_err()); @@ -268,7 +277,10 @@ pub async fn consensus_gossip() { let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2(&SignedBlockContents::new(block, blobs), validation_level) + .post_beacon_blocks_v2( + &SignedBlockContents::::new(block, blobs), + validation_level, + ) .await; assert!(response.is_err()); @@ -382,7 +394,7 @@ pub async fn consensus_full_pass() { let response: Result<(), eth2::Error> = tester .client .post_beacon_blocks_v2( - &SignedBlockContents::new(block.clone(), blobs), + &SignedBlockContents::::new(block.clone(), blobs), validation_level, ) .await; @@ -433,7 +445,10 @@ pub async fn equivocation_invalid() { let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2(&SignedBlockContents::new(block, blobs), validation_level) + .post_beacon_blocks_v2( + &SignedBlockContents::::new(block, blobs), + validation_level, + ) .await; assert!(response.is_err()); @@ -490,7 +505,7 @@ pub async fn equivocation_consensus_early_equivocation() { assert!(tester .client .post_beacon_blocks_v2( - &SignedBlockContents::new(block_a.clone(), blobs_a), + &SignedBlockContents::::new(block_a.clone(), blobs_a), validation_level ) .await @@ -504,7 +519,7 @@ pub async fn equivocation_consensus_early_equivocation() { let response: Result<(), eth2::Error> = tester .client .post_beacon_blocks_v2( - &SignedBlockContents::new(block_b.clone(), blobs_b), + &SignedBlockContents::::new(block_b.clone(), blobs_b), validation_level, ) .await; @@ -555,7 +570,10 @@ pub async fn equivocation_gossip() { let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2(&SignedBlockContents::new(block, blobs), validation_level) + .post_beacon_blocks_v2( + &SignedBlockContents::::new(block, blobs), + validation_level, + ) .await; assert!(response.is_err()); @@ -679,7 +697,7 @@ pub async fn equivocation_full_pass() { let response: Result<(), eth2::Error> = tester .client .post_beacon_blocks_v2( - &SignedBlockContents::new(block.clone(), blobs), + &SignedBlockContents::::new(block.clone(), blobs), validation_level, ) .await; @@ -1293,7 +1311,7 @@ pub async fn blinded_equivocation_full_pass() { let response: Result<(), eth2::Error> = tester .client .post_beacon_blocks_v2( - &SignedBlockContents::new(block.clone(), blobs), + &SignedBlockContents::::new(block.clone(), blobs), validation_level, ) .await; diff --git a/beacon_node/http_api/tests/interactive_tests.rs b/beacon_node/http_api/tests/interactive_tests.rs index 63f46471d7f..a42b6b2ae34 100644 --- a/beacon_node/http_api/tests/interactive_tests.rs +++ b/beacon_node/http_api/tests/interactive_tests.rs @@ -619,7 +619,7 @@ pub async fn proposer_boost_re_org_test( .into(); let unsigned_block_contents_c = tester .client - .get_validator_blocks(slot_c, &randao_reveal, None) + .get_validator_blocks::(slot_c, &randao_reveal, None) .await .unwrap() .data; diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 7d31b5a3bf9..4fda931ae4d 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -1266,7 +1266,7 @@ impl ApiTester { assert!(self .client - .post_beacon_blocks(&SignedBlockContents::from(block)) + .post_beacon_blocks(&SignedBlockContents::::from(block)) .await .is_err()); @@ -2393,7 +2393,8 @@ impl ApiTester { .0; let signed_block = block.sign(&sk, &fork, genesis_validators_root, &self.chain.spec); - let signed_block_contents = SignedBlockContents::from(signed_block.clone()); + let signed_block_contents = + SignedBlockContents::::from(signed_block.clone()); self.client .post_beacon_blocks(&signed_block_contents) @@ -2536,12 +2537,15 @@ impl ApiTester { .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() - .data; + .data + .deconstruct() + .0; let signed_block = block.sign(&sk, &fork, genesis_validators_root, &self.chain.spec); + let signed_block_contents = SignedBlockContents::::Block(signed_block.clone()); self.client - .post_beacon_blinded_blocks(&signed_block) + .post_beacon_blinded_blocks(&signed_block_contents) .await .unwrap(); @@ -2557,7 +2561,7 @@ impl ApiTester { for _ in 0..E::slots_per_epoch() { let slot = self.chain.slot().unwrap(); - let block = self + let block_contents = self .client .get_validator_blinded_blocks_modular::( slot, @@ -2568,7 +2572,7 @@ impl ApiTester { .await .unwrap() .data; - assert_eq!(block.slot(), slot); + assert_eq!(block_contents.block().slot(), slot); self.chain.slot_clock.set_slot(slot.as_u64() + 1); } @@ -3035,10 +3039,11 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::>(slot, &randao_reveal, None) + .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3075,10 +3080,11 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::>(slot, &randao_reveal, None) + .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3118,10 +3124,11 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::>(slot, &randao_reveal, None) + .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3167,10 +3174,11 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::>(slot, &randao_reveal, None) + .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3215,10 +3223,11 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::>(slot, &randao_reveal, None) + .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3262,10 +3271,11 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::>(slot, &randao_reveal, None) + .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3308,10 +3318,11 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::>(slot, &randao_reveal, None) + .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3344,10 +3355,11 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::>(slot, &randao_reveal, None) + .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3381,10 +3393,11 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::>(slot, &randao_reveal, None) + .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3424,10 +3437,15 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::>(next_slot, &randao_reveal, None) + .get_validator_blinded_blocks::( + next_slot, + &randao_reveal, + None, + ) .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3453,10 +3471,15 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::>(next_slot, &randao_reveal, None) + .get_validator_blinded_blocks::( + next_slot, + &randao_reveal, + None, + ) .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3502,10 +3525,15 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::>(next_slot, &randao_reveal, None) + .get_validator_blinded_blocks::( + next_slot, + &randao_reveal, + None, + ) .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3541,10 +3569,15 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::>(next_slot, &randao_reveal, None) + .get_validator_blinded_blocks::( + next_slot, + &randao_reveal, + None, + ) .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3584,10 +3617,11 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::>(slot, &randao_reveal, None) + .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3625,10 +3659,11 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::>(slot, &randao_reveal, None) + .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3662,10 +3697,11 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::>(slot, &randao_reveal, None) + .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3699,10 +3735,11 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::>(slot, &randao_reveal, None) + .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3736,10 +3773,11 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::>(slot, &randao_reveal, None) + .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3786,10 +3824,11 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::>(slot, &randao_reveal, None) + .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3828,10 +3867,11 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::>(slot, &randao_reveal, None) + .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 5206fa89cec..6a8c5378275 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -1445,13 +1445,20 @@ impl> SignedBlockContents { block: SignedBeaconBlock, blobs: Option>, ) -> Self { - if let Some(blobs) = blobs { - Self::BlockAndBlobSidecars(SignedBeaconBlockAndBlobSidecars { - signed_block: block, - signed_blob_sidecars: blobs, - }) - } else { - Self::Block(block) + match (B::block_type(), blobs) { + (BlockType::Blinded, Some(blobs)) => { + Self::BlockAndBlobSidecars(SignedBeaconBlockAndBlobSidecars { + signed_block: block, + signed_blob_sidecars: blobs, + }) + } + (BlockType::Full, Some(blobs)) => { + Self::BlindedBlockAndBlobSidecars(SignedBlindedBeaconBlockAndBlobSidecars { + signed_blinded_block: block, + signed_blinded_blob_sidecars: blobs, + }) + } + (_, None) => Self::Block(block), } } diff --git a/consensus/types/src/block_proposal.rs b/consensus/types/src/block_proposal.rs index c6808c6d296..4fec7eabfa4 100644 --- a/consensus/types/src/block_proposal.rs +++ b/consensus/types/src/block_proposal.rs @@ -1,21 +1,33 @@ use crate::{ - AbstractExecPayload, BlindedBlobSidecar, BlindedPayload, BlobSidecar, EthSpec, FullPayload, - Sidecar, + AbstractExecPayload, BlindedBlobSidecar, BlindedPayload, BlobSidecar, BlockType, EthSpec, + FullPayload, Sidecar, }; +use std::fmt::Debug; -pub trait BlockProposal: Send { +pub trait BlockProposal: Send + Sized + Clone + Debug { type Payload: AbstractExecPayload; type Sidecar: Sidecar; + fn block_type() -> BlockType; } +#[derive(Clone, Debug)] pub struct FullBlockProposal {} impl BlockProposal for FullBlockProposal { type Payload = FullPayload; type Sidecar = BlobSidecar; + + fn block_type() -> BlockType { + BlockType::Full + } } +#[derive(Clone, Debug)] pub struct BlindedBlockProposal {} impl BlockProposal for BlindedBlockProposal { type Payload = BlindedPayload; type Sidecar = BlindedBlobSidecar; + + fn block_type() -> BlockType { + BlockType::Blinded + } } From b344415db65512f8f7fa5b6ba8f46617e4f14e17 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Mon, 31 Jul 2023 23:24:54 +1000 Subject: [PATCH 13/24] Add local EL blobs caching in blinded flow --- beacon_node/builder_client/src/lib.rs | 4 +- beacon_node/execution_layer/src/lib.rs | 39 +++++++-- .../execution_layer/src/payload_cache.rs | 13 +-- .../src/test_utils/mock_execution_layer.rs | 2 +- beacon_node/http_api/src/publish_blocks.rs | 3 +- common/eth2/src/types.rs | 72 +--------------- consensus/types/src/lib.rs | 5 +- consensus/types/src/payload.rs | 85 ++++++++++++++++++- 8 files changed, 129 insertions(+), 94 deletions(-) diff --git a/beacon_node/builder_client/src/lib.rs b/beacon_node/builder_client/src/lib.rs index fea5353f1b1..7fbbb86bb70 100644 --- a/beacon_node/builder_client/src/lib.rs +++ b/beacon_node/builder_client/src/lib.rs @@ -1,8 +1,8 @@ use eth2::types::builder_bid::SignedBuilderBid; +use eth2::types::payload::FullPayloadContents; use eth2::types::{ AbstractExecPayload, BlindedBlockProposal, EthSpec, ExecutionBlockHash, ForkVersionedResponse, - FullPayloadContents, PublicKeyBytes, SignedBlockContents, SignedValidatorRegistrationData, - Slot, + PublicKeyBytes, SignedBlockContents, SignedValidatorRegistrationData, Slot, }; pub use eth2::Error; use eth2::{ok_or_error, StatusCode}; diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 1da8ba5fe6e..3aa18b1282c 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -13,7 +13,7 @@ pub use engine_api::*; pub use engine_api::{http, http::deposit_methods, http::HttpJsonRpc}; use engines::{Engine, EngineError}; pub use engines::{EngineState, ForkchoiceState}; -use eth2::types::{builder_bid::SignedBuilderBid, ForkVersionedResponse}; +use eth2::types::{builder_bid::SignedBuilderBid, BlobsBundle, ForkVersionedResponse}; use eth2::types::{FullPayloadContents, SignedBlockContents}; use ethers_core::abi::ethereum_types::FromStrRadixErr; use ethers_core::types::Transaction as EthersTransaction; @@ -289,6 +289,8 @@ pub enum FailedCondition { EpochsSinceFinalization, } +type PayloadContentsRefTuple<'a, T> = (ExecutionPayloadRef<'a, T>, Option<&'a BlobsBundleV1>); + struct Inner { engine: Arc, builder: Option, @@ -454,12 +456,28 @@ impl ExecutionLayer { } /// Cache a full payload, keyed on the `tree_hash_root` of the payload - fn cache_payload(&self, payload: ExecutionPayloadRef) -> Option> { - self.inner.payload_cache.put(payload.clone_from_ref()) + fn cache_payload( + &self, + payload_and_blobs: PayloadContentsRefTuple, + ) -> Option> { + let (payload_ref, maybe_json_blobs_bundle) = payload_and_blobs; + + let payload = payload_ref.clone_from_ref(); + let maybe_blobs_bundle = maybe_json_blobs_bundle + .cloned() + .map(|blobs_bundle| BlobsBundle { + commitments: blobs_bundle.commitments, + proofs: blobs_bundle.proofs, + blobs: blobs_bundle.blobs, + }); + + self.inner + .payload_cache + .put(FullPayloadContents::new(payload, maybe_blobs_bundle)) } /// Attempt to retrieve a full payload from the payload cache by the payload root - pub fn get_payload_by_root(&self, root: &Hash256) -> Option> { + pub fn get_payload_by_root(&self, root: &Hash256) -> Option> { self.inner.payload_cache.get(root) } @@ -1090,7 +1108,7 @@ impl ExecutionLayer { payload_attributes, forkchoice_update_params, current_fork, - Self::cache_payload, // TODO(jimmy): cache blobs as well? + Self::cache_payload, ) .await } @@ -1101,7 +1119,10 @@ impl ExecutionLayer { payload_attributes: &PayloadAttributes, forkchoice_update_params: ForkchoiceUpdateParameters, current_fork: ForkName, - f: fn(&ExecutionLayer, ExecutionPayloadRef) -> Option>, + cache_fn: fn( + &ExecutionLayer, + PayloadContentsRefTuple, + ) -> Option>, ) -> Result, Error> { self.engine() .request(move |engine| async move { @@ -1180,7 +1201,7 @@ impl ExecutionLayer { "suggested_fee_recipient" => ?payload_attributes.suggested_fee_recipient(), ); } - if f(self, payload_response.execution_payload_ref()).is_some() { + if cache_fn(self, (payload_response.execution_payload_ref(), payload_response.blobs_bundle().ok())).is_some() { warn!( self.log(), "Duplicate payload cached, this might indicate redundant proposal \ @@ -2202,8 +2223,8 @@ fn ethers_tx_to_ssz( fn noop( _: &ExecutionLayer, - _: ExecutionPayloadRef, -) -> Option> { + _: PayloadContentsRefTuple, +) -> Option> { None } diff --git a/beacon_node/execution_layer/src/payload_cache.rs b/beacon_node/execution_layer/src/payload_cache.rs index 1722edff465..1155b1ca3a4 100644 --- a/beacon_node/execution_layer/src/payload_cache.rs +++ b/beacon_node/execution_layer/src/payload_cache.rs @@ -1,13 +1,14 @@ +use eth2::types::FullPayloadContents; use lru::LruCache; use parking_lot::Mutex; use tree_hash::TreeHash; -use types::{EthSpec, ExecutionPayload, Hash256}; +use types::{EthSpec, Hash256}; pub const DEFAULT_PAYLOAD_CACHE_SIZE: usize = 10; /// A cache mapping execution payloads by tree hash roots. pub struct PayloadCache { - payloads: Mutex>>, + payloads: Mutex>>, } #[derive(Hash, PartialEq, Eq)] @@ -22,16 +23,16 @@ impl Default for PayloadCache { } impl PayloadCache { - pub fn put(&self, payload: ExecutionPayload) -> Option> { - let root = payload.tree_hash_root(); + pub fn put(&self, payload: FullPayloadContents) -> Option> { + let root = payload.payload_ref().tree_hash_root(); self.payloads.lock().put(PayloadCacheId(root), payload) } - pub fn pop(&self, root: &Hash256) -> Option> { + pub fn pop(&self, root: &Hash256) -> Option> { self.payloads.lock().pop(&PayloadCacheId(*root)) } - pub fn get(&self, hash: &Hash256) -> Option> { + pub fn get(&self, hash: &Hash256) -> Option> { self.payloads.lock().get(&PayloadCacheId(*hash)).cloned() } } diff --git a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs index b4a7d247adf..71796402e71 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs @@ -201,7 +201,7 @@ impl MockExecutionLayer { assert_eq!( self.el .get_payload_by_root(&payload_header.tree_hash_root()), - Some(payload.clone()) + Some(FullPayloadContents::Payload(payload.clone())) ); let status = self.el.notify_new_payload(&payload, None).await.unwrap(); diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 2e580b5a4b2..42800ff1222 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -317,12 +317,11 @@ pub async fn reconstruct_block( .into(); ProvenancedPayload::Local(FullPayloadContents::Payload(payload)) // If we already have an execution payload with this transactions root cached, use it. - // TODO(jimmy): blobs should be cached as well } else if let Some(cached_payload) = el.get_payload_by_root(&payload_header.tree_hash_root()) { info!(log, "Reconstructing a full block using a local payload"; "block_hash" => ?cached_payload.block_hash()); - ProvenancedPayload::Local(FullPayloadContents::Payload(cached_payload)) + ProvenancedPayload::Local(cached_payload) // Otherwise, this means we are attempting a blind block proposal. } else { // Perform the logging for late blocks when we publish to the diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 6a8c5378275..0ab8ed462af 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -4,14 +4,12 @@ use crate::Error as ServerError; use lighthouse_network::{ConnectionDirection, Enr, Multiaddr, PeerConnectionStatus}; use mediatype::{names, MediaType, MediaTypeList}; -use serde::{Deserialize, Deserializer, Serialize}; -use serde_json::Value; +use serde::{Deserialize, Serialize}; use ssz_derive::Encode; use std::convert::TryFrom; use std::fmt::{self, Display}; use std::str::{from_utf8, FromStr}; use std::time::Duration; -use types::beacon_block_body::KzgCommitments; pub use types::*; #[cfg(feature = "lighthouse")] @@ -1666,71 +1664,3 @@ impl> ForkVersionDeserialize }) } } - -#[derive(Debug, PartialEq, Deserialize)] -#[serde(untagged)] -#[serde(bound = "E: EthSpec")] -pub enum FullPayloadContents { - Payload(ExecutionPayload), - PayloadAndBlobs(ExecutionPayloadAndBlobs), -} - -impl FullPayloadContents { - pub fn payload_ref(&self) -> &ExecutionPayload { - match self { - FullPayloadContents::Payload(payload) => payload, - FullPayloadContents::PayloadAndBlobs(payload_and_blobs) => { - &payload_and_blobs.execution_payload - } - } - } - - pub fn block_hash(&self) -> ExecutionBlockHash { - self.payload_ref().block_hash() - } - - pub fn deconstruct(self) -> (ExecutionPayload, Option>) { - match self { - FullPayloadContents::Payload(payload) => (payload, None), - FullPayloadContents::PayloadAndBlobs(payload_and_blobs) => ( - payload_and_blobs.execution_payload, - Some(payload_and_blobs.blobs_bundle), - ), - } - } -} - -impl ForkVersionDeserialize for FullPayloadContents { - fn deserialize_by_fork<'de, D: Deserializer<'de>>( - value: Value, - fork_name: ForkName, - ) -> Result { - match fork_name { - ForkName::Merge | ForkName::Capella => serde_json::from_value(value) - .map(Self::Payload) - .map_err(serde::de::Error::custom), - ForkName::Deneb => serde_json::from_value(value) - .map(Self::PayloadAndBlobs) - .map_err(serde::de::Error::custom), - ForkName::Base | ForkName::Altair => Err(serde::de::Error::custom(format!( - "FullPayloadContents deserialization for {fork_name} not implemented" - ))), - } - } -} - -#[derive(Debug, PartialEq, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct ExecutionPayloadAndBlobs { - pub execution_payload: ExecutionPayload, - pub blobs_bundle: BlobsBundle, -} - -#[derive(Debug, PartialEq, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct BlobsBundle { - pub commitments: KzgCommitments, - pub proofs: KzgProofs, - #[serde(with = "ssz_types::serde_utils::list_of_hex_fixed_vec")] - pub blobs: Blobs, -} diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 1b9e80db3d1..a869b1d9327 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -162,8 +162,9 @@ pub use crate::participation_flags::ParticipationFlags; pub use crate::participation_list::ParticipationList; pub use crate::payload::{ AbstractExecPayload, BlindedPayload, BlindedPayloadCapella, BlindedPayloadDeneb, - BlindedPayloadMerge, BlindedPayloadRef, BlockType, ExecPayload, FullPayload, - FullPayloadCapella, FullPayloadDeneb, FullPayloadMerge, FullPayloadRef, OwnedExecPayload, + BlindedPayloadMerge, BlindedPayloadRef, BlobsBundle, BlockType, ExecPayload, + ExecutionPayloadAndBlobs, FullPayload, FullPayloadCapella, FullPayloadContents, + FullPayloadDeneb, FullPayloadMerge, FullPayloadRef, OwnedExecPayload, }; pub use crate::pending_attestation::PendingAttestation; pub use crate::preset::{AltairPreset, BasePreset, BellatrixPreset, CapellaPreset}; diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index fd15bb5d5dc..e18dabe6f38 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -1,7 +1,9 @@ +use crate::beacon_block_body::KzgCommitments; use crate::{test_utils::TestRandom, *}; use derivative::Derivative; use serde::de::DeserializeOwned; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; +use serde_json::Value; use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; use std::borrow::Cow; @@ -955,3 +957,84 @@ impl From> for ExecutionPayloadHeader { } } } + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +#[serde(bound = "E: EthSpec")] +pub enum FullPayloadContents { + Payload(ExecutionPayload), + PayloadAndBlobs(ExecutionPayloadAndBlobs), +} + +impl FullPayloadContents { + pub fn new( + execution_payload: ExecutionPayload, + maybe_blobs: Option>, + ) -> Self { + match maybe_blobs { + None => Self::Payload(execution_payload), + Some(blobs_bundle) => Self::PayloadAndBlobs(ExecutionPayloadAndBlobs { + execution_payload, + blobs_bundle, + }), + } + } + + pub fn payload_ref(&self) -> &ExecutionPayload { + match self { + FullPayloadContents::Payload(payload) => payload, + FullPayloadContents::PayloadAndBlobs(payload_and_blobs) => { + &payload_and_blobs.execution_payload + } + } + } + + pub fn block_hash(&self) -> ExecutionBlockHash { + self.payload_ref().block_hash() + } + + pub fn deconstruct(self) -> (ExecutionPayload, Option>) { + match self { + FullPayloadContents::Payload(payload) => (payload, None), + FullPayloadContents::PayloadAndBlobs(payload_and_blobs) => ( + payload_and_blobs.execution_payload, + Some(payload_and_blobs.blobs_bundle), + ), + } + } +} + +impl ForkVersionDeserialize for FullPayloadContents { + fn deserialize_by_fork<'de, D: Deserializer<'de>>( + value: Value, + fork_name: ForkName, + ) -> Result { + match fork_name { + ForkName::Merge | ForkName::Capella => serde_json::from_value(value) + .map(Self::Payload) + .map_err(serde::de::Error::custom), + ForkName::Deneb => serde_json::from_value(value) + .map(Self::PayloadAndBlobs) + .map_err(serde::de::Error::custom), + ForkName::Base | ForkName::Altair => Err(serde::de::Error::custom(format!( + "FullPayloadContents deserialization for {fork_name} not implemented" + ))), + } + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(bound = "E: EthSpec")] +pub struct ExecutionPayloadAndBlobs { + pub execution_payload: ExecutionPayload, + pub blobs_bundle: BlobsBundle, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(bound = "E: EthSpec")] +pub struct BlobsBundle { + pub commitments: KzgCommitments, + pub proofs: KzgProofs, + #[serde(with = "ssz_types::serde_utils::list_of_hex_fixed_vec")] + pub blobs: Blobs, +} From 2f28d8489f37c8931bc538b9fb19fde6ee938b41 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Tue, 1 Aug 2023 00:34:17 +1000 Subject: [PATCH 14/24] Remove BlockProposal and move associated Sidecar trait to AbstractExecPayload to simplify changes --- beacon_node/beacon_chain/src/beacon_chain.rs | 380 +++++++++--------- .../beacon_chain/src/block_verification.rs | 13 +- beacon_node/beacon_chain/src/test_utils.rs | 31 +- beacon_node/builder_client/src/lib.rs | 4 +- beacon_node/execution_layer/src/lib.rs | 5 +- .../http_api/src/build_block_contents.rs | 29 +- beacon_node/http_api/src/lib.rs | 22 +- beacon_node/http_api/src/publish_blocks.rs | 14 +- .../tests/broadcast_validation_tests.rs | 44 +- .../http_api/tests/interactive_tests.rs | 10 +- beacon_node/http_api/tests/tests.rs | 110 +++-- common/eth2/src/lib.rs | 31 +- common/eth2/src/types.rs | 120 +++--- consensus/types/src/block_proposal.rs | 33 -- consensus/types/src/lib.rs | 2 - consensus/types/src/payload.rs | 4 + testing/web3signer_tests/src/lib.rs | 14 +- validator_client/src/block_service.rs | 47 ++- validator_client/src/signing_method.rs | 16 +- validator_client/src/validator_store.rs | 38 +- 20 files changed, 440 insertions(+), 527 deletions(-) delete mode 100644 consensus/types/src/block_proposal.rs diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 3b0edaf6fa8..7d2f431f7f7 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -480,8 +480,8 @@ pub struct BeaconChain { pub kzg: Option::Kzg>>>, } -type BeaconBlockAndState = ( - BeaconBlock>::Payload>, +type BeaconBlockAndState = ( + BeaconBlock, BeaconState, Option>, ); @@ -3817,13 +3817,13 @@ impl BeaconChain { /// /// The produced block will not be inherently valid, it must be signed by a block producer. /// Block signing is out of the scope of this function and should be done by a separate program. - pub async fn produce_block + 'static>( + pub async fn produce_block + 'static>( self: &Arc, randao_reveal: Signature, slot: Slot, validator_graffiti: Option, - ) -> Result, BlockProductionError> { - self.produce_block_with_verification::( + ) -> Result, BlockProductionError> { + self.produce_block_with_verification( randao_reveal, slot, validator_graffiti, @@ -3833,13 +3833,15 @@ impl BeaconChain { } /// Same as `produce_block` but allowing for configuration of RANDAO-verification. - pub async fn produce_block_with_verification + 'static>( + pub async fn produce_block_with_verification< + Payload: AbstractExecPayload + 'static, + >( self: &Arc, randao_reveal: Signature, slot: Slot, validator_graffiti: Option, verification: ProduceBlockVerification, - ) -> Result, BlockProductionError> { + ) -> Result, BlockProductionError> { // Part 1/2 (blocking) // // Load the parent state from disk. @@ -3857,7 +3859,7 @@ impl BeaconChain { // Part 2/2 (async, with some blocking components) // // Produce the block upon the state - self.produce_block_on_state::( + self.produce_block_on_state::( state, state_root_opt, slot, @@ -4429,7 +4431,7 @@ impl BeaconChain { /// The provided `state_root_opt` should only ever be set to `Some` if the contained value is /// equal to the root of `state`. Providing this value will serve as an optimization to avoid /// performing a tree hash in some scenarios. - pub async fn produce_block_on_state + 'static>( + pub async fn produce_block_on_state + 'static>( self: &Arc, state: BeaconState, state_root_opt: Option, @@ -4437,7 +4439,7 @@ impl BeaconChain { randao_reveal: Signature, validator_graffiti: Option, verification: ProduceBlockVerification, - ) -> Result, BlockProductionError> { + ) -> Result, BlockProductionError> { // Part 1/3 (blocking) // // Perform the state advance and block-packing functions. @@ -4446,7 +4448,7 @@ impl BeaconChain { .task_executor .spawn_blocking_handle( move || { - chain.produce_partial_beacon_block::( + chain.produce_partial_beacon_block( state, state_root_opt, produce_at_slot, @@ -4482,7 +4484,7 @@ impl BeaconChain { self.task_executor .spawn_blocking_handle( move || { - chain.complete_partial_beacon_block::( + chain.complete_partial_beacon_block( partial_beacon_block, block_contents, verification, @@ -4734,14 +4736,173 @@ impl BeaconChain { }) } - fn complete_partial_beacon_block>( + fn complete_partial_beacon_block>( &self, - partial_beacon_block: PartialBeaconBlock, - block_contents: Option>, + partial_beacon_block: PartialBeaconBlock, + block_contents: Option>, verification: ProduceBlockVerification, - ) -> Result, BlockProductionError> { - let (mut state, inner_block, blobs_opt, proofs_opt) = - Self::merge_block_contents_into_beacon_block(partial_beacon_block, block_contents)?; + ) -> Result, BlockProductionError> { + let PartialBeaconBlock { + mut state, + slot, + proposer_index, + parent_root, + randao_reveal, + eth1_data, + graffiti, + proposer_slashings, + attester_slashings, + attestations, + deposits, + voluntary_exits, + sync_aggregate, + // We don't need the prepare payload handle since the `execution_payload` is passed into + // this function. We can assume that the handle has already been consumed in order to + // produce said `execution_payload`. + prepare_payload_handle: _, + bls_to_execution_changes, + } = partial_beacon_block; + + let (inner_block, blobs_opt, proofs_opt) = match &state { + BeaconState::Base(_) => ( + BeaconBlock::Base(BeaconBlockBase { + slot, + proposer_index, + parent_root, + state_root: Hash256::zero(), + body: BeaconBlockBodyBase { + randao_reveal, + eth1_data, + graffiti, + proposer_slashings: proposer_slashings.into(), + attester_slashings: attester_slashings.into(), + attestations: attestations.into(), + deposits: deposits.into(), + voluntary_exits: voluntary_exits.into(), + _phantom: PhantomData, + }, + }), + None, + None, + ), + BeaconState::Altair(_) => ( + BeaconBlock::Altair(BeaconBlockAltair { + slot, + proposer_index, + parent_root, + state_root: Hash256::zero(), + body: BeaconBlockBodyAltair { + randao_reveal, + eth1_data, + graffiti, + proposer_slashings: proposer_slashings.into(), + attester_slashings: attester_slashings.into(), + attestations: attestations.into(), + deposits: deposits.into(), + voluntary_exits: voluntary_exits.into(), + sync_aggregate: sync_aggregate + .ok_or(BlockProductionError::MissingSyncAggregate)?, + _phantom: PhantomData, + }, + }), + None, + None, + ), + BeaconState::Merge(_) => { + let (payload, _, _, _) = block_contents + .ok_or(BlockProductionError::MissingExecutionPayload)? + .deconstruct(); + ( + BeaconBlock::Merge(BeaconBlockMerge { + slot, + proposer_index, + parent_root, + state_root: Hash256::zero(), + body: BeaconBlockBodyMerge { + randao_reveal, + eth1_data, + graffiti, + proposer_slashings: proposer_slashings.into(), + attester_slashings: attester_slashings.into(), + attestations: attestations.into(), + deposits: deposits.into(), + voluntary_exits: voluntary_exits.into(), + sync_aggregate: sync_aggregate + .ok_or(BlockProductionError::MissingSyncAggregate)?, + execution_payload: payload + .try_into() + .map_err(|_| BlockProductionError::InvalidPayloadFork)?, + }, + }), + None, + None, + ) + } + BeaconState::Capella(_) => { + let (payload, _, _, _) = block_contents + .ok_or(BlockProductionError::MissingExecutionPayload)? + .deconstruct(); + ( + BeaconBlock::Capella(BeaconBlockCapella { + slot, + proposer_index, + parent_root, + state_root: Hash256::zero(), + body: BeaconBlockBodyCapella { + randao_reveal, + eth1_data, + graffiti, + proposer_slashings: proposer_slashings.into(), + attester_slashings: attester_slashings.into(), + attestations: attestations.into(), + deposits: deposits.into(), + voluntary_exits: voluntary_exits.into(), + sync_aggregate: sync_aggregate + .ok_or(BlockProductionError::MissingSyncAggregate)?, + execution_payload: payload + .try_into() + .map_err(|_| BlockProductionError::InvalidPayloadFork)?, + bls_to_execution_changes: bls_to_execution_changes.into(), + }, + }), + None, + None, + ) + } + BeaconState::Deneb(_) => { + let (payload, kzg_commitments, blobs, proofs) = block_contents + .ok_or(BlockProductionError::MissingExecutionPayload)? + .deconstruct(); + ( + BeaconBlock::Deneb(BeaconBlockDeneb { + slot, + proposer_index, + parent_root, + state_root: Hash256::zero(), + body: BeaconBlockBodyDeneb { + randao_reveal, + eth1_data, + graffiti, + proposer_slashings: proposer_slashings.into(), + attester_slashings: attester_slashings.into(), + attestations: attestations.into(), + deposits: deposits.into(), + voluntary_exits: voluntary_exits.into(), + sync_aggregate: sync_aggregate + .ok_or(BlockProductionError::MissingSyncAggregate)?, + execution_payload: payload + .try_into() + .map_err(|_| BlockProductionError::InvalidPayloadFork)?, + bls_to_execution_changes: bls_to_execution_changes.into(), + blob_kzg_commitments: kzg_commitments + .ok_or(BlockProductionError::InvalidPayloadFork)?, + }, + }), + blobs, + proofs, + ) + } + }; let block = SignedBeaconBlock::from_block( inner_block, @@ -6188,189 +6349,6 @@ impl BeaconChain { Ok(SidecarListVariant::Blinded(blob_sidecars)) } - - #[allow(clippy::type_complexity)] //TODO(jimmy): fix type complexity - fn merge_block_contents_into_beacon_block>( - partial_beacon_block: PartialBeaconBlock, - block_contents: Option>, - ) -> Result< - ( - BeaconState, - BeaconBlock, - Option>, - Option>, - ), - BlockProductionError, - > { - let PartialBeaconBlock { - state, - slot, - proposer_index, - parent_root, - randao_reveal, - eth1_data, - graffiti, - proposer_slashings, - attester_slashings, - attestations, - deposits, - voluntary_exits, - sync_aggregate, - // We don't need the prepare payload handle since the `execution_payload` is passed into - // this function. We can assume that the handle has already been consumed in order to - // produce said `execution_payload`. - prepare_payload_handle: _, - bls_to_execution_changes, - } = partial_beacon_block; - - let result = match &state { - BeaconState::Base(_) => ( - state, - BeaconBlock::Base(BeaconBlockBase { - slot, - proposer_index, - parent_root, - state_root: Hash256::zero(), - body: BeaconBlockBodyBase { - randao_reveal, - eth1_data, - graffiti, - proposer_slashings: proposer_slashings.into(), - attester_slashings: attester_slashings.into(), - attestations: attestations.into(), - deposits: deposits.into(), - voluntary_exits: voluntary_exits.into(), - _phantom: PhantomData, - }, - }), - None, - None, - ), - BeaconState::Altair(_) => ( - state, - BeaconBlock::Altair(BeaconBlockAltair { - slot, - proposer_index, - parent_root, - state_root: Hash256::zero(), - body: BeaconBlockBodyAltair { - randao_reveal, - eth1_data, - graffiti, - proposer_slashings: proposer_slashings.into(), - attester_slashings: attester_slashings.into(), - attestations: attestations.into(), - deposits: deposits.into(), - voluntary_exits: voluntary_exits.into(), - sync_aggregate: sync_aggregate - .ok_or(BlockProductionError::MissingSyncAggregate)?, - _phantom: PhantomData, - }, - }), - None, - None, - ), - BeaconState::Merge(_) => { - let (payload, _, _, _) = block_contents - .ok_or(BlockProductionError::MissingExecutionPayload)? - .deconstruct(); - ( - state, - BeaconBlock::Merge(BeaconBlockMerge { - slot, - proposer_index, - parent_root, - state_root: Hash256::zero(), - body: BeaconBlockBodyMerge { - randao_reveal, - eth1_data, - graffiti, - proposer_slashings: proposer_slashings.into(), - attester_slashings: attester_slashings.into(), - attestations: attestations.into(), - deposits: deposits.into(), - voluntary_exits: voluntary_exits.into(), - sync_aggregate: sync_aggregate - .ok_or(BlockProductionError::MissingSyncAggregate)?, - execution_payload: payload - .try_into() - .map_err(|_| BlockProductionError::InvalidPayloadFork)?, - }, - }), - None, - None, - ) - } - BeaconState::Capella(_) => { - let (payload, _, _, _) = block_contents - .ok_or(BlockProductionError::MissingExecutionPayload)? - .deconstruct(); - ( - state, - BeaconBlock::Capella(BeaconBlockCapella { - slot, - proposer_index, - parent_root, - state_root: Hash256::zero(), - body: BeaconBlockBodyCapella { - randao_reveal, - eth1_data, - graffiti, - proposer_slashings: proposer_slashings.into(), - attester_slashings: attester_slashings.into(), - attestations: attestations.into(), - deposits: deposits.into(), - voluntary_exits: voluntary_exits.into(), - sync_aggregate: sync_aggregate - .ok_or(BlockProductionError::MissingSyncAggregate)?, - execution_payload: payload - .try_into() - .map_err(|_| BlockProductionError::InvalidPayloadFork)?, - bls_to_execution_changes: bls_to_execution_changes.into(), - }, - }), - None, - None, - ) - } - BeaconState::Deneb(_) => { - let (payload, kzg_commitments, blobs_or_blob_roots, proofs) = block_contents - .ok_or(BlockProductionError::MissingExecutionPayload)? - .deconstruct(); - ( - state, - BeaconBlock::Deneb(BeaconBlockDeneb { - slot, - proposer_index, - parent_root, - state_root: Hash256::zero(), - body: BeaconBlockBodyDeneb { - randao_reveal, - eth1_data, - graffiti, - proposer_slashings: proposer_slashings.into(), - attester_slashings: attester_slashings.into(), - attestations: attestations.into(), - deposits: deposits.into(), - voluntary_exits: voluntary_exits.into(), - sync_aggregate: sync_aggregate - .ok_or(BlockProductionError::MissingSyncAggregate)?, - execution_payload: payload - .try_into() - .map_err(|_| BlockProductionError::InvalidPayloadFork)?, - bls_to_execution_changes: bls_to_execution_changes.into(), - blob_kzg_commitments: kzg_commitments - .ok_or(BlockProductionError::InvalidPayloadFork)?, - }, - }), - blobs_or_blob_roots, - proofs, - ) - } - }; - - Ok(result) - } } impl Drop for BeaconChain { diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index d9691bc722a..c9e0ee4c9d2 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -70,7 +70,7 @@ use crate::{ metrics, BeaconChain, BeaconChainError, BeaconChainTypes, }; use derivative::Derivative; -use eth2::types::{EventKind, FullBlockProposal, SignedBlockContents}; +use eth2::types::{EventKind, SignedBlockContents}; use execution_layer::PayloadStatus; pub use fork_choice::{AttestationFromBlock, PayloadVerificationStatus}; use parking_lot::RwLockReadGuard; @@ -94,13 +94,14 @@ use std::fs; use std::io::Write; use std::sync::Arc; use std::time::Duration; -use store::{Error as DBError, HotStateSummary, KeyValueStore, StoreOp}; +use store::{Error as DBError, HotStateSummary, KeyValueStore, SignedBlobSidecarList, StoreOp}; use task_executor::JoinHandle; use tree_hash::TreeHash; +use types::ExecPayload; use types::{ BeaconBlockRef, BeaconState, BeaconStateError, ChainSpec, CloneConfig, Epoch, EthSpec, - ExecPayload, ExecutionBlockHash, Hash256, InconsistentFork, PublicKey, PublicKeyBytes, - RelativeEpoch, SignedBeaconBlock, SignedBeaconBlockHeader, SignedBlobSidecarList, Slot, + ExecutionBlockHash, Hash256, InconsistentFork, PublicKey, PublicKeyBytes, RelativeEpoch, + SignedBeaconBlock, SignedBeaconBlockHeader, Slot, }; pub const POS_PANDA_BANNER: &str = r#" @@ -688,9 +689,7 @@ impl IntoGossipVerifiedBlockContents for GossipVerifiedB } } -impl IntoGossipVerifiedBlockContents - for SignedBlockContents -{ +impl IntoGossipVerifiedBlockContents for SignedBlockContents { fn into_gossip_verified_block( self, chain: &BeaconChain, diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 6d40ca19530..7b83598bbb2 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -819,7 +819,7 @@ where state: BeaconState, slot: Slot, ) -> ( - SignedBlockContentsTuple, + SignedBlockContentsTuple>, BeaconState, ) { let (unblinded, new_state) = self.make_block(state, slot).await; @@ -847,10 +847,7 @@ where &self, mut state: BeaconState, slot: Slot, - ) -> ( - SignedBlockContentsTuple, - BeaconState, - ) { + ) -> (SignedBlockContentsTuple>, BeaconState) { assert_ne!(slot, 0, "can't produce a block at slot 0"); assert!(slot >= state.slot()); @@ -870,7 +867,7 @@ where let (block, state, maybe_blob_sidecars) = self .chain - .produce_block_on_state::( + .produce_block_on_state( state, None, slot, @@ -888,7 +885,7 @@ where &self.spec, ); - let block_contents: SignedBlockContentsTuple = match &signed_block { + let block_contents: SignedBlockContentsTuple> = match &signed_block { SignedBeaconBlock::Base(_) | SignedBeaconBlock::Altair(_) | SignedBeaconBlock::Merge(_) @@ -930,10 +927,7 @@ where &self, mut state: BeaconState, slot: Slot, - ) -> ( - SignedBlockContentsTuple, - BeaconState, - ) { + ) -> (SignedBlockContentsTuple>, BeaconState) { assert_ne!(slot, 0, "can't produce a block at slot 0"); assert!(slot >= state.slot()); @@ -955,7 +949,7 @@ where let (block, state, maybe_blob_sidecars) = self .chain - .produce_block_on_state::( + .produce_block_on_state( state, None, slot, @@ -973,7 +967,7 @@ where &self.spec, ); - let block_contents: SignedBlockContentsTuple = match &signed_block { + let block_contents: SignedBlockContentsTuple> = match &signed_block { SignedBeaconBlock::Base(_) | SignedBeaconBlock::Altair(_) | SignedBeaconBlock::Merge(_) @@ -1797,10 +1791,7 @@ where state: BeaconState, slot: Slot, block_modifier: impl FnOnce(&mut BeaconBlock), - ) -> ( - SignedBlockContentsTuple, - BeaconState, - ) { + ) -> (SignedBlockContentsTuple>, BeaconState) { assert_ne!(slot, 0, "can't produce a block at slot 0"); assert!(slot >= state.slot()); @@ -1898,7 +1889,7 @@ where &self, slot: Slot, block_root: Hash256, - block_contents: SignedBlockContentsTuple, + block_contents: SignedBlockContentsTuple>, ) -> Result> { self.set_current_slot(slot); let (block, blobs) = block_contents; @@ -1928,7 +1919,7 @@ where pub async fn process_block_result( &self, - block_contents: SignedBlockContentsTuple, + block_contents: SignedBlockContentsTuple>, ) -> Result> { let (block, blobs) = block_contents; // Note: we are just dropping signatures here and skipping signature verification. @@ -2013,7 +2004,7 @@ where ) -> Result< ( SignedBeaconBlockHash, - SignedBlockContentsTuple, + SignedBlockContentsTuple>, BeaconState, ), BlockError, diff --git a/beacon_node/builder_client/src/lib.rs b/beacon_node/builder_client/src/lib.rs index 7fbbb86bb70..089e576f504 100644 --- a/beacon_node/builder_client/src/lib.rs +++ b/beacon_node/builder_client/src/lib.rs @@ -1,7 +1,7 @@ use eth2::types::builder_bid::SignedBuilderBid; use eth2::types::payload::FullPayloadContents; use eth2::types::{ - AbstractExecPayload, BlindedBlockProposal, EthSpec, ExecutionBlockHash, ForkVersionedResponse, + AbstractExecPayload, BlindedPayload, EthSpec, ExecutionBlockHash, ForkVersionedResponse, PublicKeyBytes, SignedBlockContents, SignedValidatorRegistrationData, Slot, }; pub use eth2::Error; @@ -140,7 +140,7 @@ impl BuilderHttpClient { /// `POST /eth/v1/builder/blinded_blocks` pub async fn post_builder_blinded_blocks( &self, - blinded_block: &SignedBlockContents, + blinded_block: &SignedBlockContents>, ) -> Result>, Error> { let mut path = self.server.full.clone(); diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 3aa18b1282c..4c9ad744b61 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -48,8 +48,7 @@ use types::{ AbstractExecPayload, BeaconStateError, ExecPayload, ExecutionPayloadDeneb, VersionedHash, }; use types::{ - BlindedBlockProposal, BlindedPayload, BlockType, ChainSpec, Epoch, ExecutionPayloadCapella, - ExecutionPayloadMerge, + BlindedPayload, BlockType, ChainSpec, Epoch, ExecutionPayloadCapella, ExecutionPayloadMerge, }; use types::{ProposerPreparationData, PublicKeyBytes, Signature, Slot, Transaction}; @@ -1883,7 +1882,7 @@ impl ExecutionLayer { pub async fn propose_blinded_beacon_block( &self, block_root: Hash256, - block: &SignedBlockContents, + block: &SignedBlockContents>, ) -> Result, Error> { debug!( self.log(), diff --git a/beacon_node/http_api/src/build_block_contents.rs b/beacon_node/http_api/src/build_block_contents.rs index 5392692748b..9a73e645f0d 100644 --- a/beacon_node/http_api/src/build_block_contents.rs +++ b/beacon_node/http_api/src/build_block_contents.rs @@ -1,19 +1,16 @@ -use beacon_chain::{BeaconChainTypes, BlockProductionError}; -use eth2::types::{ - BeaconBlockAndBlobSidecars, BlindedBeaconBlockAndBlobSidecars, BlindedBlockProposal, - BlockContents, FullBlockProposal, -}; -use types::{BeaconBlock, BlindedPayload, ForkName, FullPayload, SidecarListVariant}; +use beacon_chain::BlockProductionError; +use eth2::types::{BeaconBlockAndBlobSidecars, BlindedBeaconBlockAndBlobSidecars, BlockContents}; +use types::{BeaconBlock, BlindedPayload, EthSpec, ForkName, FullPayload, SidecarListVariant}; type Error = warp::reject::Rejection; -type FullBlockContents = BlockContents; -type BlindedBlockContents = BlockContents; +type FullBlockContents = BlockContents>; +type BlindedBlockContents = BlockContents>; -pub fn build_block_contents( +pub fn build_block_contents( fork_name: ForkName, - block: BeaconBlock>, - maybe_blobs: Option>, -) -> Result, Error> { + block: BeaconBlock>, + maybe_blobs: Option>, +) -> Result, Error> { match fork_name { ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { Ok(BlockContents::Block(block)) @@ -35,11 +32,11 @@ pub fn build_block_contents( } } -pub fn build_blinded_block_contents( +pub fn build_blinded_block_contents( fork_name: ForkName, - block: BeaconBlock>, - maybe_blobs: Option>, -) -> Result, Error> { + block: BeaconBlock>, + maybe_blobs: Option>, +) -> Result, Error> { match fork_name { ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { Ok(BlockContents::Block(block)) diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 6e55b855b92..cd6eca7196e 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -33,7 +33,7 @@ pub use block_id::BlockId; use directory::DEFAULT_ROOT_DIR; use eth2::types::{ self as api_types, BroadcastValidation, EndpointVersion, ForkChoice, ForkChoiceNode, - FullBlockProposal, SignedBlockContents, SkipRandaoVerification, ValidatorId, ValidatorStatus, + SignedBlockContents, SkipRandaoVerification, ValidatorId, ValidatorStatus, }; use lighthouse_network::{types::SyncState, EnrExt, NetworkGlobals, PeerId, PubsubMessage}; use lighthouse_version::version_with_platform; @@ -61,7 +61,7 @@ use tokio::sync::mpsc::{Sender, UnboundedSender}; use tokio_stream::{wrappers::BroadcastStream, StreamExt}; use types::{ Attestation, AttestationData, AttestationShufflingId, AttesterSlashing, BeaconStateError, - BlindedBlockProposal, CommitteeCache, ConfigAndPreset, Epoch, EthSpec, ForkName, + BlindedPayload, CommitteeCache, ConfigAndPreset, Epoch, EthSpec, ForkName, FullPayload, ProposerPreparationData, ProposerSlashing, RelativeEpoch, SignedAggregateAndProof, SignedBlsToExecutionChange, SignedContributionAndProof, SignedValidatorRegistrationData, SignedVoluntaryExit, Slot, SyncCommitteeMessage, SyncContributionData, @@ -1219,7 +1219,7 @@ pub fn serve( .and(network_tx_filter.clone()) .and(log_filter.clone()) .and_then( - |block_contents: SignedBlockContents, + |block_contents: SignedBlockContents, chain: Arc>, network_tx: UnboundedSender>, log: Logger| async move { @@ -1247,7 +1247,7 @@ pub fn serve( .and(log_filter.clone()) .then( |validation_level: api_types::BroadcastValidationQuery, - block_contents: SignedBlockContents, + block_contents: SignedBlockContents, chain: Arc>, network_tx: UnboundedSender>, log: Logger| async move { @@ -1288,7 +1288,7 @@ pub fn serve( .and(network_tx_filter.clone()) .and(log_filter.clone()) .and_then( - |block_contents: SignedBlockContents, + |block_contents: SignedBlockContents>, chain: Arc>, network_tx: UnboundedSender>, log: Logger| async move { @@ -1315,7 +1315,7 @@ pub fn serve( .and(log_filter.clone()) .then( |validation_level: api_types::BroadcastValidationQuery, - block_contents: SignedBlockContents, + block_contents: SignedBlockContents>, chain: Arc>, network_tx: UnboundedSender>, log: Logger| async move { @@ -2486,7 +2486,7 @@ pub fn serve( )) } }) - .await + .await } }, ); @@ -2723,7 +2723,7 @@ pub fn serve( }; let (block, _, maybe_blobs) = chain - .produce_block_with_verification::( + .produce_block_with_verification::>( randao_reveal, slot, query.graffiti.map(Into::into), @@ -2737,7 +2737,7 @@ pub fn serve( .map_err(inconsistent_fork_rejection)?; let block_contents = - build_block_contents::build_block_contents::(fork_name, block, maybe_blobs)?; + build_block_contents::build_block_contents(fork_name, block, maybe_blobs)?; fork_versioned_response(endpoint_version, fork_name, block_contents) .map(|response| warp::reply::json(&response).into_response()) @@ -2782,7 +2782,7 @@ pub fn serve( }; let (block, _, maybe_blobs) = chain - .produce_block_with_verification::( + .produce_block_with_verification::>( randao_reveal, slot, query.graffiti.map(Into::into), @@ -2796,7 +2796,7 @@ pub fn serve( .map_err(inconsistent_fork_rejection)?; let block_contents = - build_block_contents::build_blinded_block_contents::(fork_name, block, maybe_blobs)?; + build_block_contents::build_blinded_block_contents(fork_name, block, maybe_blobs)?; // Pose as a V2 endpoint so we return the fork `version`. fork_versioned_response(V2, fork_name, block_contents) diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 42800ff1222..3fe1f37c976 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -6,7 +6,7 @@ use beacon_chain::{ AvailabilityProcessingStatus, BeaconChain, BeaconChainError, BeaconChainTypes, BlockError, IntoGossipVerifiedBlockContents, NotifyExecutionLayer, }; -use eth2::types::{BroadcastValidation, FullBlockProposal}; +use eth2::types::BroadcastValidation; use eth2::types::{FullPayloadContents, SignedBlockContents}; use execution_layer::ProvenancedPayload; use lighthouse_network::PubsubMessage; @@ -19,8 +19,8 @@ use std::time::Duration; use tokio::sync::mpsc::UnboundedSender; use tree_hash::TreeHash; use types::{ - AbstractExecPayload, BeaconBlockRef, BlindedBlockProposal, EthSpec, ExecPayload, - ExecutionBlockHash, FullPayload, Hash256, SignedBeaconBlock, SignedBlobSidecarList, + AbstractExecPayload, BeaconBlockRef, BlindedPayload, EthSpec, ExecPayload, ExecutionBlockHash, + FullPayload, Hash256, SignedBeaconBlock, SignedBlobSidecarList, }; use warp::Rejection; @@ -267,14 +267,14 @@ pub async fn publish_block( - block_contents: SignedBlockContents, + block_contents: SignedBlockContents>, chain: Arc>, network_tx: &UnboundedSender>, log: Logger, validation_level: BroadcastValidation, ) -> Result<(), Rejection> { let block_root = block_contents.signed_block().canonical_root(); - let full_block = + let full_block: ProvenancedBlock> = reconstruct_block(chain.clone(), block_root, block_contents, log.clone()).await?; publish_block::( Some(block_root), @@ -293,9 +293,9 @@ pub async fn publish_blinded_block( pub async fn reconstruct_block( chain: Arc>, block_root: Hash256, - block_contents: SignedBlockContents, + block_contents: SignedBlockContents>, log: Logger, -) -> Result>, Rejection> { +) -> Result>, Rejection> { let block = block_contents.signed_block(); let full_payload_opt = if let Ok(payload_header) = block.message().body().execution_payload() { let el = chain.execution_layer.as_ref().ok_or_else(|| { diff --git a/beacon_node/http_api/tests/broadcast_validation_tests.rs b/beacon_node/http_api/tests/broadcast_validation_tests.rs index ccc8d32ca97..b525c23a5f3 100644 --- a/beacon_node/http_api/tests/broadcast_validation_tests.rs +++ b/beacon_node/http_api/tests/broadcast_validation_tests.rs @@ -9,7 +9,7 @@ use http_api::test_utils::InteractiveTester; use http_api::{publish_blinded_block, publish_block, reconstruct_block, ProvenancedBlock}; use std::sync::Arc; use tree_hash::TreeHash; -use types::{BlindedBlockProposal, FullBlockProposal, Hash256, MainnetEthSpec, Slot}; +use types::{Hash256, MainnetEthSpec, Slot}; use warp::Rejection; use warp_utils::reject::CustomBadRequest; @@ -76,10 +76,7 @@ pub async fn gossip_invalid() { let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2( - &SignedBlockContents::::new(block, blobs), - validation_level, - ) + .post_beacon_blocks_v2(&SignedBlockContents::new(block, blobs), validation_level) .await; assert!(response.is_err()); @@ -130,10 +127,7 @@ pub async fn gossip_partial_pass() { let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2( - &SignedBlockContents::::new(block, blobs), - validation_level, - ) + .post_beacon_blocks_v2(&SignedBlockContents::new(block, blobs), validation_level) .await; assert!(response.is_err()); @@ -176,7 +170,7 @@ pub async fn gossip_full_pass() { let response: Result<(), eth2::Error> = tester .client .post_beacon_blocks_v2( - &SignedBlockContents::::new(block.clone(), blobs), + &SignedBlockContents::new(block.clone(), blobs), validation_level, ) .await; @@ -225,10 +219,7 @@ pub async fn consensus_invalid() { let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2( - &SignedBlockContents::::new(block, blobs), - validation_level, - ) + .post_beacon_blocks_v2(&SignedBlockContents::new(block, blobs), validation_level) .await; assert!(response.is_err()); @@ -277,10 +268,7 @@ pub async fn consensus_gossip() { let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2( - &SignedBlockContents::::new(block, blobs), - validation_level, - ) + .post_beacon_blocks_v2(&SignedBlockContents::new(block, blobs), validation_level) .await; assert!(response.is_err()); @@ -394,7 +382,7 @@ pub async fn consensus_full_pass() { let response: Result<(), eth2::Error> = tester .client .post_beacon_blocks_v2( - &SignedBlockContents::::new(block.clone(), blobs), + &SignedBlockContents::new(block.clone(), blobs), validation_level, ) .await; @@ -445,10 +433,7 @@ pub async fn equivocation_invalid() { let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2( - &SignedBlockContents::::new(block, blobs), - validation_level, - ) + .post_beacon_blocks_v2(&SignedBlockContents::new(block, blobs), validation_level) .await; assert!(response.is_err()); @@ -505,7 +490,7 @@ pub async fn equivocation_consensus_early_equivocation() { assert!(tester .client .post_beacon_blocks_v2( - &SignedBlockContents::::new(block_a.clone(), blobs_a), + &SignedBlockContents::new(block_a.clone(), blobs_a), validation_level ) .await @@ -519,7 +504,7 @@ pub async fn equivocation_consensus_early_equivocation() { let response: Result<(), eth2::Error> = tester .client .post_beacon_blocks_v2( - &SignedBlockContents::::new(block_b.clone(), blobs_b), + &SignedBlockContents::new(block_b.clone(), blobs_b), validation_level, ) .await; @@ -570,10 +555,7 @@ pub async fn equivocation_gossip() { let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2( - &SignedBlockContents::::new(block, blobs), - validation_level, - ) + .post_beacon_blocks_v2(&SignedBlockContents::new(block, blobs), validation_level) .await; assert!(response.is_err()); @@ -697,7 +679,7 @@ pub async fn equivocation_full_pass() { let response: Result<(), eth2::Error> = tester .client .post_beacon_blocks_v2( - &SignedBlockContents::::new(block.clone(), blobs), + &SignedBlockContents::new(block.clone(), blobs), validation_level, ) .await; @@ -1311,7 +1293,7 @@ pub async fn blinded_equivocation_full_pass() { let response: Result<(), eth2::Error> = tester .client .post_beacon_blocks_v2( - &SignedBlockContents::::new(block.clone(), blobs), + &SignedBlockContents::new(block.clone(), blobs), validation_level, ) .await; diff --git a/beacon_node/http_api/tests/interactive_tests.rs b/beacon_node/http_api/tests/interactive_tests.rs index a42b6b2ae34..f27a4d9519d 100644 --- a/beacon_node/http_api/tests/interactive_tests.rs +++ b/beacon_node/http_api/tests/interactive_tests.rs @@ -4,7 +4,7 @@ use beacon_chain::{ test_utils::{AttestationStrategy, BlockStrategy, SyncCommitteeStrategy}, ChainConfig, }; -use eth2::types::{DepositContractData, FullBlockProposal, StateId}; +use eth2::types::{DepositContractData, StateId}; use execution_layer::{ForkchoiceState, PayloadAttributes}; use http_api::test_utils::InteractiveTester; use parking_lot::Mutex; @@ -17,8 +17,8 @@ use std::sync::Arc; use std::time::Duration; use tree_hash::TreeHash; use types::{ - Address, Epoch, EthSpec, ExecPayload, ExecutionBlockHash, ForkName, MainnetEthSpec, - MinimalEthSpec, ProposerPreparationData, Slot, + Address, Epoch, EthSpec, ExecPayload, ExecutionBlockHash, ForkName, FullPayload, + MainnetEthSpec, MinimalEthSpec, ProposerPreparationData, Slot, }; type E = MainnetEthSpec; @@ -619,7 +619,7 @@ pub async fn proposer_boost_re_org_test( .into(); let unsigned_block_contents_c = tester .client - .get_validator_blocks::(slot_c, &randao_reveal, None) + .get_validator_blocks(slot_c, &randao_reveal, None) .await .unwrap() .data; @@ -812,7 +812,7 @@ pub async fn fork_choice_before_proposal() { .into(); let block_d = tester .client - .get_validator_blocks::(slot_d, &randao_reveal, None) + .get_validator_blocks::>(slot_d, &randao_reveal, None) .await .unwrap() .data diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 4fda931ae4d..db04b3af86c 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -63,8 +63,8 @@ struct ApiTester { harness: Arc>>, chain: Arc>>, client: BeaconNodeHttpClient, - next_block: SignedBlockContents, - reorg_block: SignedBlockContents, + next_block: SignedBlockContents, + reorg_block: SignedBlockContents, attestations: Vec>, contribution_and_proofs: Vec>, attester_slashing: AttesterSlashing, @@ -1266,7 +1266,7 @@ impl ApiTester { assert!(self .client - .post_beacon_blocks(&SignedBlockContents::::from(block)) + .post_beacon_blocks(&SignedBlockContents::from(block)) .await .is_err()); @@ -2385,7 +2385,7 @@ impl ApiTester { let block = self .client - .get_validator_blocks::(slot, &randao_reveal, None) + .get_validator_blocks::>(slot, &randao_reveal, None) .await .unwrap() .data @@ -2393,8 +2393,7 @@ impl ApiTester { .0; let signed_block = block.sign(&sk, &fork, genesis_validators_root, &self.chain.spec); - let signed_block_contents = - SignedBlockContents::::from(signed_block.clone()); + let signed_block_contents = SignedBlockContents::from(signed_block.clone()); self.client .post_beacon_blocks(&signed_block_contents) @@ -2415,7 +2414,7 @@ impl ApiTester { let block = self .client - .get_validator_blocks_modular::( + .get_validator_blocks_modular::>( slot, &Signature::infinity().unwrap().into(), None, @@ -2473,13 +2472,13 @@ impl ApiTester { // Check failure with no `skip_randao_verification` passed. self.client - .get_validator_blocks::(slot, &bad_randao_reveal, None) + .get_validator_blocks::>(slot, &bad_randao_reveal, None) .await .unwrap_err(); // Check failure with `skip_randao_verification` (requires infinity sig). self.client - .get_validator_blocks_modular::( + .get_validator_blocks_modular::>( slot, &bad_randao_reveal, None, @@ -2494,7 +2493,7 @@ impl ApiTester { self } - pub async fn test_blinded_block_production>(&self) { + pub async fn test_blinded_block_production>(&self) { let fork = self.chain.canonical_head.cached_head().head_fork(); let genesis_validators_root = self.chain.genesis_validators_root; @@ -2534,7 +2533,7 @@ impl ApiTester { let block = self .client - .get_validator_blinded_blocks::(slot, &randao_reveal, None) + .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() .data @@ -2542,7 +2541,8 @@ impl ApiTester { .0; let signed_block = block.sign(&sk, &fork, genesis_validators_root, &self.chain.spec); - let signed_block_contents = SignedBlockContents::::Block(signed_block.clone()); + let signed_block_contents = + SignedBlockContents::::Block(signed_block.clone()); self.client .post_beacon_blinded_blocks(&signed_block_contents) @@ -2557,13 +2557,15 @@ impl ApiTester { } } - pub async fn test_blinded_block_production_no_verify_randao>(self) -> Self { + pub async fn test_blinded_block_production_no_verify_randao>( + self, + ) -> Self { for _ in 0..E::slots_per_epoch() { let slot = self.chain.slot().unwrap(); let block_contents = self .client - .get_validator_blinded_blocks_modular::( + .get_validator_blinded_blocks_modular::( slot, &Signature::infinity().unwrap().into(), None, @@ -2579,7 +2581,9 @@ impl ApiTester { self } - pub async fn test_blinded_block_production_verify_randao_invalid>( + pub async fn test_blinded_block_production_verify_randao_invalid< + Payload: AbstractExecPayload, + >( self, ) -> Self { let fork = self.chain.canonical_head.cached_head().head_fork(); @@ -2621,13 +2625,13 @@ impl ApiTester { // Check failure with full randao verification enabled. self.client - .get_validator_blinded_blocks::(slot, &bad_randao_reveal, None) + .get_validator_blinded_blocks::(slot, &bad_randao_reveal, None) .await .unwrap_err(); // Check failure with `skip_randao_verification` (requires infinity sig). self.client - .get_validator_blinded_blocks_modular::( + .get_validator_blinded_blocks_modular::( slot, &bad_randao_reveal, None, @@ -3039,7 +3043,7 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::(slot, &randao_reveal, None) + .get_validator_blinded_blocks::>(slot, &randao_reveal, None) .await .unwrap() .data @@ -3080,7 +3084,7 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::(slot, &randao_reveal, None) + .get_validator_blinded_blocks::>(slot, &randao_reveal, None) .await .unwrap() .data @@ -3124,7 +3128,7 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::(slot, &randao_reveal, None) + .get_validator_blinded_blocks::>(slot, &randao_reveal, None) .await .unwrap() .data @@ -3174,7 +3178,7 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::(slot, &randao_reveal, None) + .get_validator_blinded_blocks::>(slot, &randao_reveal, None) .await .unwrap() .data @@ -3223,7 +3227,7 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::(slot, &randao_reveal, None) + .get_validator_blinded_blocks::>(slot, &randao_reveal, None) .await .unwrap() .data @@ -3271,7 +3275,7 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::(slot, &randao_reveal, None) + .get_validator_blinded_blocks::>(slot, &randao_reveal, None) .await .unwrap() .data @@ -3318,7 +3322,7 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::(slot, &randao_reveal, None) + .get_validator_blinded_blocks::>(slot, &randao_reveal, None) .await .unwrap() .data @@ -3355,7 +3359,7 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::(slot, &randao_reveal, None) + .get_validator_blinded_blocks::>(slot, &randao_reveal, None) .await .unwrap() .data @@ -3393,7 +3397,7 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::(slot, &randao_reveal, None) + .get_validator_blinded_blocks::>(slot, &randao_reveal, None) .await .unwrap() .data @@ -3437,11 +3441,7 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::( - next_slot, - &randao_reveal, - None, - ) + .get_validator_blinded_blocks::>(next_slot, &randao_reveal, None) .await .unwrap() .data @@ -3471,11 +3471,7 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::( - next_slot, - &randao_reveal, - None, - ) + .get_validator_blinded_blocks::>(next_slot, &randao_reveal, None) .await .unwrap() .data @@ -3525,11 +3521,7 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::( - next_slot, - &randao_reveal, - None, - ) + .get_validator_blinded_blocks::>(next_slot, &randao_reveal, None) .await .unwrap() .data @@ -3569,11 +3561,7 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::( - next_slot, - &randao_reveal, - None, - ) + .get_validator_blinded_blocks::>(next_slot, &randao_reveal, None) .await .unwrap() .data @@ -3617,7 +3605,7 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::(slot, &randao_reveal, None) + .get_validator_blinded_blocks::>(slot, &randao_reveal, None) .await .unwrap() .data @@ -3659,7 +3647,7 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::(slot, &randao_reveal, None) + .get_validator_blinded_blocks::>(slot, &randao_reveal, None) .await .unwrap() .data @@ -3697,7 +3685,7 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::(slot, &randao_reveal, None) + .get_validator_blinded_blocks::>(slot, &randao_reveal, None) .await .unwrap() .data @@ -3735,7 +3723,7 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::(slot, &randao_reveal, None) + .get_validator_blinded_blocks::>(slot, &randao_reveal, None) .await .unwrap() .data @@ -3773,7 +3761,7 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::(slot, &randao_reveal, None) + .get_validator_blinded_blocks::>(slot, &randao_reveal, None) .await .unwrap() .data @@ -3824,7 +3812,7 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::(slot, &randao_reveal, None) + .get_validator_blinded_blocks::>(slot, &randao_reveal, None) .await .unwrap() .data @@ -3867,7 +3855,7 @@ impl ApiTester { let payload: BlindedPayload = self .client - .get_validator_blinded_blocks::(slot, &randao_reveal, None) + .get_validator_blinded_blocks::>(slot, &randao_reveal, None) .await .unwrap() .data @@ -4636,7 +4624,7 @@ async fn block_production_verify_randao_invalid() { async fn blinded_block_production_full_payload_premerge() { ApiTester::new() .await - .test_blinded_block_production::() + .test_blinded_block_production::>() .await; } @@ -4645,7 +4633,7 @@ async fn blinded_block_production_with_skip_slots_full_payload_premerge() { ApiTester::new() .await .skip_slots(E::slots_per_epoch() * 2) - .test_blinded_block_production::() + .test_blinded_block_production::>() .await; } @@ -4653,7 +4641,7 @@ async fn blinded_block_production_with_skip_slots_full_payload_premerge() { async fn blinded_block_production_no_verify_randao_full_payload_premerge() { ApiTester::new() .await - .test_blinded_block_production_no_verify_randao::() + .test_blinded_block_production_no_verify_randao::>() .await; } @@ -4661,7 +4649,7 @@ async fn blinded_block_production_no_verify_randao_full_payload_premerge() { async fn blinded_block_production_verify_randao_invalid_full_payload_premerge() { ApiTester::new() .await - .test_blinded_block_production_verify_randao_invalid::() + .test_blinded_block_production_verify_randao_invalid::>() .await; } @@ -4669,7 +4657,7 @@ async fn blinded_block_production_verify_randao_invalid_full_payload_premerge() async fn blinded_block_production_blinded_payload_premerge() { ApiTester::new() .await - .test_blinded_block_production::() + .test_blinded_block_production::>() .await; } @@ -4678,7 +4666,7 @@ async fn blinded_block_production_with_skip_slots_blinded_payload_premerge() { ApiTester::new() .await .skip_slots(E::slots_per_epoch() * 2) - .test_blinded_block_production::() + .test_blinded_block_production::>() .await; } @@ -4686,7 +4674,7 @@ async fn blinded_block_production_with_skip_slots_blinded_payload_premerge() { async fn blinded_block_production_no_verify_randao_blinded_payload_premerge() { ApiTester::new() .await - .test_blinded_block_production_no_verify_randao::() + .test_blinded_block_production_no_verify_randao::>() .await; } @@ -4694,7 +4682,7 @@ async fn blinded_block_production_no_verify_randao_blinded_payload_premerge() { async fn blinded_block_production_verify_randao_invalid_blinded_payload_premerge() { ApiTester::new() .await - .test_blinded_block_production_verify_randao_invalid::() + .test_blinded_block_production_verify_randao_invalid::>() .await; } diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index f214852818e..d8cceadf6c7 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -637,9 +637,9 @@ impl BeaconNodeHttpClient { /// `POST beacon/blocks` /// /// Returns `Ok(None)` on a 404 error. - pub async fn post_beacon_blocks>( + pub async fn post_beacon_blocks>( &self, - block_contents: &SignedBlockContents, + block_contents: &SignedBlockContents, ) -> Result<(), Error> { let mut path = self.eth_path(V1)?; @@ -657,9 +657,9 @@ impl BeaconNodeHttpClient { /// `POST beacon/blinded_blocks` /// /// Returns `Ok(None)` on a 404 error. - pub async fn post_beacon_blinded_blocks>( + pub async fn post_beacon_blinded_blocks>( &self, - block: &SignedBlockContents, + block: &SignedBlockContents, ) -> Result<(), Error> { let mut path = self.eth_path(V1)?; @@ -711,9 +711,9 @@ impl BeaconNodeHttpClient { } /// `POST v2/beacon/blocks` - pub async fn post_beacon_blocks_v2>( + pub async fn post_beacon_blocks_v2>( &self, - block_contents: &SignedBlockContents, + block_contents: &SignedBlockContents, validation_level: Option, ) -> Result<(), Error> { self.post_generic_with_consensus_version( @@ -1478,24 +1478,24 @@ impl BeaconNodeHttpClient { } /// `GET v2/validator/blocks/{slot}` - pub async fn get_validator_blocks>( + pub async fn get_validator_blocks>( &self, slot: Slot, randao_reveal: &SignatureBytes, graffiti: Option<&Graffiti>, - ) -> Result>, Error> { + ) -> Result>, Error> { self.get_validator_blocks_modular(slot, randao_reveal, graffiti, SkipRandaoVerification::No) .await } /// `GET v2/validator/blocks/{slot}` - pub async fn get_validator_blocks_modular>( + pub async fn get_validator_blocks_modular>( &self, slot: Slot, randao_reveal: &SignatureBytes, graffiti: Option<&Graffiti>, skip_randao_verification: SkipRandaoVerification, - ) -> Result>, Error> { + ) -> Result>, Error> { let mut path = self.eth_path(V2)?; path.path_segments_mut() @@ -1521,12 +1521,12 @@ impl BeaconNodeHttpClient { } /// `GET v2/validator/blinded_blocks/{slot}` - pub async fn get_validator_blinded_blocks>( + pub async fn get_validator_blinded_blocks>( &self, slot: Slot, randao_reveal: &SignatureBytes, graffiti: Option<&Graffiti>, - ) -> Result>, Error> { + ) -> Result>, Error> { self.get_validator_blinded_blocks_modular( slot, randao_reveal, @@ -1537,13 +1537,16 @@ impl BeaconNodeHttpClient { } /// `GET v1/validator/blinded_blocks/{slot}` - pub async fn get_validator_blinded_blocks_modular>( + pub async fn get_validator_blinded_blocks_modular< + T: EthSpec, + Payload: AbstractExecPayload, + >( &self, slot: Slot, randao_reveal: &SignatureBytes, graffiti: Option<&Graffiti>, skip_randao_verification: SkipRandaoVerification, - ) -> Result>, Error> { + ) -> Result>, Error> { let mut path = self.eth_path(V1)?; path.path_segments_mut() diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 0ab8ed462af..9f1781567fa 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -1355,19 +1355,19 @@ mod tests { #[derive(Debug, Serialize, Deserialize)] #[serde(untagged)] #[serde(bound = "T: EthSpec")] -pub enum BlockContents> { - BlockAndBlobSidecars(BeaconBlockAndBlobSidecars), - BlindedBlockAndBlobSidecars(BlindedBeaconBlockAndBlobSidecars), - Block(BeaconBlock), +pub enum BlockContents> { + BlockAndBlobSidecars(BeaconBlockAndBlobSidecars), + BlindedBlockAndBlobSidecars(BlindedBeaconBlockAndBlobSidecars), + Block(BeaconBlock), } -pub type BlockContentsTuple = ( - BeaconBlock>::Payload>, - Option>::Sidecar>>, +pub type BlockContentsTuple = ( + BeaconBlock, + Option>::Sidecar>>, ); -impl> BlockContents { - pub fn block(&self) -> &BeaconBlock { +impl> BlockContents { + pub fn block(&self) -> &BeaconBlock { match self { BlockContents::BlockAndBlobSidecars(block_and_sidecars) => &block_and_sidecars.block, BlockContents::BlindedBlockAndBlobSidecars(block_and_sidecars) => { @@ -1377,7 +1377,7 @@ impl> BlockContents { } } - pub fn deconstruct(self) -> BlockContentsTuple { + pub fn deconstruct(self) -> BlockContentsTuple { match self { BlockContents::BlockAndBlobSidecars(block_and_sidecars) => ( block_and_sidecars.block, @@ -1392,7 +1392,9 @@ impl> BlockContents { } } -impl> ForkVersionDeserialize for BlockContents { +impl> ForkVersionDeserialize + for BlockContents +{ fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( value: serde_json::value::Value, fork_name: ForkName, @@ -1411,8 +1413,10 @@ impl> ForkVersionDeserialize for BlockContents> Into> for BlockContents { - fn into(self) -> BeaconBlock { +impl> Into> + for BlockContents +{ + fn into(self) -> BeaconBlock { match self { Self::BlockAndBlobSidecars(block_and_sidecars) => block_and_sidecars.block, Self::BlindedBlockAndBlobSidecars(block_and_sidecars) => { @@ -1423,27 +1427,27 @@ impl> Into> for Block } } -pub type SignedBlockContentsTuple = ( - SignedBeaconBlock>::Payload>, - Option>::Sidecar>>, +pub type SignedBlockContentsTuple = ( + SignedBeaconBlock, + Option>::Sidecar>>, ); /// A wrapper over a [`SignedBeaconBlock`] or a [`SignedBeaconBlockAndBlobSidecars`]. #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(untagged)] #[serde(bound = "T: EthSpec")] -pub enum SignedBlockContents> { - BlockAndBlobSidecars(SignedBeaconBlockAndBlobSidecars), - BlindedBlockAndBlobSidecars(SignedBlindedBeaconBlockAndBlobSidecars), - Block(SignedBeaconBlock), +pub enum SignedBlockContents = FullPayload> { + BlockAndBlobSidecars(SignedBeaconBlockAndBlobSidecars), + BlindedBlockAndBlobSidecars(SignedBlindedBeaconBlockAndBlobSidecars), + Block(SignedBeaconBlock), } -impl> SignedBlockContents { +impl> SignedBlockContents { pub fn new( - block: SignedBeaconBlock, - blobs: Option>, + block: SignedBeaconBlock, + blobs: Option>, ) -> Self { - match (B::block_type(), blobs) { + match (Payload::block_type(), blobs) { (BlockType::Blinded, Some(blobs)) => { Self::BlockAndBlobSidecars(SignedBeaconBlockAndBlobSidecars { signed_block: block, @@ -1460,7 +1464,7 @@ impl> SignedBlockContents { } } - pub fn signed_block(&self) -> &SignedBeaconBlock { + pub fn signed_block(&self) -> &SignedBeaconBlock { match self { SignedBlockContents::BlockAndBlobSidecars(block_and_sidecars) => { &block_and_sidecars.signed_block @@ -1472,7 +1476,7 @@ impl> SignedBlockContents { } } - pub fn blobs_cloned(&self) -> Option> { + pub fn blobs_cloned(&self) -> Option> { match self { SignedBlockContents::BlockAndBlobSidecars(block_and_sidecars) => { Some(block_and_sidecars.signed_blob_sidecars.clone()) @@ -1482,7 +1486,7 @@ impl> SignedBlockContents { } } - pub fn deconstruct(self) -> SignedBlockContentsTuple { + pub fn deconstruct(self) -> SignedBlockContentsTuple { match self { SignedBlockContents::BlockAndBlobSidecars(block_and_sidecars) => ( block_and_sidecars.signed_block, @@ -1497,11 +1501,11 @@ impl> SignedBlockContents { } } -impl SignedBlockContents { +impl SignedBlockContents> { pub fn try_into_full_block_and_blobs( self, maybe_full_payload_contents: Option>, - ) -> Option> { + ) -> Option>> { match self { SignedBlockContents::BlindedBlockAndBlobSidecars(blinded_block_and_blob_sidecars) => { maybe_full_payload_contents.and_then(|full_payload_contents| { @@ -1547,10 +1551,10 @@ impl SignedBlockContents { } } -impl> From> - for SignedBlockContents +impl> From> + for SignedBlockContents { - fn from(block: SignedBeaconBlock) -> Self { + fn from(block: SignedBeaconBlock) -> Self { match block { SignedBeaconBlock::Base(_) | SignedBeaconBlock::Altair(_) @@ -1562,13 +1566,13 @@ impl> From> } } -impl> From> - for SignedBlockContents +impl> From> + for SignedBlockContents { - fn from(block_contents_tuple: SignedBlockContentsTuple) -> Self { + fn from(block_contents_tuple: SignedBlockContentsTuple) -> Self { match block_contents_tuple { (signed_block, None) => SignedBlockContents::Block(signed_block), - (signed_block, Some(signed_blob_sidecars)) => match B::Payload::block_type() { + (signed_block, Some(signed_blob_sidecars)) => match Payload::block_type() { BlockType::Blinded => SignedBlockContents::BlindedBlockAndBlobSidecars( SignedBlindedBeaconBlockAndBlobSidecars { signed_blinded_block: signed_block, @@ -1588,19 +1592,21 @@ impl> From> #[derive(Debug, Clone, Serialize, Deserialize, Encode)] #[serde(bound = "T: EthSpec")] -pub struct SignedBeaconBlockAndBlobSidecars = FullBlockProposal> { - pub signed_block: SignedBeaconBlock, - pub signed_blob_sidecars: SignedSidecarList, +pub struct SignedBeaconBlockAndBlobSidecars> { + pub signed_block: SignedBeaconBlock, + pub signed_blob_sidecars: SignedSidecarList, } #[derive(Debug, Clone, Serialize, Deserialize, Encode)] -#[serde(bound = "T: EthSpec, B: BlockProposal")] -pub struct BeaconBlockAndBlobSidecars = FullBlockProposal> { - pub block: BeaconBlock, - pub blob_sidecars: SidecarList, +#[serde(bound = "T: EthSpec, Payload: AbstractExecPayload")] +pub struct BeaconBlockAndBlobSidecars> { + pub block: BeaconBlock, + pub blob_sidecars: SidecarList, } -impl> ForkVersionDeserialize for BeaconBlockAndBlobSidecars { +impl> ForkVersionDeserialize + for BeaconBlockAndBlobSidecars +{ fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( value: serde_json::value::Value, fork_name: ForkName, @@ -1611,7 +1617,7 @@ impl> ForkVersionDeserialize for BeaconBlockAndB block: serde_json::Value, blob_sidecars: SidecarList, } - let helper: Helper = + let helper: Helper = serde_json::from_value(value).map_err(serde::de::Error::custom)?; Ok(Self { @@ -1625,22 +1631,24 @@ impl> ForkVersionDeserialize for BeaconBlockAndB #[serde(bound = "T: EthSpec")] pub struct SignedBlindedBeaconBlockAndBlobSidecars< T: EthSpec, - B: BlockProposal = BlindedBlockProposal, + Payload: AbstractExecPayload = BlindedPayload, > { - pub signed_blinded_block: SignedBeaconBlock, - pub signed_blinded_blob_sidecars: SignedSidecarList, + pub signed_blinded_block: SignedBeaconBlock, + pub signed_blinded_blob_sidecars: SignedSidecarList, } #[derive(Debug, Clone, Serialize, Deserialize, Encode)] -#[serde(bound = "T: EthSpec, B: BlockProposal")] -pub struct BlindedBeaconBlockAndBlobSidecars = BlindedBlockProposal> -{ - pub blinded_block: BeaconBlock, - pub blinded_blob_sidecars: SidecarList, +#[serde(bound = "T: EthSpec, Payload: AbstractExecPayload")] +pub struct BlindedBeaconBlockAndBlobSidecars< + T: EthSpec, + Payload: AbstractExecPayload = BlindedPayload, +> { + pub blinded_block: BeaconBlock, + pub blinded_blob_sidecars: SidecarList, } -impl> ForkVersionDeserialize - for BlindedBeaconBlockAndBlobSidecars +impl> ForkVersionDeserialize + for BlindedBeaconBlockAndBlobSidecars { fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( value: serde_json::value::Value, @@ -1652,7 +1660,7 @@ impl> ForkVersionDeserialize blinded_block: serde_json::Value, blinded_blob_sidecars: SidecarList, } - let helper: Helper = + let helper: Helper = serde_json::from_value(value).map_err(serde::de::Error::custom)?; Ok(Self { diff --git a/consensus/types/src/block_proposal.rs b/consensus/types/src/block_proposal.rs deleted file mode 100644 index 4fec7eabfa4..00000000000 --- a/consensus/types/src/block_proposal.rs +++ /dev/null @@ -1,33 +0,0 @@ -use crate::{ - AbstractExecPayload, BlindedBlobSidecar, BlindedPayload, BlobSidecar, BlockType, EthSpec, - FullPayload, Sidecar, -}; -use std::fmt::Debug; - -pub trait BlockProposal: Send + Sized + Clone + Debug { - type Payload: AbstractExecPayload; - type Sidecar: Sidecar; - fn block_type() -> BlockType; -} - -#[derive(Clone, Debug)] -pub struct FullBlockProposal {} -impl BlockProposal for FullBlockProposal { - type Payload = FullPayload; - type Sidecar = BlobSidecar; - - fn block_type() -> BlockType { - BlockType::Full - } -} - -#[derive(Clone, Debug)] -pub struct BlindedBlockProposal {} -impl BlockProposal for BlindedBlockProposal { - type Payload = BlindedPayload; - type Sidecar = BlindedBlobSidecar; - - fn block_type() -> BlockType { - BlockType::Blinded - } -} diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index a869b1d9327..dbbf7280c27 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -98,7 +98,6 @@ pub mod slot_data; pub mod sqlite; pub mod blob_sidecar; -pub mod block_proposal; pub mod signed_blob; use ethereum_types::{H160, H256}; @@ -123,7 +122,6 @@ pub use crate::blob_sidecar::{ BlindedBlobSidecar, BlindedBlobSidecarList, BlobRoots, BlobSidecar, BlobSidecarList, Blobs, BlobsOrBlobRoots, Sidecar, SidecarList, SidecarListVariant, }; -pub use crate::block_proposal::{BlindedBlockProposal, BlockProposal, FullBlockProposal}; pub use crate::bls_to_execution_change::BlsToExecutionChange; pub use crate::chain_spec::{ChainSpec, Config, Domain}; pub use crate::checkpoint::Checkpoint; diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index e18dabe6f38..c81d1b74344 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -85,6 +85,8 @@ pub trait AbstractExecPayload: + TryInto + TryInto { + type Sidecar: Sidecar; + type Ref<'a>: ExecPayload + Copy + From<&'a Self::Merge> @@ -381,6 +383,7 @@ impl<'b, T: EthSpec> ExecPayload for FullPayloadRef<'b, T> { } impl AbstractExecPayload for FullPayload { + type Sidecar = BlobSidecar; type Ref<'a> = FullPayloadRef<'a, T>; type Merge = FullPayloadMerge; type Capella = FullPayloadCapella; @@ -894,6 +897,7 @@ impl_exec_payload_for_fork!( ); impl AbstractExecPayload for BlindedPayload { + type Sidecar = BlindedBlobSidecar; type Ref<'a> = BlindedPayloadRef<'a, T>; type Merge = BlindedPayloadMerge; type Capella = BlindedPayloadCapella; diff --git a/testing/web3signer_tests/src/lib.rs b/testing/web3signer_tests/src/lib.rs index b0bbc0d6897..dd17ae23b15 100644 --- a/testing/web3signer_tests/src/lib.rs +++ b/testing/web3signer_tests/src/lib.rs @@ -512,7 +512,7 @@ mod tests { let block = BeaconBlock::Base(BeaconBlockBase::empty(spec)); let block_slot = block.slot(); validator_store - .sign_block::(pubkey, block, block_slot) + .sign_block(pubkey, block, block_slot) .await .unwrap() }) @@ -576,11 +576,7 @@ mod tests { let mut altair_block = BeaconBlockAltair::empty(spec); altair_block.slot = altair_fork_slot; validator_store - .sign_block::( - pubkey, - BeaconBlock::Altair(altair_block), - altair_fork_slot, - ) + .sign_block(pubkey, BeaconBlock::Altair(altair_block), altair_fork_slot) .await .unwrap() }, @@ -665,11 +661,7 @@ mod tests { let mut merge_block = BeaconBlockMerge::empty(spec); merge_block.slot = merge_fork_slot; validator_store - .sign_block::( - pubkey, - BeaconBlock::Merge(merge_block), - merge_fork_slot, - ) + .sign_block(pubkey, BeaconBlock::Merge(merge_block), merge_fork_slot) .await .unwrap() }) diff --git a/validator_client/src/block_service.rs b/validator_client/src/block_service.rs index 788275232fd..49ef677c2b7 100644 --- a/validator_client/src/block_service.rs +++ b/validator_client/src/block_service.rs @@ -20,8 +20,8 @@ use std::time::Duration; use tokio::sync::mpsc; use tokio::time::sleep; use types::{ - BlindedBlockProposal, BlockProposal, BlockType, EthSpec, ExecPayload, FullBlockProposal, - Graffiti, PublicKeyBytes, Slot, + AbstractExecPayload, BlindedPayload, BlockType, EthSpec, FullPayload, Graffiti, PublicKeyBytes, + Slot, }; #[derive(Debug)] @@ -343,7 +343,7 @@ impl BlockService { if builder_proposals { let result = service .clone() - .publish_block::(slot, validator_pubkey) + .publish_block::>(slot, validator_pubkey) .await; match result { Err(BlockError::Recoverable(e)) => { @@ -355,7 +355,7 @@ impl BlockService { "info" => "blinded proposal failed, attempting full block" ); if let Err(e) = service - .publish_block::(slot, validator_pubkey) + .publish_block::>(slot, validator_pubkey) .await { // Log a `crit` since a full block @@ -384,7 +384,7 @@ impl BlockService { Ok(_) => {} }; } else if let Err(e) = service - .publish_block::(slot, validator_pubkey) + .publish_block::>(slot, validator_pubkey) .await { // Log a `crit` since a full block (non-builder) @@ -406,7 +406,7 @@ impl BlockService { } /// Produce a block at the given slot for validator_pubkey - async fn publish_block>( + async fn publish_block>( self, slot: Slot, validator_pubkey: PublicKeyBytes, @@ -458,7 +458,7 @@ impl BlockService { // // Try the proposer nodes last, since it's likely that they don't have a // great view of attestations on the network. - let block_contents: BlockContents = proposer_fallback + let block_contents = proposer_fallback .first_success_try_proposers_last( RequireSynced::No, OfflineOnFailure::Yes, @@ -480,7 +480,7 @@ impl BlockService { let signed_block = self_ref .validator_store - .sign_block::(*validator_pubkey_ref, block, current_slot) + .sign_block::(*validator_pubkey_ref, block, current_slot) .await .map_err(|e| BlockError::Recoverable(format!("Unable to sign block: {:?}", e)))?; @@ -488,7 +488,7 @@ impl BlockService { Some(blob_sidecars) => Some( self_ref .validator_store - .sign_blobs::(*validator_pubkey_ref, blob_sidecars) + .sign_blobs::(*validator_pubkey_ref, blob_sidecars) .await .map_err(|e| { BlockError::Recoverable(format!("Unable to sign blob: {:?}", e)) @@ -519,8 +519,11 @@ impl BlockService { RequireSynced::No, OfflineOnFailure::Yes, |beacon_node| async { - Self::publish_signed_block_contents::(&signed_block_contents, beacon_node) - .await + Self::publish_signed_block_contents::( + &signed_block_contents, + beacon_node, + ) + .await }, ) .await?; @@ -528,7 +531,7 @@ impl BlockService { info!( log, "Successfully published block"; - "block_type" => ?B::Payload::block_type(), + "block_type" => ?Payload::block_type(), "deposits" => signed_block_contents.signed_block().message().body().deposits().len(), "attestations" => signed_block_contents.signed_block().message().body().attestations().len(), "graffiti" => ?graffiti.map(|g| g.as_utf8_lossy()), @@ -538,11 +541,11 @@ impl BlockService { Ok(()) } - async fn publish_signed_block_contents>( - signed_block_contents: &SignedBlockContents, + async fn publish_signed_block_contents>( + signed_block_contents: &SignedBlockContents, beacon_node: &BeaconNodeHttpClient, ) -> Result<(), BlockError> { - match B::Payload::block_type() { + match Payload::block_type() { BlockType::Full => { let _post_timer = metrics::start_timer_vec( &metrics::BLOCK_SERVICE_TIMES, @@ -577,22 +580,22 @@ impl BlockService { Ok::<_, BlockError>(()) } - async fn get_validator_block>( + async fn get_validator_block>( beacon_node: &BeaconNodeHttpClient, slot: Slot, randao_reveal_ref: &SignatureBytes, graffiti: Option, proposer_index: Option, log: &Logger, - ) -> Result, BlockError> { - let block_contents: BlockContents = match B::Payload::block_type() { + ) -> Result, BlockError> { + let block_contents: BlockContents = match Payload::block_type() { BlockType::Full => { let _get_timer = metrics::start_timer_vec( &metrics::BLOCK_SERVICE_TIMES, &[metrics::BEACON_BLOCK_HTTP_GET], ); beacon_node - .get_validator_blocks(slot, randao_reveal_ref, graffiti.as_ref()) + .get_validator_blocks::(slot, randao_reveal_ref, graffiti.as_ref()) .await .map_err(|e| { BlockError::Recoverable(format!( @@ -608,7 +611,11 @@ impl BlockService { &[metrics::BLINDED_BEACON_BLOCK_HTTP_GET], ); beacon_node - .get_validator_blinded_blocks(slot, randao_reveal_ref, graffiti.as_ref()) + .get_validator_blinded_blocks::( + slot, + randao_reveal_ref, + graffiti.as_ref(), + ) .await .map_err(|e| { BlockError::Recoverable(format!( diff --git a/validator_client/src/signing_method.rs b/validator_client/src/signing_method.rs index 660b6302657..96bfd2511f1 100644 --- a/validator_client/src/signing_method.rs +++ b/validator_client/src/signing_method.rs @@ -34,10 +34,10 @@ pub enum Error { } /// Enumerates all messages that can be signed by a validator. -pub enum SignableMessage<'a, T: EthSpec, B: BlockProposal = FullBlockProposal> { +pub enum SignableMessage<'a, T: EthSpec, Payload: AbstractExecPayload = FullPayload> { RandaoReveal(Epoch), - BeaconBlock(&'a BeaconBlock), - BlobSidecar(&'a B::Sidecar), + BeaconBlock(&'a BeaconBlock), + BlobSidecar(&'a Payload::Sidecar), AttestationData(&'a AttestationData), SignedAggregateAndProof(&'a AggregateAndProof), SelectionProof(Slot), @@ -51,7 +51,7 @@ pub enum SignableMessage<'a, T: EthSpec, B: BlockProposal = FullBlockProposal VoluntaryExit(&'a VoluntaryExit), } -impl<'a, T: EthSpec, B: BlockProposal> SignableMessage<'a, T, B> { +impl<'a, T: EthSpec, Payload: AbstractExecPayload> SignableMessage<'a, T, Payload> { /// Returns the `SignedRoot` for the contained message. /// /// The actual `SignedRoot` trait is not used since it also requires a `TreeHash` impl, which is @@ -120,9 +120,9 @@ impl SigningContext { impl SigningMethod { /// Return the signature of `signable_message`, with respect to the `signing_context`. - pub async fn get_signature>( + pub async fn get_signature>( &self, - signable_message: SignableMessage<'_, T, B>, + signable_message: SignableMessage<'_, T, Payload>, signing_context: SigningContext, spec: &ChainSpec, executor: &TaskExecutor, @@ -145,9 +145,9 @@ impl SigningMethod { .await } - pub async fn get_signature_from_root>( + pub async fn get_signature_from_root>( &self, - signable_message: SignableMessage<'_, T, B>, + signable_message: SignableMessage<'_, T, Payload>, signing_root: Hash256, executor: &TaskExecutor, fork_info: Option, diff --git a/validator_client/src/validator_store.rs b/validator_client/src/validator_store.rs index 69a2a3f2d81..939bc737cdd 100644 --- a/validator_client/src/validator_store.rs +++ b/validator_client/src/validator_store.rs @@ -19,8 +19,8 @@ use std::path::Path; use std::sync::Arc; use task_executor::TaskExecutor; use types::{ - attestation::Error as AttestationError, graffiti::GraffitiString, Address, AggregateAndProof, - Attestation, BeaconBlock, BlindedBlockProposal, BlockProposal, ChainSpec, ContributionAndProof, + attestation::Error as AttestationError, graffiti::GraffitiString, AbstractExecPayload, Address, + AggregateAndProof, Attestation, BeaconBlock, BlindedPayload, ChainSpec, ContributionAndProof, Domain, Epoch, EthSpec, Fork, Graffiti, Hash256, Keypair, PublicKeyBytes, SelectionProof, Sidecar, SidecarList, Signature, SignedAggregateAndProof, SignedBeaconBlock, SignedContributionAndProof, SignedRoot, SignedSidecar, SignedSidecarList, @@ -388,7 +388,7 @@ impl ValidatorStore { let signing_context = self.signing_context(Domain::Randao, signing_epoch); let signature = signing_method - .get_signature::( + .get_signature::>( SignableMessage::RandaoReveal(signing_epoch), signing_context, &self.spec, @@ -465,12 +465,12 @@ impl ValidatorStore { .unwrap_or(self.builder_proposals) } - pub async fn sign_block>( + pub async fn sign_block>( &self, validator_pubkey: PublicKeyBytes, - block: BeaconBlock, + block: BeaconBlock, current_slot: Slot, - ) -> Result, Error> { + ) -> Result, Error> { // Make sure the block slot is not higher than the current slot to avoid potential attacks. if block.slot() > current_slot { warn!( @@ -503,7 +503,7 @@ impl ValidatorStore { let signing_method = self.doppelganger_checked_signing_method(validator_pubkey)?; let signature = signing_method - .get_signature::( + .get_signature::( SignableMessage::BeaconBlock(&block), signing_context, &self.spec, @@ -542,11 +542,11 @@ impl ValidatorStore { } } - pub async fn sign_blobs>( + pub async fn sign_blobs>( &self, validator_pubkey: PublicKeyBytes, - blob_sidecars: SidecarList, - ) -> Result, Error> { + blob_sidecars: SidecarList, + ) -> Result, Error> { let mut signed_blob_sidecars = Vec::new(); for blob_sidecar in blob_sidecars.into_iter() { let slot = blob_sidecar.slot(); @@ -555,7 +555,7 @@ impl ValidatorStore { let signing_method = self.doppelganger_checked_signing_method(validator_pubkey)?; let signature = signing_method - .get_signature::( + .get_signature::( SignableMessage::BlobSidecar(blob_sidecar.as_ref()), signing_context, &self.spec, @@ -605,7 +605,7 @@ impl ValidatorStore { Ok(Safe::Valid) => { let signing_method = self.doppelganger_checked_signing_method(validator_pubkey)?; let signature = signing_method - .get_signature::( + .get_signature::>( SignableMessage::AttestationData(&attestation.data), signing_context, &self.spec, @@ -670,7 +670,7 @@ impl ValidatorStore { let signing_method = self.doppelganger_bypassed_signing_method(validator_pubkey)?; let signature = signing_method - .get_signature::( + .get_signature::>( SignableMessage::VoluntaryExit(&voluntary_exit), signing_context, &self.spec, @@ -696,7 +696,7 @@ impl ValidatorStore { let signing_method = self.doppelganger_bypassed_signing_method(validator_registration_data.pubkey)?; let signature = signing_method - .get_signature_from_root::( + .get_signature_from_root::>( SignableMessage::ValidatorRegistration(&validator_registration_data), signing_root, &self.task_executor, @@ -737,7 +737,7 @@ impl ValidatorStore { let signing_method = self.doppelganger_checked_signing_method(validator_pubkey)?; let signature = signing_method - .get_signature::( + .get_signature::>( SignableMessage::SignedAggregateAndProof(&message), signing_context, &self.spec, @@ -770,7 +770,7 @@ impl ValidatorStore { let signing_method = self.doppelganger_bypassed_signing_method(validator_pubkey)?; let signature = signing_method - .get_signature::( + .get_signature::>( SignableMessage::SelectionProof(slot), signing_context, &self.spec, @@ -809,7 +809,7 @@ impl ValidatorStore { }; let signature = signing_method - .get_signature::( + .get_signature::>( SignableMessage::SyncSelectionProof(&message), signing_context, &self.spec, @@ -835,7 +835,7 @@ impl ValidatorStore { let signing_method = self.doppelganger_bypassed_signing_method(*validator_pubkey)?; let signature = signing_method - .get_signature::( + .get_signature::>( SignableMessage::SyncCommitteeSignature { beacon_block_root, slot, @@ -880,7 +880,7 @@ impl ValidatorStore { }; let signature = signing_method - .get_signature::( + .get_signature::>( SignableMessage::SignedContributionAndProof(&message), signing_context, &self.spec, From 9fec52aec3b362a85fa43f64c7206bb992b13cd1 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Thu, 3 Aug 2023 17:09:14 -0400 Subject: [PATCH 15/24] add blob roots associated type --- beacon_node/beacon_chain/src/beacon_chain.rs | 118 +------------ beacon_node/beacon_chain/src/test_utils.rs | 4 +- beacon_node/execution_layer/src/lib.rs | 14 +- .../http_api/src/build_block_contents.rs | 13 +- common/eth2/src/types.rs | 20 +-- consensus/types/src/blob_sidecar.rs | 160 ++++++++++++++---- consensus/types/src/lib.rs | 2 +- consensus/types/src/payload.rs | 20 ++- validator_client/src/signing_method.rs | 2 +- validator_client/src/validator_store.rs | 4 +- 10 files changed, 186 insertions(+), 171 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index bea17df634d..3010e4764ce 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -115,9 +115,8 @@ use store::{ }; use task_executor::{ShutdownReason, TaskExecutor}; use tokio_stream::Stream; -use tree_hash::TreeHash; -use types::beacon_block_body::KzgCommitments; use types::beacon_state::CloneConfig; +use types::blob_sidecar::SidecarLess; use types::consts::deneb::MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS; use types::*; @@ -483,7 +482,7 @@ pub struct BeaconChain { type BeaconBlockAndState = ( BeaconBlock, BeaconState, - Option>, + Option>::BlobSidecar>>, ); impl FinalizationAndCanonicity { @@ -4970,23 +4969,11 @@ impl BeaconChain { } let kzg_proofs = Vec::from(proofs); - - match blobs_or_blobs_roots { - BlobsOrBlobRoots::Blobs(blobs) => Some(self.build_blob_sidecars( - &block, - blobs, - expected_kzg_commitments, - kzg_proofs, - )?), - BlobsOrBlobRoots::BlobRoots(blob_roots) => { - Some(self.build_blinded_blob_sidecars( - &block, - blob_roots, - expected_kzg_commitments, - kzg_proofs, - )?) - } - } + Some(blobs_or_blobs_roots.to_sidecar( + &block, + expected_kzg_commitments, + kzg_proofs, + )?) } _ => None, }; @@ -6259,97 +6246,6 @@ impl BeaconChain { .map(|fork_epoch| fork_epoch <= current_epoch) .unwrap_or(false)) } - - fn build_blob_sidecars>( - &self, - block: &BeaconBlock, - blobs: Blobs, - expected_kzg_commitments: &KzgCommitments, - kzg_proofs: Vec, - ) -> Result, BlockProductionError> { - let kzg = self - .kzg - .as_ref() - .ok_or(BlockProductionError::TrustedSetupNotInitialized)?; - let beacon_block_root = block.canonical_root(); - let slot = block.slot(); - - kzg_utils::validate_blobs::(kzg, expected_kzg_commitments, &blobs, &kzg_proofs) - .map_err(BlockProductionError::KzgError)?; - - let blob_sidecars = BlobSidecarList::from( - blobs - .into_iter() - .enumerate() - .map(|(blob_index, blob)| { - let kzg_commitment = expected_kzg_commitments - .get(blob_index) - .expect("KZG commitment should exist for blob"); - - let kzg_proof = kzg_proofs - .get(blob_index) - .expect("KZG proof should exist for blob"); - - Ok(Arc::new(BlobSidecar { - block_root: beacon_block_root, - index: blob_index as u64, - slot, - block_parent_root: block.parent_root(), - proposer_index: block.proposer_index(), - blob, - kzg_commitment: *kzg_commitment, - kzg_proof: *kzg_proof, - })) - }) - .collect::, BlockProductionError>>()?, - ); - - Ok(SidecarListVariant::Full(blob_sidecars)) - } - - fn build_blinded_blob_sidecars>( - &self, - block: &BeaconBlock, - blob_roots: BlobRoots, - expected_kzg_commitments: &KzgCommitments, - kzg_proofs: Vec, - ) -> Result, BlockProductionError> { - let beacon_block_root = block.canonical_root(); - let slot = block.slot(); - - let blob_sidecars = BlindedBlobSidecarList::::from( - blob_roots - .into_iter() - .enumerate() - .map(|(blob_index, blob_root)| { - let kzg_commitment = expected_kzg_commitments - .get(blob_index) - .expect("KZG commitment should exist for blob"); - - let kzg_proof = - kzg_proofs - .get(blob_index) - .ok_or(BlockProductionError::MissingKzgProof(format!( - "Missing KZG proof for slot {} blob index: {}", - slot, blob_index - )))?; - - Ok(Arc::new(BlindedBlobSidecar { - block_root: beacon_block_root, - index: blob_index as u64, - slot, - block_parent_root: block.parent_root(), - proposer_index: block.proposer_index(), - blob_root, - kzg_commitment: *kzg_commitment, - kzg_proof: *kzg_proof, - })) - }) - .collect::, BlockProductionError>>()?, - ); - - Ok(SidecarListVariant::Blinded(blob_sidecars)) - } } impl Drop for BeaconChain { diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 29afb0e2107..68fe389fff4 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -891,7 +891,7 @@ where | SignedBeaconBlock::Merge(_) | SignedBeaconBlock::Capella(_) => (signed_block, None), SignedBeaconBlock::Deneb(_) => { - if let Some(SidecarListVariant::Full(blobs)) = maybe_blob_sidecars { + if let Some(blobs) = maybe_blob_sidecars { let signed_blobs: SignedSidecarList> = Vec::from(blobs) .into_iter() .map(|blob| { @@ -973,7 +973,7 @@ where | SignedBeaconBlock::Merge(_) | SignedBeaconBlock::Capella(_) => (signed_block, None), SignedBeaconBlock::Deneb(_) => { - if let Some(SidecarListVariant::Full(blobs)) = maybe_blob_sidecars { + if let Some(blobs) = maybe_blob_sidecars { let signed_blobs: SignedSidecarList> = Vec::from(blobs) .into_iter() .map(|blob| { diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 4c9ad744b61..c70ad71ed0a 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -41,15 +41,15 @@ use tokio::{ use tokio_stream::wrappers::WatchStream; use tree_hash::TreeHash; use types::beacon_block_body::KzgCommitments; -use types::blob_sidecar::BlobsOrBlobRoots; +use types::blob_sidecar::SidecarLess; use types::builder_bid::BuilderBid; -use types::KzgProofs; use types::{ AbstractExecPayload, BeaconStateError, ExecPayload, ExecutionPayloadDeneb, VersionedHash, }; use types::{ BlindedPayload, BlockType, ChainSpec, Epoch, ExecutionPayloadCapella, ExecutionPayloadMerge, }; +use types::{KzgProofs, Sidecar, }; use types::{ProposerPreparationData, PublicKeyBytes, Signature, Slot, Transaction}; mod block_hash; @@ -108,7 +108,7 @@ impl> From> payload: builder_bid.header, block_value: builder_bid.value, kzg_commitments: builder_bid.blinded_blobs_bundle.commitments, - blobs: BlobsOrBlobRoots::BlobRoots(builder_bid.blinded_blobs_bundle.blob_roots), + blobs: SidecarLess::from_blob_roots(builder_bid.blinded_blobs_bundle.blob_roots), proofs: builder_bid.blinded_blobs_bundle.proofs, }, }; @@ -161,7 +161,7 @@ pub enum BlockProposalContents> { payload: Payload, block_value: Uint256, kzg_commitments: KzgCommitments, - blobs: BlobsOrBlobRoots, + blobs: Payload::Blob, proofs: KzgProofs, }, } @@ -176,7 +176,7 @@ impl> From> payload: execution_payload.into(), block_value, kzg_commitments: bundle.commitments, - blobs: BlobsOrBlobRoots::Blobs(bundle.blobs), + blobs: SidecarLess::from_blobs(bundle.blobs), proofs: bundle.proofs, }, None => Self::Payload { @@ -194,7 +194,7 @@ impl> BlockProposalContents ( Payload, Option>, - Option>, + Option, Option>, ) { match self { @@ -241,7 +241,7 @@ impl> BlockProposalContents BlockProposalContents::PayloadAndBlobs { payload: Payload::default_at_fork(fork_name)?, block_value: Uint256::zero(), - blobs: BlobsOrBlobRoots::Blobs(VariableList::default()), + blobs: Payload::default_blobs_at_fork(fork_name)?, kzg_commitments: VariableList::default(), proofs: VariableList::default(), }, diff --git a/beacon_node/http_api/src/build_block_contents.rs b/beacon_node/http_api/src/build_block_contents.rs index 9a73e645f0d..c8f28fa9ae3 100644 --- a/beacon_node/http_api/src/build_block_contents.rs +++ b/beacon_node/http_api/src/build_block_contents.rs @@ -1,6 +1,9 @@ use beacon_chain::BlockProductionError; use eth2::types::{BeaconBlockAndBlobSidecars, BlindedBeaconBlockAndBlobSidecars, BlockContents}; -use types::{BeaconBlock, BlindedPayload, EthSpec, ForkName, FullPayload, SidecarListVariant}; +use types::{ + BeaconBlock, BlindedBlobSidecarList, BlindedPayload, BlobSidecarList, EthSpec, ForkName, + FullPayload, +}; type Error = warp::reject::Rejection; type FullBlockContents = BlockContents>; @@ -9,14 +12,14 @@ type BlindedBlockContents = BlockContents>; pub fn build_block_contents( fork_name: ForkName, block: BeaconBlock>, - maybe_blobs: Option>, + maybe_blobs: Option>, ) -> Result, Error> { match fork_name { ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { Ok(BlockContents::Block(block)) } ForkName::Deneb => { - if let Some(SidecarListVariant::Full(blob_sidecars)) = maybe_blobs { + if let Some(blob_sidecars) = maybe_blobs { let block_and_blobs = BeaconBlockAndBlobSidecars { block, blob_sidecars, @@ -35,14 +38,14 @@ pub fn build_block_contents( pub fn build_blinded_block_contents( fork_name: ForkName, block: BeaconBlock>, - maybe_blobs: Option>, + maybe_blobs: Option>, ) -> Result, Error> { match fork_name { ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { Ok(BlockContents::Block(block)) } ForkName::Deneb => { - if let Some(SidecarListVariant::Blinded(blinded_blob_sidecars)) = maybe_blobs { + if let Some(blinded_blob_sidecars) = maybe_blobs { let block_and_blobs = BlindedBeaconBlockAndBlobSidecars { blinded_block: block, blinded_blob_sidecars, diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 66cbbcc203d..71d33e8a938 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -1363,7 +1363,7 @@ pub enum BlockContents> { pub type BlockContentsTuple = ( BeaconBlock, - Option>::Sidecar>>, + Option>::BlobSidecar>>, ); impl> BlockContents { @@ -1429,7 +1429,7 @@ impl> Into> pub type SignedBlockContentsTuple = ( SignedBeaconBlock, - Option>::Sidecar>>, + Option>::BlobSidecar>>, ); /// A wrapper over a [`SignedBeaconBlock`] or a [`SignedBeaconBlockAndBlobSidecars`]. @@ -1445,7 +1445,7 @@ pub enum SignedBlockContents = FullP impl> SignedBlockContents { pub fn new( block: SignedBeaconBlock, - blobs: Option>, + blobs: Option>, ) -> Self { match (Payload::block_type(), blobs) { (BlockType::Blinded, Some(blobs)) => { @@ -1476,7 +1476,7 @@ impl> SignedBlockContents Option> { + pub fn blobs_cloned(&self) -> Option> { match self { SignedBlockContents::BlockAndBlobSidecars(block_and_sidecars) => { Some(block_and_sidecars.signed_blob_sidecars.clone()) @@ -1596,14 +1596,14 @@ impl> From> { pub signed_block: SignedBeaconBlock, - pub signed_blob_sidecars: SignedSidecarList, + pub signed_blob_sidecars: SignedSidecarList, } #[derive(Debug, Clone, Serialize, Deserialize, Encode)] #[serde(bound = "T: EthSpec, Payload: AbstractExecPayload")] pub struct BeaconBlockAndBlobSidecars> { pub block: BeaconBlock, - pub blob_sidecars: SidecarList, + pub blob_sidecars: SidecarList, } impl> ForkVersionDeserialize @@ -1619,7 +1619,7 @@ impl> ForkVersionDeserialize block: serde_json::Value, blob_sidecars: SidecarList, } - let helper: Helper = + let helper: Helper = serde_json::from_value(value).map_err(serde::de::Error::custom)?; Ok(Self { @@ -1636,7 +1636,7 @@ pub struct SignedBlindedBeaconBlockAndBlobSidecars< Payload: AbstractExecPayload = BlindedPayload, > { pub signed_blinded_block: SignedBeaconBlock, - pub signed_blinded_blob_sidecars: SignedSidecarList, + pub signed_blinded_blob_sidecars: SignedSidecarList, } #[derive(Debug, Clone, Serialize, Deserialize, Encode)] @@ -1646,7 +1646,7 @@ pub struct BlindedBeaconBlockAndBlobSidecars< Payload: AbstractExecPayload = BlindedPayload, > { pub blinded_block: BeaconBlock, - pub blinded_blob_sidecars: SidecarList, + pub blinded_blob_sidecars: SidecarList, } impl> ForkVersionDeserialize @@ -1662,7 +1662,7 @@ impl> ForkVersionDeserialize blinded_block: serde_json::Value, blinded_blob_sidecars: SidecarList, } - let helper: Helper = + let helper: Helper = serde_json::from_value(value).map_err(serde::de::Error::custom)?; Ok(Self { diff --git a/consensus/types/src/blob_sidecar.rs b/consensus/types/src/blob_sidecar.rs index 76967ecec62..62d1c90c946 100644 --- a/consensus/types/src/blob_sidecar.rs +++ b/consensus/types/src/blob_sidecar.rs @@ -17,13 +17,12 @@ use tree_hash_derive::TreeHash; use bls::SecretKey; use test_random_derive::TestRandom; +use crate::beacon_block_body::KzgCommitments; use crate::test_utils::TestRandom; -use crate::{Blob, ChainSpec, Domain, EthSpec, Fork, Hash256, SignedBlobSidecar, SignedRoot, Slot}; - -pub enum SidecarListVariant { - Full(BlobSidecarList), - Blinded(BlindedBlobSidecarList), -} +use crate::{ + AbstractExecPayload, BeaconBlock, Blob, ChainSpec, Domain, EthSpec, Fork, Hash256, + SignedBlobSidecar, SignedRoot, Slot, +}; pub trait Sidecar: serde::Serialize @@ -36,11 +35,137 @@ pub trait Sidecar: + TestRandom + Debug + SignedRoot + + Sync + + Send + for<'a> arbitrary::Arbitrary<'a> { fn slot(&self) -> Slot; } +pub trait SidecarLess: Sync + Send { + fn to_sidecar,P: AbstractExecPayload>( + self, + block: &BeaconBlock, + expected_kzg_commitments: &KzgCommitments, + kzg_proofs: Vec, + ) -> Result, String> ; + fn from_blob_roots(roots: BlobRoots) -> Self; + fn from_blobs(blobs: Blobs) -> Self; + fn len(&self) -> usize; + fn is_empty(&self) -> bool; +} + +impl SidecarLess for Blobs { + fn to_sidecar, P: AbstractExecPayload>( + self, + block: &BeaconBlock, + expected_kzg_commitments: &KzgCommitments, + kzg_proofs: Vec, + ) -> Result, String> { + let blobs = self; + let beacon_block_root = block.canonical_root(); + let slot = block.slot(); + + //TODO(sean) kzg verify + + let blob_sidecars = BlobSidecarList::from( + blobs + .into_iter() + .enumerate() + .map(|(blob_index, blob)| { + let kzg_commitment = expected_kzg_commitments + .get(blob_index) + .expect("KZG commitment should exist for blob"); + + let kzg_proof = kzg_proofs + .get(blob_index) + .expect("KZG proof should exist for blob"); + + Ok(Arc::new(BlobSidecar { + block_root: beacon_block_root, + index: blob_index as u64, + slot, + block_parent_root: block.parent_root(), + proposer_index: block.proposer_index(), + blob, + kzg_commitment: *kzg_commitment, + kzg_proof: *kzg_proof, + })) + }) + .collect::, String>>()?, + ); + + Ok(blob_sidecars) + } + fn from_blob_roots(roots: BlobRoots) -> Self { + todo!() + } + fn from_blobs(blobs: Blobs) -> Self { + todo!() + } + fn len(&self) -> usize { + VariableList::len(self) + } + fn is_empty(&self) -> bool { + VariableList::is_empty(self) + } +} + +impl SidecarLess for BlobRoots { + fn to_sidecar, P: AbstractExecPayload>( + self, + block: &BeaconBlock, + expected_kzg_commitments: &KzgCommitments, + kzg_proofs: Vec, + ) -> Result, String> { + let blob_roots = self; + let beacon_block_root = block.canonical_root(); + let slot = block.slot(); + + let blob_sidecars = BlindedBlobSidecarList::::from( + blob_roots + .into_iter() + .enumerate() + .map(|(blob_index, blob_root)| { + let kzg_commitment = expected_kzg_commitments + .get(blob_index) + .expect("KZG commitment should exist for blob"); + + let kzg_proof = kzg_proofs.get(blob_index).ok_or(format!( + "Missing KZG proof for slot {} blob index: {}", + slot, blob_index + ))?; + + Ok(Arc::new(BlindedBlobSidecar { + block_root: beacon_block_root, + index: blob_index as u64, + slot, + block_parent_root: block.parent_root(), + proposer_index: block.proposer_index(), + blob_root, + kzg_commitment: *kzg_commitment, + kzg_proof: *kzg_proof, + })) + }) + .collect::, String>>()?, + ); + + Ok(blob_sidecars) + } + fn from_blob_roots(roots: BlobRoots) -> Self { + todo!() + } + fn from_blobs(blobs: Blobs) -> Self { + todo!() + } + fn len(&self) -> usize { + VariableList::len(self) + } + fn is_empty(&self) -> bool { + VariableList::is_empty(self) + } +} + /// Container of the data that identifies an individual blob. #[derive( Serialize, Deserialize, Encode, Decode, TreeHash, Copy, Clone, Debug, PartialEq, Eq, Hash, @@ -248,26 +373,3 @@ pub type FixedBlobSidecarList = pub type Blobs = VariableList, ::MaxBlobsPerBlock>; pub type BlobRoots = VariableList::MaxBlobsPerBlock>; -#[derive(Debug, Serialize, Deserialize)] -#[serde(untagged)] -#[serde(bound = "T: EthSpec")] -pub enum BlobsOrBlobRoots { - Blobs(Blobs), - BlobRoots(BlobRoots), -} - -impl BlobsOrBlobRoots { - pub fn len(&self) -> usize { - match self { - Self::Blobs(blobs) => blobs.len(), - Self::BlobRoots(blob_roots) => blob_roots.len(), - } - } - - pub fn is_empty(&self) -> bool { - match self { - Self::Blobs(blobs) => blobs.is_empty(), - Self::BlobRoots(blob_roots) => blob_roots.is_empty(), - } - } -} diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index dbbf7280c27..0bff91156a3 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -120,7 +120,7 @@ pub use crate::beacon_committee::{BeaconCommittee, OwnedBeaconCommittee}; pub use crate::beacon_state::{BeaconTreeHashCache, Error as BeaconStateError, *}; pub use crate::blob_sidecar::{ BlindedBlobSidecar, BlindedBlobSidecarList, BlobRoots, BlobSidecar, BlobSidecarList, Blobs, - BlobsOrBlobRoots, Sidecar, SidecarList, SidecarListVariant, + Sidecar, SidecarList, }; pub use crate::bls_to_execution_change::BlsToExecutionChange; pub use crate::chain_spec::{ChainSpec, Config, Domain}; diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index c81d1b74344..7b30dac8737 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -13,6 +13,7 @@ use std::hash::Hash; use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; +use crate::blob_sidecar::SidecarLess; #[derive(Debug, PartialEq)] pub enum BlockType { @@ -85,7 +86,8 @@ pub trait AbstractExecPayload: + TryInto + TryInto { - type Sidecar: Sidecar; + type BlobSidecar: Sidecar; + type Blob: SidecarLess; type Ref<'a>: ExecPayload + Copy @@ -107,6 +109,9 @@ pub trait AbstractExecPayload: + TryFrom>; fn default_at_fork(fork_name: ForkName) -> Result; + fn default_blobs_at_fork( + fork_name: ForkName, + ) -> Result; } #[superstruct( @@ -383,7 +388,8 @@ impl<'b, T: EthSpec> ExecPayload for FullPayloadRef<'b, T> { } impl AbstractExecPayload for FullPayload { - type Sidecar = BlobSidecar; + type BlobSidecar = BlobSidecar; + type Blob = Blobs; type Ref<'a> = FullPayloadRef<'a, T>; type Merge = FullPayloadMerge; type Capella = FullPayloadCapella; @@ -397,6 +403,9 @@ impl AbstractExecPayload for FullPayload { ForkName::Deneb => Ok(FullPayloadDeneb::default().into()), } } + fn default_blobs_at_fork(fork_name: ForkName) -> Result, Error> { + Ok(VariableList::default()) + } } impl From> for FullPayload { @@ -897,12 +906,14 @@ impl_exec_payload_for_fork!( ); impl AbstractExecPayload for BlindedPayload { - type Sidecar = BlindedBlobSidecar; type Ref<'a> = BlindedPayloadRef<'a, T>; type Merge = BlindedPayloadMerge; type Capella = BlindedPayloadCapella; type Deneb = BlindedPayloadDeneb; + type BlobSidecar = BlindedBlobSidecar; + type Blob = BlobRoots; + fn default_at_fork(fork_name: ForkName) -> Result { match fork_name { ForkName::Base | ForkName::Altair => Err(Error::IncorrectStateVariant), @@ -911,6 +922,9 @@ impl AbstractExecPayload for BlindedPayload { ForkName::Deneb => Ok(BlindedPayloadDeneb::default().into()), } } + fn default_blobs_at_fork(fork_name: ForkName) -> Result, Error> { + Ok(VariableList::default()) + } } impl From> for BlindedPayload { diff --git a/validator_client/src/signing_method.rs b/validator_client/src/signing_method.rs index 96bfd2511f1..591d533fbef 100644 --- a/validator_client/src/signing_method.rs +++ b/validator_client/src/signing_method.rs @@ -37,7 +37,7 @@ pub enum Error { pub enum SignableMessage<'a, T: EthSpec, Payload: AbstractExecPayload = FullPayload> { RandaoReveal(Epoch), BeaconBlock(&'a BeaconBlock), - BlobSidecar(&'a Payload::Sidecar), + BlobSidecar(&'a Payload::BlobSidecar), AttestationData(&'a AttestationData), SignedAggregateAndProof(&'a AggregateAndProof), SelectionProof(Slot), diff --git a/validator_client/src/validator_store.rs b/validator_client/src/validator_store.rs index 939bc737cdd..e8595d96639 100644 --- a/validator_client/src/validator_store.rs +++ b/validator_client/src/validator_store.rs @@ -545,8 +545,8 @@ impl ValidatorStore { pub async fn sign_blobs>( &self, validator_pubkey: PublicKeyBytes, - blob_sidecars: SidecarList, - ) -> Result, Error> { + blob_sidecars: SidecarList, + ) -> Result, Error> { let mut signed_blob_sidecars = Vec::new(); for blob_sidecar in blob_sidecars.into_iter() { let slot = blob_sidecar.slot(); From 74a91c482d1442de47c2c5b3c905d1b9b260f346 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Sun, 6 Aug 2023 16:10:54 -0400 Subject: [PATCH 16/24] move raw blobs associated type to sidecar trait --- beacon_node/beacon_chain/src/beacon_chain.rs | 9 +- beacon_node/execution_layer/src/lib.rs | 10 +- common/eth2/src/types.rs | 20 +-- consensus/types/src/blob_sidecar.rs | 179 ++++++++++--------- consensus/types/src/payload.rs | 13 +- validator_client/src/signing_method.rs | 2 +- validator_client/src/validator_store.rs | 4 +- 7 files changed, 119 insertions(+), 118 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 3010e4764ce..5e3bebac3ca 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -9,6 +9,7 @@ use crate::beacon_proposer_cache::compute_proposer_duties_from_head; use crate::beacon_proposer_cache::BeaconProposerCache; use crate::blob_verification::{self, GossipBlobError, GossipVerifiedBlob}; use crate::block_times_cache::BlockTimesCache; +use tree_hash::TreeHash; use crate::block_verification::POS_PANDA_BANNER; use crate::block_verification::{ check_block_is_finalized_checkpoint_or_descendant, check_block_relevancy, get_block_root, @@ -116,7 +117,7 @@ use store::{ use task_executor::{ShutdownReason, TaskExecutor}; use tokio_stream::Stream; use types::beacon_state::CloneConfig; -use types::blob_sidecar::SidecarLess; +use types::blob_sidecar::RawBlobs; use types::consts::deneb::MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS; use types::*; @@ -482,7 +483,7 @@ pub struct BeaconChain { type BeaconBlockAndState = ( BeaconBlock, BeaconState, - Option>::BlobSidecar>>, + Option>::Sidecar>>, ); impl FinalizationAndCanonicity { @@ -4969,11 +4970,11 @@ impl BeaconChain { } let kzg_proofs = Vec::from(proofs); - Some(blobs_or_blobs_roots.to_sidecar( + Some(Sidecar::build_sidecar(blobs_or_blobs_roots, &block, expected_kzg_commitments, kzg_proofs, - )?) + ).unwrap()) //TODO: remove unwrap } _ => None, }; diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index c70ad71ed0a..24154ed5e95 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -41,7 +41,7 @@ use tokio::{ use tokio_stream::wrappers::WatchStream; use tree_hash::TreeHash; use types::beacon_block_body::KzgCommitments; -use types::blob_sidecar::SidecarLess; +use types::blob_sidecar::RawBlobs; use types::builder_bid::BuilderBid; use types::{ AbstractExecPayload, BeaconStateError, ExecPayload, ExecutionPayloadDeneb, VersionedHash, @@ -108,7 +108,7 @@ impl> From> payload: builder_bid.header, block_value: builder_bid.value, kzg_commitments: builder_bid.blinded_blobs_bundle.commitments, - blobs: SidecarLess::from_blob_roots(builder_bid.blinded_blobs_bundle.blob_roots), + blobs: RawBlobs::from_blob_roots(builder_bid.blinded_blobs_bundle.blob_roots), proofs: builder_bid.blinded_blobs_bundle.proofs, }, }; @@ -161,7 +161,7 @@ pub enum BlockProposalContents> { payload: Payload, block_value: Uint256, kzg_commitments: KzgCommitments, - blobs: Payload::Blob, + blobs: >::RawBlobs, proofs: KzgProofs, }, } @@ -176,7 +176,7 @@ impl> From> payload: execution_payload.into(), block_value, kzg_commitments: bundle.commitments, - blobs: SidecarLess::from_blobs(bundle.blobs), + blobs: RawBlobs::from_blobs(bundle.blobs), proofs: bundle.proofs, }, None => Self::Payload { @@ -194,7 +194,7 @@ impl> BlockProposalContents ( Payload, Option>, - Option, + Option<>::RawBlobs>, Option>, ) { match self { diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 71d33e8a938..66cbbcc203d 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -1363,7 +1363,7 @@ pub enum BlockContents> { pub type BlockContentsTuple = ( BeaconBlock, - Option>::BlobSidecar>>, + Option>::Sidecar>>, ); impl> BlockContents { @@ -1429,7 +1429,7 @@ impl> Into> pub type SignedBlockContentsTuple = ( SignedBeaconBlock, - Option>::BlobSidecar>>, + Option>::Sidecar>>, ); /// A wrapper over a [`SignedBeaconBlock`] or a [`SignedBeaconBlockAndBlobSidecars`]. @@ -1445,7 +1445,7 @@ pub enum SignedBlockContents = FullP impl> SignedBlockContents { pub fn new( block: SignedBeaconBlock, - blobs: Option>, + blobs: Option>, ) -> Self { match (Payload::block_type(), blobs) { (BlockType::Blinded, Some(blobs)) => { @@ -1476,7 +1476,7 @@ impl> SignedBlockContents Option> { + pub fn blobs_cloned(&self) -> Option> { match self { SignedBlockContents::BlockAndBlobSidecars(block_and_sidecars) => { Some(block_and_sidecars.signed_blob_sidecars.clone()) @@ -1596,14 +1596,14 @@ impl> From> { pub signed_block: SignedBeaconBlock, - pub signed_blob_sidecars: SignedSidecarList, + pub signed_blob_sidecars: SignedSidecarList, } #[derive(Debug, Clone, Serialize, Deserialize, Encode)] #[serde(bound = "T: EthSpec, Payload: AbstractExecPayload")] pub struct BeaconBlockAndBlobSidecars> { pub block: BeaconBlock, - pub blob_sidecars: SidecarList, + pub blob_sidecars: SidecarList, } impl> ForkVersionDeserialize @@ -1619,7 +1619,7 @@ impl> ForkVersionDeserialize block: serde_json::Value, blob_sidecars: SidecarList, } - let helper: Helper = + let helper: Helper = serde_json::from_value(value).map_err(serde::de::Error::custom)?; Ok(Self { @@ -1636,7 +1636,7 @@ pub struct SignedBlindedBeaconBlockAndBlobSidecars< Payload: AbstractExecPayload = BlindedPayload, > { pub signed_blinded_block: SignedBeaconBlock, - pub signed_blinded_blob_sidecars: SignedSidecarList, + pub signed_blinded_blob_sidecars: SignedSidecarList, } #[derive(Debug, Clone, Serialize, Deserialize, Encode)] @@ -1646,7 +1646,7 @@ pub struct BlindedBeaconBlockAndBlobSidecars< Payload: AbstractExecPayload = BlindedPayload, > { pub blinded_block: BeaconBlock, - pub blinded_blob_sidecars: SidecarList, + pub blinded_blob_sidecars: SidecarList, } impl> ForkVersionDeserialize @@ -1662,7 +1662,7 @@ impl> ForkVersionDeserialize blinded_block: serde_json::Value, blinded_blob_sidecars: SidecarList, } - let helper: Helper = + let helper: Helper = serde_json::from_value(value).map_err(serde::de::Error::custom)?; Ok(Self { diff --git a/consensus/types/src/blob_sidecar.rs b/consensus/types/src/blob_sidecar.rs index 62d1c90c946..6289b6f5864 100644 --- a/consensus/types/src/blob_sidecar.rs +++ b/consensus/types/src/blob_sidecar.rs @@ -39,64 +39,25 @@ pub trait Sidecar: + Send + for<'a> arbitrary::Arbitrary<'a> { + type RawBlobs: RawBlobs; fn slot(&self) -> Slot; + fn build_sidecar>( + a: Self::RawBlobs, + block: &BeaconBlock, + expected_kzg_commitments: &KzgCommitments, + kzg_proofs: Vec, + ) -> Result, String>; } -pub trait SidecarLess: Sync + Send { - fn to_sidecar,P: AbstractExecPayload>( - self, - block: &BeaconBlock, - expected_kzg_commitments: &KzgCommitments, - kzg_proofs: Vec, - ) -> Result, String> ; +pub trait RawBlobs: Sync + Send { fn from_blob_roots(roots: BlobRoots) -> Self; fn from_blobs(blobs: Blobs) -> Self; fn len(&self) -> usize; fn is_empty(&self) -> bool; } -impl SidecarLess for Blobs { - fn to_sidecar, P: AbstractExecPayload>( - self, - block: &BeaconBlock, - expected_kzg_commitments: &KzgCommitments, - kzg_proofs: Vec, - ) -> Result, String> { - let blobs = self; - let beacon_block_root = block.canonical_root(); - let slot = block.slot(); - - //TODO(sean) kzg verify - - let blob_sidecars = BlobSidecarList::from( - blobs - .into_iter() - .enumerate() - .map(|(blob_index, blob)| { - let kzg_commitment = expected_kzg_commitments - .get(blob_index) - .expect("KZG commitment should exist for blob"); - - let kzg_proof = kzg_proofs - .get(blob_index) - .expect("KZG proof should exist for blob"); - - Ok(Arc::new(BlobSidecar { - block_root: beacon_block_root, - index: blob_index as u64, - slot, - block_parent_root: block.parent_root(), - proposer_index: block.proposer_index(), - blob, - kzg_commitment: *kzg_commitment, - kzg_proof: *kzg_proof, - })) - }) - .collect::, String>>()?, - ); +impl RawBlobs for Blobs { - Ok(blob_sidecars) - } fn from_blob_roots(roots: BlobRoots) -> Self { todo!() } @@ -111,47 +72,8 @@ impl SidecarLess for Blobs { } } -impl SidecarLess for BlobRoots { - fn to_sidecar, P: AbstractExecPayload>( - self, - block: &BeaconBlock, - expected_kzg_commitments: &KzgCommitments, - kzg_proofs: Vec, - ) -> Result, String> { - let blob_roots = self; - let beacon_block_root = block.canonical_root(); - let slot = block.slot(); - - let blob_sidecars = BlindedBlobSidecarList::::from( - blob_roots - .into_iter() - .enumerate() - .map(|(blob_index, blob_root)| { - let kzg_commitment = expected_kzg_commitments - .get(blob_index) - .expect("KZG commitment should exist for blob"); - - let kzg_proof = kzg_proofs.get(blob_index).ok_or(format!( - "Missing KZG proof for slot {} blob index: {}", - slot, blob_index - ))?; +impl RawBlobs for BlobRoots { - Ok(Arc::new(BlindedBlobSidecar { - block_root: beacon_block_root, - index: blob_index as u64, - slot, - block_parent_root: block.parent_root(), - proposer_index: block.proposer_index(), - blob_root, - kzg_commitment: *kzg_commitment, - kzg_proof: *kzg_proof, - })) - }) - .collect::, String>>()?, - ); - - Ok(blob_sidecars) - } fn from_blob_roots(roots: BlobRoots) -> Self { todo!() } @@ -218,9 +140,50 @@ pub struct BlobSidecar { } impl Sidecar for BlobSidecar { + type RawBlobs = Blobs; fn slot(&self) -> Slot { self.slot } + fn build_sidecar>( + blobs: Blobs, + block: &BeaconBlock, + expected_kzg_commitments: &KzgCommitments, + kzg_proofs: Vec, + ) -> Result, String> { + let beacon_block_root = block.canonical_root(); + let slot = block.slot(); + + //TODO: kzg verify + + let blob_sidecars = BlobSidecarList::from( + blobs + .into_iter() + .enumerate() + .map(|(blob_index, blob)| { + let kzg_commitment = expected_kzg_commitments + .get(blob_index) + .expect("KZG commitment should exist for blob"); + + let kzg_proof = kzg_proofs + .get(blob_index) + .expect("KZG proof should exist for blob"); + + Ok(Arc::new(BlobSidecar { + block_root: beacon_block_root, + index: blob_index as u64, + slot, + block_parent_root: block.parent_root(), + proposer_index: block.proposer_index(), + blob, + kzg_commitment: *kzg_commitment, + kzg_proof: *kzg_proof, + })) + }) + .collect::, String>>()?, + ); + + Ok(blob_sidecars) + } } impl From>> for BlindedBlobSidecar { @@ -358,9 +321,49 @@ pub struct BlindedBlobSidecar { impl SignedRoot for BlindedBlobSidecar {} impl Sidecar for BlindedBlobSidecar { + type RawBlobs = BlobRoots; fn slot(&self) -> Slot { self.slot } + fn build_sidecar>( + blob_roots: BlobRoots, + block: &BeaconBlock, + expected_kzg_commitments: &KzgCommitments, + kzg_proofs: Vec, + ) -> Result, String> { + let beacon_block_root = block.canonical_root(); + let slot = block.slot(); + + let blob_sidecars = BlindedBlobSidecarList::::from( + blob_roots + .into_iter() + .enumerate() + .map(|(blob_index, blob_root)| { + let kzg_commitment = expected_kzg_commitments + .get(blob_index) + .expect("KZG commitment should exist for blob"); + + let kzg_proof = kzg_proofs.get(blob_index).ok_or(format!( + "Missing KZG proof for slot {} blob index: {}", + slot, blob_index + ))?; + + Ok(Arc::new(BlindedBlobSidecar { + block_root: beacon_block_root, + index: blob_index as u64, + slot, + block_parent_root: block.parent_root(), + proposer_index: block.proposer_index(), + blob_root, + kzg_commitment: *kzg_commitment, + kzg_proof: *kzg_proof, + })) + }) + .collect::, String>>()?, + ); + + Ok(blob_sidecars) + } } pub type SidecarList = VariableList, ::MaxBlobsPerBlock>; diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index 7b30dac8737..b7f4552b23e 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -13,7 +13,7 @@ use std::hash::Hash; use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; -use crate::blob_sidecar::SidecarLess; +use crate::blob_sidecar::RawBlobs; #[derive(Debug, PartialEq)] pub enum BlockType { @@ -86,8 +86,7 @@ pub trait AbstractExecPayload: + TryInto + TryInto { - type BlobSidecar: Sidecar; - type Blob: SidecarLess; + type Sidecar: Sidecar; type Ref<'a>: ExecPayload + Copy @@ -111,7 +110,7 @@ pub trait AbstractExecPayload: fn default_at_fork(fork_name: ForkName) -> Result; fn default_blobs_at_fork( fork_name: ForkName, - ) -> Result; + ) -> Result<>::RawBlobs, Error>; } #[superstruct( @@ -388,8 +387,7 @@ impl<'b, T: EthSpec> ExecPayload for FullPayloadRef<'b, T> { } impl AbstractExecPayload for FullPayload { - type BlobSidecar = BlobSidecar; - type Blob = Blobs; + type Sidecar = BlobSidecar; type Ref<'a> = FullPayloadRef<'a, T>; type Merge = FullPayloadMerge; type Capella = FullPayloadCapella; @@ -911,8 +909,7 @@ impl AbstractExecPayload for BlindedPayload { type Capella = BlindedPayloadCapella; type Deneb = BlindedPayloadDeneb; - type BlobSidecar = BlindedBlobSidecar; - type Blob = BlobRoots; + type Sidecar = BlindedBlobSidecar; fn default_at_fork(fork_name: ForkName) -> Result { match fork_name { diff --git a/validator_client/src/signing_method.rs b/validator_client/src/signing_method.rs index 591d533fbef..96bfd2511f1 100644 --- a/validator_client/src/signing_method.rs +++ b/validator_client/src/signing_method.rs @@ -37,7 +37,7 @@ pub enum Error { pub enum SignableMessage<'a, T: EthSpec, Payload: AbstractExecPayload = FullPayload> { RandaoReveal(Epoch), BeaconBlock(&'a BeaconBlock), - BlobSidecar(&'a Payload::BlobSidecar), + BlobSidecar(&'a Payload::Sidecar), AttestationData(&'a AttestationData), SignedAggregateAndProof(&'a AggregateAndProof), SelectionProof(Slot), diff --git a/validator_client/src/validator_store.rs b/validator_client/src/validator_store.rs index e8595d96639..939bc737cdd 100644 --- a/validator_client/src/validator_store.rs +++ b/validator_client/src/validator_store.rs @@ -545,8 +545,8 @@ impl ValidatorStore { pub async fn sign_blobs>( &self, validator_pubkey: PublicKeyBytes, - blob_sidecars: SidecarList, - ) -> Result, Error> { + blob_sidecars: SidecarList, + ) -> Result, Error> { let mut signed_blob_sidecars = Vec::new(); for blob_sidecar in blob_sidecars.into_iter() { let slot = blob_sidecar.slot(); From 97ee1f49d97c990dc21c06b742198afd2362cadc Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Tue, 8 Aug 2023 15:41:14 +1000 Subject: [PATCH 17/24] Fix todos and improve error handling --- beacon_node/beacon_chain/src/beacon_chain.rs | 36 +++++++++--- beacon_node/beacon_chain/src/errors.rs | 1 + beacon_node/execution_layer/src/engine_api.rs | 1 - beacon_node/execution_layer/src/lib.rs | 40 +++++++------ consensus/types/src/blob_sidecar.rs | 56 ++++++++++++------- consensus/types/src/payload.rs | 5 +- 6 files changed, 91 insertions(+), 48 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 5e3bebac3ca..505e9488888 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -9,7 +9,6 @@ use crate::beacon_proposer_cache::compute_proposer_duties_from_head; use crate::beacon_proposer_cache::BeaconProposerCache; use crate::blob_verification::{self, GossipBlobError, GossipVerifiedBlob}; use crate::block_times_cache::BlockTimesCache; -use tree_hash::TreeHash; use crate::block_verification::POS_PANDA_BANNER; use crate::block_verification::{ check_block_is_finalized_checkpoint_or_descendant, check_block_relevancy, get_block_root, @@ -33,7 +32,6 @@ use crate::execution_payload::{get_execution_payload, NotifyExecutionLayer, Prep use crate::fork_choice_signal::{ForkChoiceSignalRx, ForkChoiceSignalTx, ForkChoiceWaitResult}; use crate::head_tracker::HeadTracker; use crate::historical_blocks::HistoricalBlockError; -use crate::kzg_utils; use crate::light_client_finality_update_verification::{ Error as LightClientFinalityUpdateError, VerifiedLightClientFinalityUpdate, }; @@ -68,7 +66,9 @@ use crate::validator_monitor::{ HISTORIC_EPOCHS as VALIDATOR_MONITOR_HISTORIC_EPOCHS, }; use crate::validator_pubkey_cache::ValidatorPubkeyCache; -use crate::{metrics, BeaconChainError, BeaconForkChoiceStore, BeaconSnapshot, CachedHead}; +use crate::{ + kzg_utils, metrics, BeaconChainError, BeaconForkChoiceStore, BeaconSnapshot, CachedHead, +}; use eth2::types::{EventKind, SseBlock, SseExtendedPayloadAttributes, SyncDuty}; use execution_layer::{ BlockProposalContents, BuilderParams, ChainHealth, ExecutionLayer, FailedCondition, @@ -116,6 +116,7 @@ use store::{ }; use task_executor::{ShutdownReason, TaskExecutor}; use tokio_stream::Stream; +use tree_hash::TreeHash; use types::beacon_state::CloneConfig; use types::blob_sidecar::RawBlobs; use types::consts::deneb::MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS; @@ -4970,11 +4971,30 @@ impl BeaconChain { } let kzg_proofs = Vec::from(proofs); - Some(Sidecar::build_sidecar(blobs_or_blobs_roots, - &block, - expected_kzg_commitments, - kzg_proofs, - ).unwrap()) //TODO: remove unwrap + + if let Some(blobs) = blobs_or_blobs_roots.blobs() { + let kzg = self + .kzg + .as_ref() + .ok_or(BlockProductionError::TrustedSetupNotInitialized)?; + kzg_utils::validate_blobs::( + kzg, + expected_kzg_commitments, + blobs, + &kzg_proofs, + ) + .map_err(BlockProductionError::KzgError)?; + } + + Some( + Sidecar::build_sidecar( + blobs_or_blobs_roots, + &block, + expected_kzg_commitments, + kzg_proofs, + ) + .map_err(BlockProductionError::FailedToBuildBlobSidecars)?, + ) } _ => None, }; diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs index d0da91406b5..8b4493d49d4 100644 --- a/beacon_node/beacon_chain/src/errors.rs +++ b/beacon_node/beacon_chain/src/errors.rs @@ -290,6 +290,7 @@ pub enum BlockProductionError { TrustedSetupNotInitialized, InvalidBlockVariant(String), KzgError(kzg::Error), + FailedToBuildBlobSidecars(String), } easy_from_to!(BlockProcessingError, BlockProductionError); diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index 382364e010b..8f9d23906aa 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -58,7 +58,6 @@ pub enum Error { IncorrectStateVariant, RequiredMethodUnsupported(&'static str), UnsupportedForkVariant(String), - BadConversion(String), RlpDecoderError(rlp::DecoderError), BlobTxConversionError(BlobTxConversionError), } diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 24154ed5e95..057666b6122 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -49,7 +49,7 @@ use types::{ use types::{ BlindedPayload, BlockType, ChainSpec, Epoch, ExecutionPayloadCapella, ExecutionPayloadMerge, }; -use types::{KzgProofs, Sidecar, }; +use types::{KzgProofs, Sidecar}; use types::{ProposerPreparationData, PublicKeyBytes, Signature, Slot, Transaction}; mod block_hash; @@ -91,10 +91,12 @@ pub enum ProvenancedPayload

{ Builder(P), } -impl> From> +impl> TryFrom> for ProvenancedPayload> { - fn from(value: BuilderBid) -> Self { + type Error = Error; + + fn try_from(value: BuilderBid) -> Result { let block_proposal_contents = match value { BuilderBid::Merge(builder_bid) => BlockProposalContents::Payload { payload: builder_bid.header, @@ -108,11 +110,12 @@ impl> From> payload: builder_bid.header, block_value: builder_bid.value, kzg_commitments: builder_bid.blinded_blobs_bundle.commitments, - blobs: RawBlobs::from_blob_roots(builder_bid.blinded_blobs_bundle.blob_roots), + blobs: RawBlobs::try_from_blob_roots(builder_bid.blinded_blobs_bundle.blob_roots) + .map_err(Error::InvalidBlobConversion)?, proofs: builder_bid.blinded_blobs_bundle.proofs, }, }; - ProvenancedPayload::Builder(block_proposal_contents) + Ok(ProvenancedPayload::Builder(block_proposal_contents)) } } @@ -137,6 +140,7 @@ pub enum Error { InvalidJWTSecret(String), InvalidForkForPayload, InvalidPayloadBody(String), + InvalidBlobConversion(String), BeaconStateError(BeaconStateError), } @@ -166,23 +170,26 @@ pub enum BlockProposalContents> { }, } -impl> From> +impl> TryFrom> for BlockProposalContents { - fn from(response: GetPayloadResponse) -> Self { + type Error = Error; + + fn try_from(response: GetPayloadResponse) -> Result { let (execution_payload, block_value, maybe_bundle) = response.into(); match maybe_bundle { - Some(bundle) => Self::PayloadAndBlobs { + Some(bundle) => Ok(Self::PayloadAndBlobs { payload: execution_payload.into(), block_value, kzg_commitments: bundle.commitments, - blobs: RawBlobs::from_blobs(bundle.blobs), + blobs: RawBlobs::try_from_blobs(bundle.blobs) + .map_err(Error::InvalidBlobConversion)?, proofs: bundle.proofs, - }, - None => Self::Payload { + }), + None => Ok(Self::Payload { payload: execution_payload.into(), block_value, - }, + }), } } } @@ -951,7 +958,7 @@ impl ExecutionLayer { current_fork, spec, ) { - Ok(()) => Ok(ProvenancedPayload::from(relay.data.message)), + Ok(()) => Ok(ProvenancedPayload::try_from(relay.data.message)?), Err(reason) if !reason.payload_invalid() => { info!( self.log(), @@ -1000,11 +1007,11 @@ impl ExecutionLayer { current_fork, spec, ) { - Ok(()) => Ok(ProvenancedPayload::from(relay.data.message)), + Ok(()) => Ok(ProvenancedPayload::try_from(relay.data.message)?), // If the payload is valid then use it. The local EE failed // to produce a payload so we have no alternative. Err(e) if !e.payload_invalid() => { - Ok(ProvenancedPayload::from(relay.data.message)) + Ok(ProvenancedPayload::try_from(relay.data.message)?) } Err(reason) => { metrics::inc_counter_vec( @@ -1208,11 +1215,12 @@ impl ExecutionLayer { ); } - Ok(payload_response.into()) + Ok(payload_response) }) .await .map_err(Box::new) .map_err(Error::EngineError) + .and_then(|payload| payload.try_into()) } /// Maps to the `engine_newPayload` JSON-RPC call. diff --git a/consensus/types/src/blob_sidecar.rs b/consensus/types/src/blob_sidecar.rs index 6289b6f5864..a1237598e0e 100644 --- a/consensus/types/src/blob_sidecar.rs +++ b/consensus/types/src/blob_sidecar.rs @@ -49,43 +49,59 @@ pub trait Sidecar: ) -> Result, String>; } -pub trait RawBlobs: Sync + Send { - fn from_blob_roots(roots: BlobRoots) -> Self; - fn from_blobs(blobs: Blobs) -> Self; +pub trait RawBlobs: Sync + Send + Sized { + fn try_from_blob_roots(roots: BlobRoots) -> Result; + fn try_from_blobs(blobs: Blobs) -> Result; fn len(&self) -> usize; fn is_empty(&self) -> bool; + fn blobs(&self) -> Option<&Blobs>; } impl RawBlobs for Blobs { - - fn from_blob_roots(roots: BlobRoots) -> Self { - todo!() + fn try_from_blob_roots(_roots: BlobRoots) -> Result { + Err("Unexpected conversion from blob roots to blobs".to_string()) } - fn from_blobs(blobs: Blobs) -> Self { - todo!() + + fn try_from_blobs(blobs: Blobs) -> Result { + Ok(blobs) } + fn len(&self) -> usize { VariableList::len(self) } + fn is_empty(&self) -> bool { VariableList::is_empty(self) } + + fn blobs(&self) -> Option<&Blobs> { + Some(self) + } } impl RawBlobs for BlobRoots { - - fn from_blob_roots(roots: BlobRoots) -> Self { - todo!() + fn try_from_blob_roots(roots: BlobRoots) -> Result { + Ok(roots) } - fn from_blobs(blobs: Blobs) -> Self { - todo!() + + fn try_from_blobs(_blobs: Blobs) -> Result { + // It is possible to convert from blobs to blob roots, however this should be done using + // `From` or `Into` instead of this generic implementation; this function implementation + // should be unreachable, and attempt to use this indicates a bug somewhere. + Err("Unexpected conversion from blob to blob roots".to_string()) } + fn len(&self) -> usize { VariableList::len(self) } + fn is_empty(&self) -> bool { VariableList::is_empty(self) } + + fn blobs(&self) -> Option<&Blobs> { + None + } } /// Container of the data that identifies an individual blob. @@ -141,9 +157,11 @@ pub struct BlobSidecar { impl Sidecar for BlobSidecar { type RawBlobs = Blobs; + fn slot(&self) -> Slot { self.slot } + fn build_sidecar>( blobs: Blobs, block: &BeaconBlock, @@ -152,9 +170,6 @@ impl Sidecar for BlobSidecar { ) -> Result, String> { let beacon_block_root = block.canonical_root(); let slot = block.slot(); - - //TODO: kzg verify - let blob_sidecars = BlobSidecarList::from( blobs .into_iter() @@ -162,11 +177,11 @@ impl Sidecar for BlobSidecar { .map(|(blob_index, blob)| { let kzg_commitment = expected_kzg_commitments .get(blob_index) - .expect("KZG commitment should exist for blob"); + .ok_or("KZG commitment should exist for blob")?; let kzg_proof = kzg_proofs .get(blob_index) - .expect("KZG proof should exist for blob"); + .ok_or("KZG proof should exist for blob")?; Ok(Arc::new(BlobSidecar { block_root: beacon_block_root, @@ -322,9 +337,11 @@ impl SignedRoot for BlindedBlobSidecar {} impl Sidecar for BlindedBlobSidecar { type RawBlobs = BlobRoots; + fn slot(&self) -> Slot { self.slot } + fn build_sidecar>( blob_roots: BlobRoots, block: &BeaconBlock, @@ -341,7 +358,7 @@ impl Sidecar for BlindedBlobSidecar { .map(|(blob_index, blob_root)| { let kzg_commitment = expected_kzg_commitments .get(blob_index) - .expect("KZG commitment should exist for blob"); + .ok_or("KZG commitment should exist for blob")?; let kzg_proof = kzg_proofs.get(blob_index).ok_or(format!( "Missing KZG proof for slot {} blob index: {}", @@ -375,4 +392,3 @@ pub type FixedBlobSidecarList = pub type Blobs = VariableList, ::MaxBlobsPerBlock>; pub type BlobRoots = VariableList::MaxBlobsPerBlock>; - diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index b7f4552b23e..50cf49e2caa 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -13,7 +13,6 @@ use std::hash::Hash; use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; -use crate::blob_sidecar::RawBlobs; #[derive(Debug, PartialEq)] pub enum BlockType { @@ -401,7 +400,7 @@ impl AbstractExecPayload for FullPayload { ForkName::Deneb => Ok(FullPayloadDeneb::default().into()), } } - fn default_blobs_at_fork(fork_name: ForkName) -> Result, Error> { + fn default_blobs_at_fork(_fork_name: ForkName) -> Result, Error> { Ok(VariableList::default()) } } @@ -919,7 +918,7 @@ impl AbstractExecPayload for BlindedPayload { ForkName::Deneb => Ok(BlindedPayloadDeneb::default().into()), } } - fn default_blobs_at_fork(fork_name: ForkName) -> Result, Error> { + fn default_blobs_at_fork(_fork_name: ForkName) -> Result, Error> { Ok(VariableList::default()) } } From 803045be8563f7551399912ed519485ba71f5116 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Wed, 9 Aug 2023 16:16:23 +1000 Subject: [PATCH 18/24] Consolidate BlobsBundle from `execution_layer` into `consensus/types` --- beacon_node/execution_layer/src/engine_api.rs | 17 +++++------------ .../src/engine_api/json_structures.rs | 11 ++++++----- beacon_node/execution_layer/src/lib.rs | 2 +- .../src/test_utils/execution_block_generator.rs | 16 ++++++++-------- consensus/types/src/payload.rs | 2 +- 5 files changed, 21 insertions(+), 27 deletions(-) diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index 8b1354846a0..3b5dac80c17 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -17,14 +17,14 @@ use serde::{Deserialize, Serialize}; use std::convert::TryFrom; use strum::IntoStaticStr; use superstruct::superstruct; -use types::beacon_block_body::KzgCommitments; -use types::blob_sidecar::Blobs; pub use types::{ Address, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadHeader, ExecutionPayloadRef, FixedVector, ForkName, Hash256, Transactions, Uint256, VariableList, Withdrawal, Withdrawals, }; -use types::{ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadMerge, KzgProofs}; +use types::{ + BlobsBundle, ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadMerge, KzgProofs, +}; pub mod auth; pub mod http; @@ -384,7 +384,7 @@ pub struct GetPayloadResponse { pub execution_payload: ExecutionPayloadDeneb, pub block_value: Uint256, #[superstruct(only(Deneb))] - pub blobs_bundle: BlobsBundleV1, + pub blobs_bundle: BlobsBundle, } impl<'a, T: EthSpec> From> for ExecutionPayloadRef<'a, T> { @@ -404,7 +404,7 @@ impl From> for ExecutionPayload { } impl From> - for (ExecutionPayload, Uint256, Option>) + for (ExecutionPayload, Uint256, Option>) { fn from(response: GetPayloadResponse) -> Self { match response { @@ -527,13 +527,6 @@ impl ExecutionPayloadBodyV1 { } } -#[derive(Clone, Default, Debug, PartialEq)] -pub struct BlobsBundleV1 { - pub commitments: KzgCommitments, - pub proofs: KzgProofs, - pub blobs: Blobs, -} - #[derive(Clone, Copy, Debug)] pub struct EngineCapabilities { pub new_payload_v1: bool, diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index d94e7078764..f6e1decad77 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -5,8 +5,9 @@ use superstruct::superstruct; use types::beacon_block_body::KzgCommitments; use types::blob_sidecar::Blobs; use types::{ - EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadDeneb, - ExecutionPayloadMerge, FixedVector, Transactions, Unsigned, VariableList, Withdrawal, + BlobsBundle, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, + ExecutionPayloadDeneb, ExecutionPayloadMerge, FixedVector, Transactions, Unsigned, + VariableList, Withdrawal, }; #[derive(Debug, PartialEq, Serialize, Deserialize)] @@ -425,8 +426,8 @@ pub struct JsonBlobsBundleV1 { pub blobs: Blobs, } -impl From> for JsonBlobsBundleV1 { - fn from(blobs_bundle: BlobsBundleV1) -> Self { +impl From> for JsonBlobsBundleV1 { + fn from(blobs_bundle: BlobsBundle) -> Self { Self { commitments: blobs_bundle.commitments, proofs: blobs_bundle.proofs, @@ -434,7 +435,7 @@ impl From> for JsonBlobsBundleV1 { } } } -impl From> for BlobsBundleV1 { +impl From> for BlobsBundle { fn from(json_blobs_bundle: JsonBlobsBundleV1) -> Self { Self { commitments: json_blobs_bundle.commitments, diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 6525f68b0d6..5234978fa35 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -293,7 +293,7 @@ pub enum FailedCondition { EpochsSinceFinalization, } -type PayloadContentsRefTuple<'a, T> = (ExecutionPayloadRef<'a, T>, Option<&'a BlobsBundleV1>); +type PayloadContentsRefTuple<'a, T> = (ExecutionPayloadRef<'a, T>, Option<&'a BlobsBundle>); struct Inner { engine: Arc, diff --git a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs index 937c6d9da33..0cdf02f8d48 100644 --- a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs +++ b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs @@ -6,7 +6,7 @@ use crate::{ }, ExecutionBlock, PayloadAttributes, PayloadId, PayloadStatusV1, PayloadStatusV1Status, }, - random_valid_tx, BlobsBundleV1, ExecutionBlockWithTransactions, + random_valid_tx, ExecutionBlockWithTransactions, }; use kzg::Kzg; use rand::thread_rng; @@ -16,9 +16,9 @@ use std::sync::Arc; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; use types::{ - BlobSidecar, ChainSpec, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, - ExecutionPayloadDeneb, ExecutionPayloadHeader, ExecutionPayloadMerge, ForkName, Hash256, - Transactions, Uint256, + BlobSidecar, BlobsBundle, ChainSpec, EthSpec, ExecutionBlockHash, ExecutionPayload, + ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadHeader, ExecutionPayloadMerge, + ForkName, Hash256, Transactions, Uint256, }; use super::DEFAULT_TERMINAL_BLOCK; @@ -128,7 +128,7 @@ pub struct ExecutionBlockGenerator { /* * deneb stuff */ - pub blobs_bundles: HashMap>, + pub blobs_bundles: HashMap>, pub kzg: Option>>, } @@ -406,7 +406,7 @@ impl ExecutionBlockGenerator { self.payload_ids.get(id).cloned() } - pub fn get_blobs_bundle(&mut self, id: &PayloadId) -> Option> { + pub fn get_blobs_bundle(&mut self, id: &PayloadId) -> Option> { self.blobs_bundles.get(id).cloned() } @@ -630,8 +630,8 @@ impl ExecutionBlockGenerator { pub fn generate_random_blobs( n_blobs: usize, kzg: &Kzg, -) -> Result<(BlobsBundleV1, Transactions), String> { - let mut bundle = BlobsBundleV1::::default(); +) -> Result<(BlobsBundle, Transactions), String> { + let mut bundle = BlobsBundle::::default(); let mut transactions = vec![]; for blob_index in 0..n_blobs { let random_valid_sidecar = BlobSidecar::::random_valid(&mut thread_rng(), kzg)?; diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index 50cf49e2caa..6d1a1a4e2c9 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -1044,7 +1044,7 @@ pub struct ExecutionPayloadAndBlobs { pub blobs_bundle: BlobsBundle, } -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] #[serde(bound = "E: EthSpec")] pub struct BlobsBundle { pub commitments: KzgCommitments, From d3befb05fa1133fe92d7723f735f8bdb19c35f69 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Wed, 9 Aug 2023 17:22:52 +1000 Subject: [PATCH 19/24] Rename RawBlobs, Blobs, and BlobRoots --- beacon_node/beacon_chain/src/beacon_chain.rs | 2 +- .../src/engine_api/json_structures.rs | 4 +- beacon_node/execution_layer/src/lib.rs | 10 ++--- beacon_node/http_api/tests/tests.rs | 4 +- consensus/types/src/blob_sidecar.rs | 40 +++++++++---------- consensus/types/src/lib.rs | 4 +- consensus/types/src/payload.rs | 8 ++-- 7 files changed, 37 insertions(+), 35 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index c2f5ea2dc11..53792f85604 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -119,7 +119,7 @@ use task_executor::{ShutdownReason, TaskExecutor}; use tokio_stream::Stream; use tree_hash::TreeHash; use types::beacon_state::CloneConfig; -use types::blob_sidecar::{BlobSidecarList, FixedBlobSidecarList, RawBlobs}; +use types::blob_sidecar::{BlobItems, BlobSidecarList, FixedBlobSidecarList}; use types::consts::deneb::MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS; use types::*; diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index f6e1decad77..cb9ce2080f8 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize}; use strum::EnumString; use superstruct::superstruct; use types::beacon_block_body::KzgCommitments; -use types::blob_sidecar::Blobs; +use types::blob_sidecar::BlobsList; use types::{ BlobsBundle, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadMerge, FixedVector, Transactions, Unsigned, @@ -423,7 +423,7 @@ pub struct JsonBlobsBundleV1 { pub commitments: KzgCommitments, pub proofs: KzgProofs, #[serde(with = "ssz_types::serde_utils::list_of_hex_fixed_vec")] - pub blobs: Blobs, + pub blobs: BlobsList, } impl From> for JsonBlobsBundleV1 { diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 5234978fa35..f6456988ce0 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -41,7 +41,7 @@ use tokio::{ use tokio_stream::wrappers::WatchStream; use tree_hash::TreeHash; use types::beacon_block_body::KzgCommitments; -use types::blob_sidecar::RawBlobs; +use types::blob_sidecar::BlobItems; use types::builder_bid::BuilderBid; use types::{ AbstractExecPayload, BeaconStateError, ExecPayload, ExecutionPayloadDeneb, VersionedHash, @@ -108,7 +108,7 @@ impl> TryFrom payload: builder_bid.header, block_value: builder_bid.value, kzg_commitments: builder_bid.blinded_blobs_bundle.commitments, - blobs: RawBlobs::try_from_blob_roots(builder_bid.blinded_blobs_bundle.blob_roots) + blobs: BlobItems::try_from_blob_roots(builder_bid.blinded_blobs_bundle.blob_roots) .map_err(Error::InvalidBlobConversion)?, proofs: builder_bid.blinded_blobs_bundle.proofs, }, @@ -163,7 +163,7 @@ pub enum BlockProposalContents> { payload: Payload, block_value: Uint256, kzg_commitments: KzgCommitments, - blobs: >::RawBlobs, + blobs: >::BlobItems, proofs: KzgProofs, }, } @@ -180,7 +180,7 @@ impl> TryFrom> payload: execution_payload.into(), block_value, kzg_commitments: bundle.commitments, - blobs: RawBlobs::try_from_blobs(bundle.blobs) + blobs: BlobItems::try_from_blobs(bundle.blobs) .map_err(Error::InvalidBlobConversion)?, proofs: bundle.proofs, }), @@ -199,7 +199,7 @@ impl> BlockProposalContents ( Payload, Option>, - Option<>::RawBlobs>, + Option<>::BlobItems>, Option>, ) { match self { diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index a95e82d2316..9f95159abf9 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -2646,7 +2646,9 @@ impl ApiTester { .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() - .data; + .data + .deconstruct() + .0; let signed_block = block.sign(&sk, &fork, genesis_validators_root, &self.chain.spec); diff --git a/consensus/types/src/blob_sidecar.rs b/consensus/types/src/blob_sidecar.rs index a1237598e0e..b51cc0f323f 100644 --- a/consensus/types/src/blob_sidecar.rs +++ b/consensus/types/src/blob_sidecar.rs @@ -39,30 +39,30 @@ pub trait Sidecar: + Send + for<'a> arbitrary::Arbitrary<'a> { - type RawBlobs: RawBlobs; + type BlobItems: BlobItems; fn slot(&self) -> Slot; fn build_sidecar>( - a: Self::RawBlobs, + blob_items: Self::BlobItems, block: &BeaconBlock, expected_kzg_commitments: &KzgCommitments, kzg_proofs: Vec, ) -> Result, String>; } -pub trait RawBlobs: Sync + Send + Sized { - fn try_from_blob_roots(roots: BlobRoots) -> Result; - fn try_from_blobs(blobs: Blobs) -> Result; +pub trait BlobItems: Sync + Send + Sized { + fn try_from_blob_roots(roots: BlobRootsList) -> Result; + fn try_from_blobs(blobs: BlobsList) -> Result; fn len(&self) -> usize; fn is_empty(&self) -> bool; - fn blobs(&self) -> Option<&Blobs>; + fn blobs(&self) -> Option<&BlobsList>; } -impl RawBlobs for Blobs { - fn try_from_blob_roots(_roots: BlobRoots) -> Result { +impl BlobItems for BlobsList { + fn try_from_blob_roots(_roots: BlobRootsList) -> Result { Err("Unexpected conversion from blob roots to blobs".to_string()) } - fn try_from_blobs(blobs: Blobs) -> Result { + fn try_from_blobs(blobs: BlobsList) -> Result { Ok(blobs) } @@ -74,17 +74,17 @@ impl RawBlobs for Blobs { VariableList::is_empty(self) } - fn blobs(&self) -> Option<&Blobs> { + fn blobs(&self) -> Option<&BlobsList> { Some(self) } } -impl RawBlobs for BlobRoots { - fn try_from_blob_roots(roots: BlobRoots) -> Result { +impl BlobItems for BlobRootsList { + fn try_from_blob_roots(roots: BlobRootsList) -> Result { Ok(roots) } - fn try_from_blobs(_blobs: Blobs) -> Result { + fn try_from_blobs(_blobs: BlobsList) -> Result { // It is possible to convert from blobs to blob roots, however this should be done using // `From` or `Into` instead of this generic implementation; this function implementation // should be unreachable, and attempt to use this indicates a bug somewhere. @@ -99,7 +99,7 @@ impl RawBlobs for BlobRoots { VariableList::is_empty(self) } - fn blobs(&self) -> Option<&Blobs> { + fn blobs(&self) -> Option<&BlobsList> { None } } @@ -156,14 +156,14 @@ pub struct BlobSidecar { } impl Sidecar for BlobSidecar { - type RawBlobs = Blobs; + type BlobItems = BlobsList; fn slot(&self) -> Slot { self.slot } fn build_sidecar>( - blobs: Blobs, + blobs: BlobsList, block: &BeaconBlock, expected_kzg_commitments: &KzgCommitments, kzg_proofs: Vec, @@ -336,14 +336,14 @@ pub struct BlindedBlobSidecar { impl SignedRoot for BlindedBlobSidecar {} impl Sidecar for BlindedBlobSidecar { - type RawBlobs = BlobRoots; + type BlobItems = BlobRootsList; fn slot(&self) -> Slot { self.slot } fn build_sidecar>( - blob_roots: BlobRoots, + blob_roots: BlobRootsList, block: &BeaconBlock, expected_kzg_commitments: &KzgCommitments, kzg_proofs: Vec, @@ -390,5 +390,5 @@ pub type BlindedBlobSidecarList = SidecarList; pub type FixedBlobSidecarList = FixedVector>>, ::MaxBlobsPerBlock>; -pub type Blobs = VariableList, ::MaxBlobsPerBlock>; -pub type BlobRoots = VariableList::MaxBlobsPerBlock>; +pub type BlobsList = VariableList, ::MaxBlobsPerBlock>; +pub type BlobRootsList = VariableList::MaxBlobsPerBlock>; diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 039bfe3dfba..5008bdf6320 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -120,8 +120,8 @@ pub use crate::beacon_block_header::BeaconBlockHeader; pub use crate::beacon_committee::{BeaconCommittee, OwnedBeaconCommittee}; pub use crate::beacon_state::{BeaconTreeHashCache, Error as BeaconStateError, *}; pub use crate::blob_sidecar::{ - BlindedBlobSidecar, BlindedBlobSidecarList, BlobRoots, BlobSidecar, BlobSidecarList, Blobs, - Sidecar, SidecarList, + BlindedBlobSidecar, BlindedBlobSidecarList, BlobRootsList, BlobSidecar, BlobSidecarList, + BlobsList, Sidecar, SidecarList, }; pub use crate::bls_to_execution_change::BlsToExecutionChange; pub use crate::chain_spec::{ChainSpec, Config, Domain}; diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index 6d1a1a4e2c9..8016aaad774 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -109,7 +109,7 @@ pub trait AbstractExecPayload: fn default_at_fork(fork_name: ForkName) -> Result; fn default_blobs_at_fork( fork_name: ForkName, - ) -> Result<>::RawBlobs, Error>; + ) -> Result<>::BlobItems, Error>; } #[superstruct( @@ -400,7 +400,7 @@ impl AbstractExecPayload for FullPayload { ForkName::Deneb => Ok(FullPayloadDeneb::default().into()), } } - fn default_blobs_at_fork(_fork_name: ForkName) -> Result, Error> { + fn default_blobs_at_fork(_fork_name: ForkName) -> Result, Error> { Ok(VariableList::default()) } } @@ -918,7 +918,7 @@ impl AbstractExecPayload for BlindedPayload { ForkName::Deneb => Ok(BlindedPayloadDeneb::default().into()), } } - fn default_blobs_at_fork(_fork_name: ForkName) -> Result, Error> { + fn default_blobs_at_fork(_fork_name: ForkName) -> Result, Error> { Ok(VariableList::default()) } } @@ -1050,5 +1050,5 @@ pub struct BlobsBundle { pub commitments: KzgCommitments, pub proofs: KzgProofs, #[serde(with = "ssz_types::serde_utils::list_of_hex_fixed_vec")] - pub blobs: Blobs, + pub blobs: BlobsList, } From e0aba921a9e9c670ab099d4ac1376098c72b0ff4 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Wed, 9 Aug 2023 17:24:48 +1000 Subject: [PATCH 20/24] Use `BlobRoots` type alias --- consensus/types/src/builder_bid.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/consensus/types/src/builder_bid.rs b/consensus/types/src/builder_bid.rs index 652d275151a..2c586cd14c8 100644 --- a/consensus/types/src/builder_bid.rs +++ b/consensus/types/src/builder_bid.rs @@ -3,7 +3,6 @@ use std::marker::PhantomData; use serde::{Deserialize as De, Deserializer, Serialize as Ser, Serializer}; use serde_derive::{Deserialize, Serialize}; use serde_with::{As, DeserializeAs, SerializeAs}; -use ssz_types::VariableList; use superstruct::superstruct; use tree_hash_derive::TreeHash; @@ -12,8 +11,8 @@ use bls::Signature; use crate::beacon_block_body::KzgCommitments; use crate::{ - AbstractExecPayload, ChainSpec, EthSpec, ExecPayload, ExecutionPayloadHeader, ForkName, - ForkVersionDeserialize, Hash256, KzgProofs, SignedRoot, Uint256, + AbstractExecPayload, BlobRootsList, ChainSpec, EthSpec, ExecPayload, ExecutionPayloadHeader, + ForkName, ForkVersionDeserialize, KzgProofs, SignedRoot, Uint256, }; #[derive(PartialEq, Debug, Serialize, Deserialize, TreeHash, Clone)] @@ -21,7 +20,7 @@ use crate::{ pub struct BlindedBlobsBundle { pub commitments: KzgCommitments, pub proofs: KzgProofs, - pub blob_roots: VariableList, + pub blob_roots: BlobRootsList, } #[superstruct( From f2ed354a8d0234e75651a20b7125f4685f2e47c2 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Wed, 9 Aug 2023 17:25:40 +1000 Subject: [PATCH 21/24] Update error message. Co-authored-by: realbigsean --- beacon_node/beacon_chain/src/beacon_chain.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 53792f85604..43ed13e17d7 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -4985,7 +4985,7 @@ impl BeaconChain { let expected_kzg_commitments = block.body().blob_kzg_commitments().map_err(|_| { BlockProductionError::InvalidBlockVariant( - "DENEB block does not contain kzg commitments".to_string(), + "deneb block does not contain kzg commitments".to_string(), ) })?; From 3b9de468107f47b23d0bc8bee4defb1dbcdfdae4 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Tue, 8 Aug 2023 16:34:31 -0400 Subject: [PATCH 22/24] update builder bid type # Conflicts: # consensus/types/src/builder_bid.rs --- Cargo.lock | 1 - beacon_node/execution_layer/src/lib.rs | 6 +- consensus/types/Cargo.toml | 1 - consensus/types/src/builder_bid.rs | 79 ++++++++++++-------------- 4 files changed, 38 insertions(+), 49 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 57589abc123..c992870fbee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8497,7 +8497,6 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "serde_with", "serde_yaml", "slog", "smallvec 1.11.0", diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index f6456988ce0..ff49583b9da 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -97,15 +97,15 @@ impl> TryFrom fn try_from(value: BuilderBid) -> Result { let block_proposal_contents = match value { BuilderBid::Merge(builder_bid) => BlockProposalContents::Payload { - payload: builder_bid.header, + payload: builder_bid.header.into(), block_value: builder_bid.value, }, BuilderBid::Capella(builder_bid) => BlockProposalContents::Payload { - payload: builder_bid.header, + payload: builder_bid.header.into(), block_value: builder_bid.value, }, BuilderBid::Deneb(builder_bid) => BlockProposalContents::PayloadAndBlobs { - payload: builder_bid.header, + payload: builder_bid.header.into(), block_value: builder_bid.value, kzg_commitments: builder_bid.blinded_blobs_bundle.commitments, blobs: BlobItems::try_from_blob_roots(builder_bid.blinded_blobs_bundle.blob_roots) diff --git a/consensus/types/Cargo.toml b/consensus/types/Cargo.toml index 9c105404b19..1b8aa203775 100644 --- a/consensus/types/Cargo.toml +++ b/consensus/types/Cargo.toml @@ -51,7 +51,6 @@ superstruct = "0.6.0" metastruct = "0.1.0" serde_json = "1.0.74" smallvec = "1.8.0" -serde_with = "1.13.0" maplit = "1.0.2" strum = { version = "0.24.0", features = ["derive"] } diff --git a/consensus/types/src/builder_bid.rs b/consensus/types/src/builder_bid.rs index 2c586cd14c8..8d4927f0bfd 100644 --- a/consensus/types/src/builder_bid.rs +++ b/consensus/types/src/builder_bid.rs @@ -1,19 +1,14 @@ -use std::marker::PhantomData; - -use serde::{Deserialize as De, Deserializer, Serialize as Ser, Serializer}; -use serde_derive::{Deserialize, Serialize}; -use serde_with::{As, DeserializeAs, SerializeAs}; -use superstruct::superstruct; -use tree_hash_derive::TreeHash; - -use bls::PublicKeyBytes; -use bls::Signature; - use crate::beacon_block_body::KzgCommitments; use crate::{ - AbstractExecPayload, BlobRootsList, ChainSpec, EthSpec, ExecPayload, ExecutionPayloadHeader, - ForkName, ForkVersionDeserialize, KzgProofs, SignedRoot, Uint256, + AbstractExecPayload, BlobRootsList, ChainSpec, EthSpec, ExecPayload, ForkName, + ForkVersionDeserialize, KzgProofs, SignedRoot, Uint256, }; +use bls::PublicKeyBytes; +use bls::Signature; +use serde::Deserializer; +use serde_derive::{Deserialize, Serialize}; +use superstruct::superstruct; +use tree_hash_derive::TreeHash; #[derive(PartialEq, Debug, Serialize, Deserialize, TreeHash, Clone)] #[serde(bound = "E: EthSpec")] @@ -27,27 +22,47 @@ pub struct BlindedBlobsBundle { variants(Merge, Capella, Deneb), variant_attributes( derive(PartialEq, Debug, Serialize, Deserialize, TreeHash, Clone), - serde(bound = "E: EthSpec, Payload: ExecPayload", deny_unknown_fields) + serde( + bound = "E: EthSpec, Payload: AbstractExecPayload", + deny_unknown_fields + ) ) )] #[derive(PartialEq, Debug, Serialize, Deserialize, TreeHash, Clone)] #[serde( - bound = "E: EthSpec, Payload: ExecPayload", + bound = "E: EthSpec, Payload: AbstractExecPayload", deny_unknown_fields, untagged )] #[tree_hash(enum_behaviour = "transparent")] pub struct BuilderBid> { - #[serde(with = "As::>")] - pub header: Payload, + #[superstruct(only(Merge), partial_getter(rename = "header_merge"))] + pub header: Payload::Merge, + #[superstruct(only(Capella), partial_getter(rename = "header_capella"))] + pub header: Payload::Capella, + #[superstruct(only(Deneb), partial_getter(rename = "header_deneb"))] + pub header: Payload::Deneb, #[superstruct(only(Deneb))] pub blinded_blobs_bundle: BlindedBlobsBundle, #[serde(with = "serde_utils::quoted_u256")] pub value: Uint256, pub pubkey: PublicKeyBytes, - #[serde(skip)] - #[tree_hash(skip_hashing)] - _phantom_data: PhantomData, +} + +impl> BuilderBid { + pub fn header(&self) -> Payload::Ref<'_> { + self.to_ref().header() + } +} + +impl<'a, T: EthSpec, Payload: AbstractExecPayload> BuilderBidRef<'a, T, Payload> { + pub fn header(&self) -> Payload::Ref<'a> { + match self { + Self::Merge(bid) => Payload::Ref::from(&bid.header), + Self::Capella(bid) => Payload::Ref::from(&bid.header), + Self::Deneb(bid) => Payload::Ref::from(&bid.header), + } + } } impl> SignedRoot for BuilderBid {} @@ -105,30 +120,6 @@ impl> ForkVersionDeserialize } } -struct BlindedPayloadAsHeader(PhantomData); - -impl> SerializeAs for BlindedPayloadAsHeader { - fn serialize_as(source: &Payload, serializer: S) -> Result - where - S: Serializer, - { - source.to_execution_payload_header().serialize(serializer) - } -} - -impl<'de, E: EthSpec, Payload: AbstractExecPayload> DeserializeAs<'de, Payload> - for BlindedPayloadAsHeader -{ - fn deserialize_as(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let payload_header = ExecutionPayloadHeader::deserialize(deserializer)?; - Payload::try_from(payload_header) - .map_err(|_| serde::de::Error::custom("unable to convert payload header to payload")) - } -} - impl> SignedBuilderBid { pub fn verify_signature(&self, spec: &ChainSpec) -> bool { self.message From 1ac148c052f5b089b71a89da5403fb62bc6d7b07 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Wed, 9 Aug 2023 17:45:33 +1000 Subject: [PATCH 23/24] Fix lint --- beacon_node/network/src/sync/block_lookups/tests.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/beacon_node/network/src/sync/block_lookups/tests.rs b/beacon_node/network/src/sync/block_lookups/tests.rs index b0217e5b74b..5793adfbb57 100644 --- a/beacon_node/network/src/sync/block_lookups/tests.rs +++ b/beacon_node/network/src/sync/block_lookups/tests.rs @@ -12,7 +12,6 @@ use beacon_chain::builder::Witness; use beacon_chain::eth1_chain::CachingEth1Backend; use beacon_chain::test_utils::{build_log, BeaconChainHarness, EphemeralHarnessType}; use beacon_processor::WorkEvent; -use execution_layer::BlobsBundleV1; use lighthouse_network::rpc::RPCResponseErrorCode; use lighthouse_network::{NetworkGlobals, Request}; use slot_clock::{ManualSlotClock, SlotClock, TestingSlotClock}; @@ -21,8 +20,8 @@ use tokio::sync::mpsc; use types::{ map_fork_name, map_fork_name_with, test_utils::{SeedableRng, TestRandom, XorShiftRng}, - BeaconBlock, BlobSidecar, EthSpec, ForkName, FullPayloadDeneb, MinimalEthSpec as E, - SignedBeaconBlock, + BeaconBlock, BlobSidecar, BlobsBundle, EthSpec, ForkName, FullPayloadDeneb, + MinimalEthSpec as E, SignedBeaconBlock, }; type T = Witness, E, MemoryStore, MemoryStore>; @@ -126,7 +125,7 @@ impl TestRig { } message.body.blob_kzg_commitments = bundle.commitments.clone(); - let BlobsBundleV1 { + let BlobsBundle { commitments, proofs, blobs, From ee2f1a1e192c3e30f17ab71001dec35174e49bd0 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 9 Aug 2023 11:53:20 -0400 Subject: [PATCH 24/24] remove generic from builder bid --- beacon_node/builder_client/src/lib.rs | 8 ++-- beacon_node/execution_layer/src/lib.rs | 25 +++++++---- consensus/types/src/builder_bid.rs | 61 +++++++++++--------------- 3 files changed, 45 insertions(+), 49 deletions(-) diff --git a/beacon_node/builder_client/src/lib.rs b/beacon_node/builder_client/src/lib.rs index 089e576f504..aebc3f5e2be 100644 --- a/beacon_node/builder_client/src/lib.rs +++ b/beacon_node/builder_client/src/lib.rs @@ -1,8 +1,8 @@ use eth2::types::builder_bid::SignedBuilderBid; use eth2::types::payload::FullPayloadContents; use eth2::types::{ - AbstractExecPayload, BlindedPayload, EthSpec, ExecutionBlockHash, ForkVersionedResponse, - PublicKeyBytes, SignedBlockContents, SignedValidatorRegistrationData, Slot, + BlindedPayload, EthSpec, ExecutionBlockHash, ForkVersionedResponse, PublicKeyBytes, + SignedBlockContents, SignedValidatorRegistrationData, Slot, }; pub use eth2::Error; use eth2::{ok_or_error, StatusCode}; @@ -163,12 +163,12 @@ impl BuilderHttpClient { } /// `GET /eth/v1/builder/header` - pub async fn get_builder_header>( + pub async fn get_builder_header( &self, slot: Slot, parent_hash: ExecutionBlockHash, pubkey: &PublicKeyBytes, - ) -> Result>>, Error> { + ) -> Result>>, Error> { let mut path = self.server.full.clone(); path.path_segments_mut() diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index ff49583b9da..6709abbae9e 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -89,23 +89,29 @@ pub enum ProvenancedPayload

{ Builder(P), } -impl> TryFrom> +impl> TryFrom> for ProvenancedPayload> { type Error = Error; - fn try_from(value: BuilderBid) -> Result { + fn try_from(value: BuilderBid) -> Result { let block_proposal_contents = match value { BuilderBid::Merge(builder_bid) => BlockProposalContents::Payload { - payload: builder_bid.header.into(), + payload: ExecutionPayloadHeader::Merge(builder_bid.header) + .try_into() + .map_err(|_| Error::InvalidPayloadConversion)?, block_value: builder_bid.value, }, BuilderBid::Capella(builder_bid) => BlockProposalContents::Payload { - payload: builder_bid.header.into(), + payload: ExecutionPayloadHeader::Capella(builder_bid.header) + .try_into() + .map_err(|_| Error::InvalidPayloadConversion)?, block_value: builder_bid.value, }, BuilderBid::Deneb(builder_bid) => BlockProposalContents::PayloadAndBlobs { - payload: builder_bid.header.into(), + payload: ExecutionPayloadHeader::Deneb(builder_bid.header) + .try_into() + .map_err(|_| Error::InvalidPayloadConversion)?, block_value: builder_bid.value, kzg_commitments: builder_bid.blinded_blobs_bundle.commitments, blobs: BlobItems::try_from_blob_roots(builder_bid.blinded_blobs_bundle.blob_roots) @@ -138,6 +144,7 @@ pub enum Error { InvalidJWTSecret(String), InvalidForkForPayload, InvalidPayloadBody(String), + InvalidPayloadConversion, InvalidBlobConversion(String), BeaconStateError(BeaconStateError), } @@ -844,7 +851,7 @@ impl ExecutionLayer { let ((relay_result, relay_duration), (local_result, local_duration)) = tokio::join!( timed_future(metrics::GET_BLINDED_PAYLOAD_BUILDER, async { builder - .get_builder_header::(slot, parent_hash, &pubkey) + .get_builder_header::(slot, parent_hash, &pubkey) .await }), timed_future(metrics::GET_BLINDED_PAYLOAD_LOCAL, async { @@ -1991,8 +1998,8 @@ impl fmt::Display for InvalidBuilderPayload { } /// Perform some cursory, non-exhaustive validation of the bid returned from the builder. -fn verify_builder_bid>( - bid: &ForkVersionedResponse>, +fn verify_builder_bid( + bid: &ForkVersionedResponse>, parent_hash: ExecutionBlockHash, payload_attributes: &PayloadAttributes, block_number: Option, @@ -2019,7 +2026,7 @@ fn verify_builder_bid>( .ok() .cloned() .map(|withdrawals| Withdrawals::::from(withdrawals).tree_hash_root()); - let payload_withdrawals_root = header.withdrawals_root().ok(); + let payload_withdrawals_root = header.withdrawals_root().ok().copied(); if *payload_value < profit_threshold { Err(Box::new(InvalidBuilderPayload::LowValue { diff --git a/consensus/types/src/builder_bid.rs b/consensus/types/src/builder_bid.rs index 8d4927f0bfd..e0c0bb306e8 100644 --- a/consensus/types/src/builder_bid.rs +++ b/consensus/types/src/builder_bid.rs @@ -1,7 +1,8 @@ use crate::beacon_block_body::KzgCommitments; use crate::{ - AbstractExecPayload, BlobRootsList, ChainSpec, EthSpec, ExecPayload, ForkName, - ForkVersionDeserialize, KzgProofs, SignedRoot, Uint256, + BlobRootsList, ChainSpec, EthSpec, ExecutionPayloadHeaderCapella, ExecutionPayloadHeaderDeneb, + ExecutionPayloadHeaderMerge, ExecutionPayloadHeaderRef, ForkName, ForkVersionDeserialize, + KzgProofs, SignedRoot, Uint256, }; use bls::PublicKeyBytes; use bls::Signature; @@ -22,26 +23,20 @@ pub struct BlindedBlobsBundle { variants(Merge, Capella, Deneb), variant_attributes( derive(PartialEq, Debug, Serialize, Deserialize, TreeHash, Clone), - serde( - bound = "E: EthSpec, Payload: AbstractExecPayload", - deny_unknown_fields - ) - ) + serde(bound = "E: EthSpec", deny_unknown_fields) + ), + map_ref_into(ExecutionPayloadHeaderRef) )] #[derive(PartialEq, Debug, Serialize, Deserialize, TreeHash, Clone)] -#[serde( - bound = "E: EthSpec, Payload: AbstractExecPayload", - deny_unknown_fields, - untagged -)] +#[serde(bound = "E: EthSpec", deny_unknown_fields, untagged)] #[tree_hash(enum_behaviour = "transparent")] -pub struct BuilderBid> { +pub struct BuilderBid { #[superstruct(only(Merge), partial_getter(rename = "header_merge"))] - pub header: Payload::Merge, + pub header: ExecutionPayloadHeaderMerge, #[superstruct(only(Capella), partial_getter(rename = "header_capella"))] - pub header: Payload::Capella, + pub header: ExecutionPayloadHeaderCapella, #[superstruct(only(Deneb), partial_getter(rename = "header_deneb"))] - pub header: Payload::Deneb, + pub header: ExecutionPayloadHeaderDeneb, #[superstruct(only(Deneb))] pub blinded_blobs_bundle: BlindedBlobsBundle, #[serde(with = "serde_utils::quoted_u256")] @@ -49,35 +44,31 @@ pub struct BuilderBid> { pub pubkey: PublicKeyBytes, } -impl> BuilderBid { - pub fn header(&self) -> Payload::Ref<'_> { +impl BuilderBid { + pub fn header(&self) -> ExecutionPayloadHeaderRef<'_, E> { self.to_ref().header() } } -impl<'a, T: EthSpec, Payload: AbstractExecPayload> BuilderBidRef<'a, T, Payload> { - pub fn header(&self) -> Payload::Ref<'a> { - match self { - Self::Merge(bid) => Payload::Ref::from(&bid.header), - Self::Capella(bid) => Payload::Ref::from(&bid.header), - Self::Deneb(bid) => Payload::Ref::from(&bid.header), - } +impl<'a, E: EthSpec> BuilderBidRef<'a, E> { + pub fn header(&self) -> ExecutionPayloadHeaderRef<'a, E> { + map_builder_bid_ref_into_execution_payload_header_ref!(&'a _, self, |bid, cons| cons( + &bid.header + )) } } -impl> SignedRoot for BuilderBid {} +impl SignedRoot for BuilderBid {} /// Validator registration, for use in interacting with servers implementing the builder API. #[derive(PartialEq, Debug, Serialize, Deserialize, Clone)] -#[serde(bound = "E: EthSpec, Payload: ExecPayload")] -pub struct SignedBuilderBid> { - pub message: BuilderBid, +#[serde(bound = "E: EthSpec")] +pub struct SignedBuilderBid { + pub message: BuilderBid, pub signature: Signature, } -impl> ForkVersionDeserialize - for BuilderBid -{ +impl ForkVersionDeserialize for BuilderBid { fn deserialize_by_fork<'de, D: Deserializer<'de>>( value: serde_json::value::Value, fork_name: ForkName, @@ -99,9 +90,7 @@ impl> ForkVersionDeserialize } } -impl> ForkVersionDeserialize - for SignedBuilderBid -{ +impl ForkVersionDeserialize for SignedBuilderBid { fn deserialize_by_fork<'de, D: Deserializer<'de>>( value: serde_json::value::Value, fork_name: ForkName, @@ -120,7 +109,7 @@ impl> ForkVersionDeserialize } } -impl> SignedBuilderBid { +impl SignedBuilderBid { pub fn verify_signature(&self, spec: &ChainSpec) -> bool { self.message .pubkey()