diff --git a/Cargo.lock b/Cargo.lock index 08d11a69934..204bc390827 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1665,6 +1665,26 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" +[[package]] +name = "context_deserialize" +version = "0.1.0" +dependencies = [ + "milhouse", + "serde", + "ssz_types", +] + +[[package]] +name = "context_deserialize_derive" +version = "0.1.0" +dependencies = [ + "context_deserialize", + "quote", + "serde", + "serde_json", + "syn 1.0.109", +] + [[package]] name = "convert_case" version = "0.4.0" @@ -9390,6 +9410,8 @@ dependencies = [ "bls", "compare_fields", "compare_fields_derive", + "context_deserialize", + "context_deserialize_derive", "criterion", "derivative", "eth2_interop_keypairs", diff --git a/Cargo.toml b/Cargo.toml index fa1cd9a0fd7..57794c2db1c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,6 +53,8 @@ members = [ "common/warp_utils", "common/workspace_members", + "consensus/context_deserialize", + "consensus/context_deserialize_derive", "consensus/fixed_bytes", "consensus/fork_choice", "consensus/int_to_bytes", @@ -127,6 +129,8 @@ clap = { version = "4.5.4", features = ["derive", "cargo", "wrap_help"] } # feature ourselves when desired. c-kzg = { version = "1", default-features = false } compare_fields_derive = { path = "common/compare_fields_derive" } +context_deserialize = { path = "consensus/context_deserialize" } +context_deserialize_derive = { path = "consensus/context_deserialize_derive" } criterion = "0.5" delay_map = "0.4" derivative = "2" diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 6dee8105956..c1d30253a36 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -6115,7 +6115,7 @@ impl BeaconChain { payload_attributes: payload_attributes.into(), }, metadata: Default::default(), - version: Some(self.spec.fork_name_at_slot::(prepare_slot)), + version: self.spec.fork_name_at_slot::(prepare_slot), })); } } diff --git a/beacon_node/builder_client/src/lib.rs b/beacon_node/builder_client/src/lib.rs index 6d82542cefc..d193eaf1d80 100644 --- a/beacon_node/builder_client/src/lib.rs +++ b/beacon_node/builder_client/src/lib.rs @@ -1,7 +1,7 @@ +use eth2::types::beacon_response::EmptyMetadata; use eth2::types::builder_bid::SignedBuilderBid; -use eth2::types::fork_versioned_response::EmptyMetadata; use eth2::types::{ - ContentType, EthSpec, ExecutionBlockHash, ForkName, ForkVersionDecode, ForkVersionDeserialize, + ContentType, ContextDeserialize, EthSpec, ExecutionBlockHash, ForkName, ForkVersionDecode, ForkVersionedResponse, PublicKeyBytes, SignedValidatorRegistrationData, Slot, }; use eth2::types::{FullPayloadContents, SignedBlindedBeaconBlock}; @@ -119,7 +119,7 @@ impl BuilderHttpClient { } async fn get_with_header< - T: DeserializeOwned + ForkVersionDecode + ForkVersionDeserialize, + T: DeserializeOwned + ForkVersionDecode + for<'de> ContextDeserialize<'de, ForkName>, U: IntoUrl, >( &self, @@ -147,7 +147,7 @@ impl BuilderHttpClient { self.ssz_available.store(true, Ordering::SeqCst); T::from_ssz_bytes_by_fork(&response_bytes, fork_name) .map(|data| ForkVersionedResponse { - version: Some(fork_name), + version: fork_name, metadata: EmptyMetadata {}, data, }) @@ -155,7 +155,15 @@ impl BuilderHttpClient { } ContentType::Json => { self.ssz_available.store(false, Ordering::SeqCst); - serde_json::from_slice(&response_bytes).map_err(Error::InvalidJson) + let mut de = serde_json::Deserializer::from_slice(&response_bytes); + let data = + T::context_deserialize(&mut de, fork_name).map_err(Error::InvalidJson)?; + + Ok(ForkVersionedResponse { + version: fork_name, + metadata: EmptyMetadata {}, + data, + }) } } } diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs index 43506c82deb..a581d5c128b 100644 --- a/beacon_node/client/src/builder.rs +++ b/beacon_node/client/src/builder.rs @@ -463,12 +463,12 @@ where let blobs = if block.message().body().has_blobs() { debug!("Downloading finalized blobs"); if let Some(response) = remote - .get_blobs::(BlockId::Root(block_root), None) + .get_blobs::(BlockId::Root(block_root), None, &spec) .await .map_err(|e| format!("Error fetching finalized blobs from remote: {e:?}"))? { debug!("Downloaded finalized blobs"); - Some(response.data) + Some(response.into_data()) } else { warn!( block_root = %block_root, diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index bbdf1a054b2..ddd8bb50088 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -1980,7 +1980,7 @@ enum InvalidBuilderPayload { expected: Option, }, Fork { - payload: Option, + payload: ForkName, expected: ForkName, }, Signature { @@ -2013,7 +2013,7 @@ impl fmt::Display for InvalidBuilderPayload { write!(f, "payload block number was {} not {:?}", payload, expected) } InvalidBuilderPayload::Fork { payload, expected } => { - write!(f, "payload fork was {:?} not {}", payload, expected) + write!(f, "payload fork was {} not {}", payload, expected) } InvalidBuilderPayload::Signature { signature, pubkey } => write!( f, @@ -2116,7 +2116,7 @@ fn verify_builder_bid( payload: header.block_number(), expected: block_number, })) - } else if bid.version != Some(current_fork) { + } else if bid.version != current_fork { Err(Box::new(InvalidBuilderPayload::Fork { payload: bid.version, expected: current_fork, diff --git a/beacon_node/execution_layer/src/test_utils/mock_builder.rs b/beacon_node/execution_layer/src/test_utils/mock_builder.rs index 87ea8642be5..3704bcc592b 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_builder.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_builder.rs @@ -743,7 +743,7 @@ impl MockBuilder { .await .map_err(|_| "couldn't get head".to_string())? .ok_or_else(|| "missing head block".to_string())? - .data; + .into_data(); let head_block_root = head_block_root.unwrap_or(head.canonical_root()); @@ -761,7 +761,7 @@ impl MockBuilder { .await .map_err(|_| "couldn't get finalized block".to_string())? .ok_or_else(|| "missing finalized block".to_string())? - .data + .data() .message() .body() .execution_payload() @@ -774,7 +774,7 @@ impl MockBuilder { .await .map_err(|_| "couldn't get justified block".to_string())? .ok_or_else(|| "missing justified block".to_string())? - .data + .data() .message() .body() .execution_payload() @@ -815,7 +815,7 @@ impl MockBuilder { .await .map_err(|_| "couldn't get state".to_string())? .ok_or_else(|| "missing state".to_string())? - .data; + .into_data(); let prev_randao = head_state .get_randao_mix(head_state.current_epoch()) @@ -980,7 +980,7 @@ pub fn serve( .await .map_err(|e| warp::reject::custom(Custom(e)))?; let resp: ForkVersionedResponse<_> = ForkVersionedResponse { - version: Some(fork_name), + version: fork_name, metadata: Default::default(), data: payload, }; @@ -1040,7 +1040,7 @@ pub fn serve( ), eth2::types::Accept::Json | eth2::types::Accept::Any => { let resp: ForkVersionedResponse<_> = ForkVersionedResponse { - version: Some(fork_name), + version: fork_name, metadata: Default::default(), data: signed_bid, }; diff --git a/beacon_node/http_api/src/aggregate_attestation.rs b/beacon_node/http_api/src/aggregate_attestation.rs index 23af5b0cb55..809f381139a 100644 --- a/beacon_node/http_api/src/aggregate_attestation.rs +++ b/beacon_node/http_api/src/aggregate_attestation.rs @@ -4,7 +4,7 @@ use crate::version::{add_consensus_version_header, V1, V2}; use beacon_chain::{BeaconChain, BeaconChainTypes}; use eth2::types::{self, EndpointVersion, Hash256, Slot}; use std::sync::Arc; -use types::fork_versioned_response::EmptyMetadata; +use types::beacon_response::EmptyMetadata; use types::{CommitteeIndex, ForkVersionedResponse}; use warp::{ hyper::{Body, Response}, @@ -52,7 +52,7 @@ pub fn get_aggregate_attestation( if endpoint_version == V2 { let fork_versioned_response = ForkVersionedResponse { - version: Some(fork_name), + version: fork_name, metadata: EmptyMetadata {}, data: aggregate_attestation, }; diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 93f978e3c93..ff3bfce19da 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -34,7 +34,7 @@ mod validators; mod version; use crate::light_client::{get_light_client_bootstrap, get_light_client_updates}; use crate::produce_block::{produce_blinded_block_v2, produce_block_v2, produce_block_v3}; -use crate::version::fork_versioned_response; +use crate::version::beacon_response; use beacon_chain::{ attestation_verification::VerifiedAttestation, observed_operations::ObservationOutcome, validator_monitor::timestamp_now, AttestationError as AttnError, BeaconChain, BeaconChainError, @@ -47,9 +47,9 @@ use bytes::Bytes; use directory::DEFAULT_ROOT_DIR; use either::Either; use eth2::types::{ - self as api_types, BroadcastValidation, EndpointVersion, ForkChoice, ForkChoiceNode, - LightClientUpdatesQuery, PublishBlockRequest, ValidatorBalancesRequestBody, ValidatorId, - ValidatorStatus, ValidatorsRequestBody, + self as api_types, BroadcastValidation, ContextDeserialize, EndpointVersion, ForkChoice, + ForkChoiceNode, LightClientUpdatesQuery, PublishBlockRequest, ValidatorBalancesRequestBody, + ValidatorId, ValidatorStatus, ValidatorsRequestBody, }; use eth2::{CONSENSUS_VERSION_HEADER, CONTENT_TYPE_HEADER, SSZ_CONTENT_TYPE_HEADER}; use health_metrics::observe::Observe; @@ -89,18 +89,17 @@ use tokio_stream::{ use tracing::{debug, error, info, warn}; use types::AttestationData; use types::{ - fork_versioned_response::EmptyMetadata, Attestation, AttestationShufflingId, AttesterSlashing, - BeaconStateError, ChainSpec, Checkpoint, CommitteeCache, ConfigAndPreset, Epoch, EthSpec, - ForkName, ForkVersionedResponse, Hash256, ProposerPreparationData, ProposerSlashing, - RelativeEpoch, SignedAggregateAndProof, SignedBlindedBeaconBlock, SignedBlsToExecutionChange, - SignedContributionAndProof, SignedValidatorRegistrationData, SignedVoluntaryExit, Slot, - SyncCommitteeMessage, SyncContributionData, + Attestation, AttestationShufflingId, AttesterSlashing, BeaconStateError, ChainSpec, Checkpoint, + CommitteeCache, ConfigAndPreset, Epoch, EthSpec, ForkName, Hash256, ProposerPreparationData, + ProposerSlashing, RelativeEpoch, SignedAggregateAndProof, SignedBlindedBeaconBlock, + SignedBlsToExecutionChange, SignedContributionAndProof, SignedValidatorRegistrationData, + SignedVoluntaryExit, Slot, SyncCommitteeMessage, SyncContributionData, }; use validator::pubkey_to_validator_index; use version::{ add_consensus_version_header, add_ssz_content_type_header, - execution_optimistic_finalized_fork_versioned_response, inconsistent_fork_rejection, - unsupported_version_rejection, V1, V2, V3, + execution_optimistic_finalized_beacon_response, inconsistent_fork_rejection, + unsupported_version_rejection, ResponseIncludesVersion, V1, V2, V3, }; use warp::http::StatusCode; use warp::hyper::Body; @@ -1153,8 +1152,8 @@ pub fn serve( |state_id: StateId, task_spawner: TaskSpawner, chain: Arc>| { - task_spawner.blocking_json_task(Priority::P1, move || { - let (data, execution_optimistic, finalized) = state_id + task_spawner.blocking_response_task(Priority::P1, move || { + let (data, execution_optimistic, finalized, fork_name) = state_id .map_state_and_execution_optimistic_and_finalized( &chain, |state, execution_optimistic, finalized| { @@ -1164,15 +1163,23 @@ pub fn serve( )); }; - Ok((deposits.clone(), execution_optimistic, finalized)) + Ok(( + deposits.clone(), + execution_optimistic, + finalized, + state.fork_name_unchecked(), + )) }, )?; - Ok(api_types::ExecutionOptimisticFinalizedResponse { + execution_optimistic_finalized_beacon_response( + ResponseIncludesVersion::Yes(fork_name), + execution_optimistic, + finalized, data, - execution_optimistic: Some(execution_optimistic), - finalized: Some(finalized), - }) + ) + .map(|res| warp::reply::json(&res).into_response()) + .map(|resp| add_consensus_version_header(resp, fork_name)) }) }, ); @@ -1186,8 +1193,8 @@ pub fn serve( |state_id: StateId, task_spawner: TaskSpawner, chain: Arc>| { - task_spawner.blocking_json_task(Priority::P1, move || { - let (data, execution_optimistic, finalized) = state_id + task_spawner.blocking_response_task(Priority::P1, move || { + let (data, execution_optimistic, finalized, fork_name) = state_id .map_state_and_execution_optimistic_and_finalized( &chain, |state, execution_optimistic, finalized| { @@ -1197,15 +1204,23 @@ pub fn serve( )); }; - Ok((withdrawals.clone(), execution_optimistic, finalized)) + Ok(( + withdrawals.clone(), + execution_optimistic, + finalized, + state.fork_name_unchecked(), + )) }, )?; - Ok(api_types::ExecutionOptimisticFinalizedResponse { + execution_optimistic_finalized_beacon_response( + ResponseIncludesVersion::Yes(fork_name), + execution_optimistic, + finalized, data, - execution_optimistic: Some(execution_optimistic), - finalized: Some(finalized), - }) + ) + .map(|res| warp::reply::json(&res).into_response()) + .map(|resp| add_consensus_version_header(resp, fork_name)) }) }, ); @@ -1405,21 +1420,30 @@ pub fn serve( .and(warp::path("beacon")) .and(warp::path("blocks")) .and(warp::path::end()) - .and(warp_utils::json::json()) + .and(warp::body::json()) + .and(consensus_version_header_filter) .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and(network_tx_filter.clone()) .and(network_globals.clone()) .then( - move |block_contents: PublishBlockRequest, + move |value: serde_json::Value, + consensus_version: ForkName, task_spawner: TaskSpawner, chain: Arc>, network_tx: UnboundedSender>, network_globals: Arc>| { task_spawner.spawn_async_with_rejection(Priority::P0, async move { + let request = PublishBlockRequest::::context_deserialize( + &value, + consensus_version, + ) + .map_err(|e| { + warp_utils::reject::custom_bad_request(format!("invalid JSON: {e:?}")) + })?; publish_blocks::publish_block( None, - ProvenancedBlock::local_from_publish_request(block_contents), + ProvenancedBlock::local_from_publish_request(request), chain, &network_tx, BroadcastValidation::default(), @@ -1475,22 +1499,32 @@ pub fn serve( .and(warp::path("blocks")) .and(warp::query::()) .and(warp::path::end()) - .and(warp_utils::json::json()) + .and(warp::body::json()) + .and(consensus_version_header_filter) .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and(network_tx_filter.clone()) .and(network_globals.clone()) .then( move |validation_level: api_types::BroadcastValidationQuery, - block_contents: PublishBlockRequest, + value: serde_json::Value, + consensus_version: ForkName, task_spawner: TaskSpawner, chain: Arc>, network_tx: UnboundedSender>, network_globals: Arc>| { task_spawner.spawn_async_with_rejection(Priority::P0, async move { + let request = PublishBlockRequest::::context_deserialize( + &value, + consensus_version, + ) + .map_err(|e| { + warp_utils::reject::custom_bad_request(format!("invalid JSON: {e:?}")) + })?; + publish_blocks::publish_block( None, - ProvenancedBlock::local_from_publish_request(block_contents), + ProvenancedBlock::local_from_publish_request(request), chain, &network_tx, validation_level.broadcast_validation, @@ -1723,6 +1757,12 @@ pub fn serve( .fork_name(&chain.spec) .map_err(inconsistent_fork_rejection)?; + let require_version = match endpoint_version { + V1 => ResponseIncludesVersion::No, + V2 => ResponseIncludesVersion::Yes(fork_name), + _ => return Err(unsupported_version_rejection(endpoint_version)), + }; + match accept_header { Some(api_types::Accept::Ssz) => Response::builder() .status(200) @@ -1734,9 +1774,8 @@ pub fn serve( e )) }), - _ => execution_optimistic_finalized_fork_versioned_response( - endpoint_version, - fork_name, + _ => execution_optimistic_finalized_beacon_response( + require_version, execution_optimistic, finalized, block, @@ -1796,9 +1835,15 @@ pub fn serve( .attestations() .map(|att| att.clone_as_attestation()) .collect::>(); - let res = execution_optimistic_finalized_fork_versioned_response( - endpoint_version, - fork_name, + + let require_version = match endpoint_version { + V1 => ResponseIncludesVersion::No, + V2 => ResponseIncludesVersion::Yes(fork_name), + _ => return Err(unsupported_version_rejection(endpoint_version)), + }; + + let res = execution_optimistic_finalized_beacon_response( + require_version, execution_optimistic, finalized, &atts, @@ -1845,9 +1890,8 @@ pub fn serve( }), _ => { // Post as a V2 endpoint so we return the fork version. - execution_optimistic_finalized_fork_versioned_response( - V2, - fork_name, + execution_optimistic_finalized_beacon_response( + ResponseIncludesVersion::Yes(fork_name), execution_optimistic, finalized, block, @@ -1901,9 +1945,8 @@ pub fn serve( }), _ => { // Post as a V2 endpoint so we return the fork version. - let res = execution_optimistic_finalized_fork_versioned_response( - V2, - fork_name, + let res = execution_optimistic_finalized_beacon_response( + ResponseIncludesVersion::Yes(fork_name), execution_optimistic, finalized, &blob_sidecar_list_filtered, @@ -2063,7 +2106,13 @@ pub fn serve( }) .collect::>(); - let res = fork_versioned_response(endpoint_version, fork_name, &attestations)?; + let require_version = match endpoint_version { + V1 => ResponseIncludesVersion::No, + V2 => ResponseIncludesVersion::Yes(fork_name), + _ => return Err(unsupported_version_rejection(endpoint_version)), + }; + + let res = beacon_response(require_version, &attestations); Ok(add_consensus_version_header( warp::reply::json(&res).into_response(), fork_name, @@ -2152,7 +2201,13 @@ pub fn serve( }) .collect::>(); - let res = fork_versioned_response(endpoint_version, fork_name, &slashings)?; + let require_version = match endpoint_version { + V1 => ResponseIncludesVersion::No, + V2 => ResponseIncludesVersion::Yes(fork_name), + _ => return Err(unsupported_version_rejection(endpoint_version)), + }; + + let res = beacon_response(require_version, &slashings); Ok(add_consensus_version_header( warp::reply::json(&res).into_response(), fork_name, @@ -2588,7 +2643,7 @@ pub fn serve( let fork_name = chain .spec - .fork_name_at_slot::(*update.signature_slot()); + .fork_name_at_slot::(update.get_slot()); match accept_header { Some(api_types::Accept::Ssz) => Response::builder() .status(200) @@ -2600,11 +2655,10 @@ pub fn serve( e )) }), - _ => Ok(warp::reply::json(&ForkVersionedResponse { - version: Some(fork_name), - metadata: EmptyMetadata {}, - data: update, - }) + _ => Ok(warp::reply::json(&beacon_response( + ResponseIncludesVersion::Yes(fork_name), + update, + )) .into_response()), } .map(|resp| add_consensus_version_header(resp, fork_name)) @@ -2649,11 +2703,10 @@ pub fn serve( e )) }), - _ => Ok(warp::reply::json(&ForkVersionedResponse { - version: Some(fork_name), - metadata: EmptyMetadata {}, - data: update, - }) + _ => Ok(warp::reply::json(&beacon_response( + ResponseIncludesVersion::Yes(fork_name), + update, + )) .into_response()), } .map(|resp| add_consensus_version_header(resp, fork_name)) @@ -2845,7 +2898,7 @@ pub fn serve( .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .then( - |endpoint_version: EndpointVersion, + |_endpoint_version: EndpointVersion, state_id: StateId, accept_header: Option, task_spawner: TaskSpawner, @@ -2889,9 +2942,8 @@ pub fn serve( let fork_name = state .fork_name(&chain.spec) .map_err(inconsistent_fork_rejection)?; - let res = execution_optimistic_finalized_fork_versioned_response( - endpoint_version, - fork_name, + let res = execution_optimistic_finalized_beacon_response( + ResponseIncludesVersion::Yes(fork_name), execution_optimistic, finalized, &state, @@ -3372,7 +3424,7 @@ pub fn serve( if endpoint_version == V3 { produce_block_v3(accept_header, chain, slot, query).await } else { - produce_block_v2(endpoint_version, accept_header, chain, slot, query).await + produce_block_v2(accept_header, chain, slot, query).await } }) }, @@ -3402,8 +3454,7 @@ pub fn serve( chain: Arc>| { task_spawner.spawn_async_with_rejection(Priority::P0, async move { not_synced_filter?; - produce_blinded_block_v2(EndpointVersion(2), accept_header, chain, slot, query) - .await + produce_blinded_block_v2(accept_header, chain, slot, query).await }) }, ); diff --git a/beacon_node/http_api/src/light_client.rs b/beacon_node/http_api/src/light_client.rs index 2d0a5d09a1a..24b1338a724 100644 --- a/beacon_node/http_api/src/light_client.rs +++ b/beacon_node/http_api/src/light_client.rs @@ -1,14 +1,15 @@ use crate::version::{ - add_consensus_version_header, add_ssz_content_type_header, fork_versioned_response, V1, + add_consensus_version_header, add_ssz_content_type_header, beacon_response, + ResponseIncludesVersion, }; use beacon_chain::{BeaconChain, BeaconChainError, BeaconChainTypes}; use eth2::types::{ - self as api_types, ChainSpec, ForkVersionedResponse, LightClientUpdate, - LightClientUpdateResponseChunk, LightClientUpdateResponseChunkInner, LightClientUpdatesQuery, + self as api_types, ChainSpec, LightClientUpdate, LightClientUpdateResponseChunk, + LightClientUpdateResponseChunkInner, LightClientUpdatesQuery, }; use ssz::Encode; use std::sync::Arc; -use types::{ForkName, Hash256, LightClientBootstrap}; +use types::{BeaconResponse, ForkName, Hash256, LightClientBootstrap}; use warp::{ hyper::{Body, Response}, reply::Reply, @@ -52,7 +53,7 @@ pub fn get_light_client_updates( let fork_versioned_response = light_client_updates .iter() .map(|update| map_light_client_update_to_json_response::(&chain, update.clone())) - .collect::>>, Rejection>>()?; + .collect::>>>(); Ok(warp::reply::json(&fork_versioned_response).into_response()) } } @@ -88,10 +89,8 @@ pub fn get_light_client_bootstrap( warp_utils::reject::custom_server_error(format!("failed to create response: {}", e)) }), _ => { - let fork_versioned_response = map_light_client_bootstrap_to_json_response::( - fork_name, - light_client_bootstrap, - )?; + let fork_versioned_response = + map_light_client_bootstrap_to_json_response::(fork_name, light_client_bootstrap); Ok(warp::reply::json(&fork_versioned_response).into_response()) } } @@ -177,17 +176,20 @@ fn map_light_client_update_to_ssz_chunk( fn map_light_client_bootstrap_to_json_response( fork_name: ForkName, light_client_bootstrap: LightClientBootstrap, -) -> Result>, Rejection> { - fork_versioned_response(V1, fork_name, light_client_bootstrap) +) -> BeaconResponse> { + beacon_response( + ResponseIncludesVersion::Yes(fork_name), + light_client_bootstrap, + ) } fn map_light_client_update_to_json_response( chain: &BeaconChain, light_client_update: LightClientUpdate, -) -> Result>, Rejection> { +) -> BeaconResponse> { let fork_name = chain .spec .fork_name_at_slot::(*light_client_update.signature_slot()); - fork_versioned_response(V1, fork_name, light_client_update) + beacon_response(ResponseIncludesVersion::Yes(fork_name), light_client_update) } diff --git a/beacon_node/http_api/src/produce_block.rs b/beacon_node/http_api/src/produce_block.rs index 22d6f0e7ae1..db82ff214c8 100644 --- a/beacon_node/http_api/src/produce_block.rs +++ b/beacon_node/http_api/src/produce_block.rs @@ -3,15 +3,14 @@ use crate::{ version::{ add_consensus_block_value_header, add_consensus_version_header, add_execution_payload_blinded_header, add_execution_payload_value_header, - add_ssz_content_type_header, fork_versioned_response, inconsistent_fork_rejection, + add_ssz_content_type_header, beacon_response, inconsistent_fork_rejection, + ResponseIncludesVersion, }, }; use beacon_chain::{ BeaconBlockResponseWrapper, BeaconChain, BeaconChainTypes, ProduceBlockVerification, }; -use eth2::types::{ - self as api_types, EndpointVersion, ProduceBlockV3Metadata, SkipRandaoVerification, -}; +use eth2::types::{self as api_types, ProduceBlockV3Metadata, SkipRandaoVerification}; use ssz::Encode; use std::sync::Arc; use types::{payload::BlockProductionVersion, *}; @@ -115,7 +114,7 @@ pub fn build_response_v3( warp_utils::reject::custom_server_error(format!("failed to create response: {}", e)) }), _ => Ok(warp::reply::json(&ForkVersionedResponse { - version: Some(fork_name), + version: fork_name, metadata, data: block_contents, }) @@ -129,7 +128,6 @@ pub fn build_response_v3( } pub async fn produce_blinded_block_v2( - endpoint_version: EndpointVersion, accept_header: Option, chain: Arc>, slot: Slot, @@ -155,11 +153,10 @@ pub async fn produce_blinded_block_v2( .await .map_err(warp_utils::reject::unhandled_error)?; - build_response_v2(chain, block_response_type, endpoint_version, accept_header) + build_response_v2(chain, block_response_type, accept_header) } pub async fn produce_block_v2( - endpoint_version: EndpointVersion, accept_header: Option, chain: Arc>, slot: Slot, @@ -186,13 +183,12 @@ pub async fn produce_block_v2( .await .map_err(warp_utils::reject::unhandled_error)?; - build_response_v2(chain, block_response_type, endpoint_version, accept_header) + build_response_v2(chain, block_response_type, accept_header) } pub fn build_response_v2( chain: Arc>, block_response: BeaconBlockResponseWrapper, - endpoint_version: EndpointVersion, accept_header: Option, ) -> Result, warp::Rejection> { let fork_name = block_response @@ -210,8 +206,10 @@ pub fn build_response_v2( .map_err(|e| { warp_utils::reject::custom_server_error(format!("failed to create response: {}", e)) }), - _ => fork_versioned_response(endpoint_version, fork_name, block_contents) - .map(|response| warp::reply::json(&response).into_response()) - .map(|res| add_consensus_version_header(res, fork_name)), + _ => Ok(warp::reply::json(&beacon_response( + ResponseIncludesVersion::Yes(fork_name), + block_contents, + )) + .into_response()), } } diff --git a/beacon_node/http_api/src/version.rs b/beacon_node/http_api/src/version.rs index 59816cb8972..361e8e78eae 100644 --- a/beacon_node/http_api/src/version.rs +++ b/beacon_node/http_api/src/version.rs @@ -5,10 +5,11 @@ use eth2::{ }; use serde::Serialize; use types::{ - fork_versioned_response::{ - ExecutionOptimisticFinalizedForkVersionedResponse, ExecutionOptimisticFinalizedMetadata, + beacon_response::{ + ExecutionOptimisticFinalizedBeaconResponse, ExecutionOptimisticFinalizedMetadata, }, - ForkName, ForkVersionedResponse, InconsistentFork, Uint256, + BeaconResponse, ForkName, ForkVersionedResponse, InconsistentFork, Uint256, + UnversionedResponse, }; use warp::reply::{self, Reply, Response}; @@ -16,47 +17,54 @@ pub const V1: EndpointVersion = EndpointVersion(1); pub const V2: EndpointVersion = EndpointVersion(2); pub const V3: EndpointVersion = EndpointVersion(3); -pub fn fork_versioned_response( - endpoint_version: EndpointVersion, - fork_name: ForkName, +#[derive(Debug, PartialEq, Clone, Serialize)] +pub enum ResponseIncludesVersion { + Yes(ForkName), + No, +} + +pub fn beacon_response( + require_version: ResponseIncludesVersion, data: T, -) -> Result, warp::reject::Rejection> { - let fork_name = if endpoint_version == V1 { - None - } else if endpoint_version == V2 || endpoint_version == V3 { - Some(fork_name) - } else { - return Err(unsupported_version_rejection(endpoint_version)); - }; - Ok(ForkVersionedResponse { - version: fork_name, - metadata: Default::default(), - data, - }) +) -> BeaconResponse { + match require_version { + ResponseIncludesVersion::Yes(fork_name) => { + BeaconResponse::ForkVersioned(ForkVersionedResponse { + version: fork_name, + metadata: Default::default(), + data, + }) + } + ResponseIncludesVersion::No => BeaconResponse::Unversioned(UnversionedResponse { + metadata: Default::default(), + data, + }), + } } -pub fn execution_optimistic_finalized_fork_versioned_response( - endpoint_version: EndpointVersion, - fork_name: ForkName, +pub fn execution_optimistic_finalized_beacon_response( + require_version: ResponseIncludesVersion, execution_optimistic: bool, finalized: bool, data: T, -) -> Result, warp::reject::Rejection> { - let fork_name = if endpoint_version == V1 { - None - } else if endpoint_version == V2 { - Some(fork_name) - } else { - return Err(unsupported_version_rejection(endpoint_version)); +) -> Result, warp::reject::Rejection> { + let metadata = ExecutionOptimisticFinalizedMetadata { + execution_optimistic: Some(execution_optimistic), + finalized: Some(finalized), }; - Ok(ExecutionOptimisticFinalizedForkVersionedResponse { - version: fork_name, - metadata: ExecutionOptimisticFinalizedMetadata { - execution_optimistic: Some(execution_optimistic), - finalized: Some(finalized), - }, - data, - }) + match require_version { + ResponseIncludesVersion::Yes(fork_name) => { + Ok(BeaconResponse::ForkVersioned(ForkVersionedResponse { + version: fork_name, + metadata, + data, + })) + } + ResponseIncludesVersion::No => Ok(BeaconResponse::Unversioned(UnversionedResponse { + metadata, + data, + })), + } } /// Add the 'Content-Type application/octet-stream` header to a response. diff --git a/beacon_node/http_api/tests/interactive_tests.rs b/beacon_node/http_api/tests/interactive_tests.rs index bb3086945bc..4f3cd6c8285 100644 --- a/beacon_node/http_api/tests/interactive_tests.rs +++ b/beacon_node/http_api/tests/interactive_tests.rs @@ -115,10 +115,10 @@ async fn state_by_root_pruned_from_fork_choice() { .unwrap() .unwrap(); - assert!(response.metadata.finalized.unwrap()); - assert!(!response.metadata.execution_optimistic.unwrap()); + assert!(response.metadata().finalized.unwrap()); + assert!(!response.metadata().execution_optimistic.unwrap()); - let mut state = response.data; + let mut state = response.into_data(); assert_eq!(state.update_tree_hash_cache().unwrap(), state_root); } } @@ -846,7 +846,7 @@ pub async fn fork_choice_before_proposal() { .get_validator_blocks::(slot_d, &randao_reveal, None) .await .unwrap() - .data + .into_data() .deconstruct() .0; diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index b2b9ab8d093..33ac8e413de 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -682,7 +682,7 @@ impl ApiTester { .await .unwrap() .unwrap() - .metadata + .metadata() .finalized .unwrap(); @@ -719,7 +719,7 @@ impl ApiTester { .await .unwrap() .unwrap() - .metadata + .metadata() .finalized .unwrap(); @@ -757,7 +757,7 @@ impl ApiTester { .await .unwrap() .unwrap() - .metadata + .metadata() .finalized .unwrap(); @@ -1597,9 +1597,9 @@ impl ApiTester { let json_result = self.client.get_beacon_blocks(block_id.0).await.unwrap(); if let (Some(json), Some(expected)) = (&json_result, &expected) { - assert_eq!(&json.data, expected.as_ref(), "{:?}", block_id); + assert_eq!(json.data(), expected.as_ref(), "{:?}", block_id); assert_eq!( - json.version, + json.version(), Some(expected.fork_name(&self.chain.spec).unwrap()) ); } else { @@ -1623,8 +1623,8 @@ impl ApiTester { // Check that the legacy v1 API still works but doesn't return a version field. let v1_result = self.client.get_beacon_blocks_v1(block_id.0).await.unwrap(); if let (Some(v1_result), Some(expected)) = (&v1_result, &expected) { - assert_eq!(v1_result.version, None); - assert_eq!(&v1_result.data, expected.as_ref()); + assert_eq!(v1_result.version(), None); + assert_eq!(v1_result.data(), expected.as_ref()); } else { assert_eq!(v1_result, None); assert_eq!(expected, None); @@ -1685,9 +1685,9 @@ impl ApiTester { .unwrap(); if let (Some(json), Some(expected)) = (&json_result, &expected) { - assert_eq!(&json.data, expected, "{:?}", block_id); + assert_eq!(json.data(), expected, "{:?}", block_id); assert_eq!( - json.version, + json.version(), Some(expected.fork_name(&self.chain.spec).unwrap()) ); } else { @@ -1750,10 +1750,14 @@ impl ApiTester { }; let result = match self .client - .get_blobs::(CoreBlockId::Root(block_root), blob_indices.as_deref()) + .get_blobs::( + CoreBlockId::Root(block_root), + blob_indices.as_deref(), + &self.chain.spec, + ) .await { - Ok(result) => result.unwrap().data, + Ok(result) => result.unwrap().into_data(), Err(e) => panic!("query failed incorrectly: {e:?}"), }; @@ -1806,13 +1810,13 @@ impl ApiTester { match self .client - .get_blobs::(CoreBlockId::Slot(test_slot), None) + .get_blobs::(CoreBlockId::Slot(test_slot), None, &self.chain.spec) .await { Ok(result) => { if zero_blobs { assert_eq!( - &result.unwrap().data[..], + &result.unwrap().into_data()[..], &[], "empty blobs are always available" ); @@ -1844,7 +1848,7 @@ impl ApiTester { match self .client - .get_blobs::(CoreBlockId::Slot(test_slot), None) + .get_blobs::(CoreBlockId::Slot(test_slot), None, &self.chain.spec) .await { Ok(result) => panic!("queries for pre-Deneb slots should fail. got: {result:?}"), @@ -1861,7 +1865,7 @@ impl ApiTester { .get_beacon_blocks_attestations_v2(block_id.0) .await .unwrap() - .map(|res| res.data); + .map(|res| res.into_data()); let expected = block_id.full_block(&self.chain).await.ok().map( |(block, _execution_optimistic, _finalized)| { @@ -2071,7 +2075,7 @@ impl ApiTester { .get_light_client_bootstrap(&self.chain.store, &block_root, 1u64, &self.chain.spec); assert!(expected.is_ok()); - assert_eq!(result.unwrap().data, expected.unwrap().unwrap().0); + assert_eq!(result.unwrap().data(), &expected.unwrap().unwrap().0); self } @@ -2083,7 +2087,7 @@ impl ApiTester { .get_beacon_light_client_optimistic_update::() .await { - Ok(result) => result.map(|res| res.data), + Ok(result) => result.map(|res| res.into_data()), Err(e) => panic!("query failed incorrectly: {e:?}"), }; @@ -2102,7 +2106,7 @@ impl ApiTester { .get_beacon_light_client_finality_update::() .await { - Ok(result) => result.map(|res| res.data), + Ok(result) => result.map(|res| res.into_data()), Err(e) => panic!("query failed incorrectly: {e:?}"), }; @@ -2133,7 +2137,7 @@ impl ApiTester { .get_beacon_pool_attestations_v2(None, None) .await .unwrap() - .data; + .into_data(); assert_eq!(result, expected); @@ -2195,7 +2199,7 @@ impl ApiTester { .get_beacon_pool_attestations_v2(None, Some(0)) .await .unwrap() - .data; + .into_data(); let mut expected = self.chain.op_pool.get_all_attestations(); expected.extend(self.chain.naive_aggregation_pool.read().iter().cloned()); let expected_committee_index_filtered = expected @@ -2311,7 +2315,7 @@ impl ApiTester { .get_beacon_pool_attester_slashings_v2() .await .unwrap() - .data; + .into_data(); assert_eq!(result, expected); self @@ -2649,9 +2653,9 @@ impl ApiTester { expected.as_mut().map(|state| state.drop_all_caches()); if let (Some(json), Some(expected)) = (&result_json, &expected) { - assert_eq!(json.data, *expected, "{:?}", state_id); + assert_eq!(json.data(), expected, "{:?}", state_id); assert_eq!( - json.version, + json.version(), Some(expected.fork_name(&self.chain.spec).unwrap()) ); } else { @@ -3157,7 +3161,7 @@ impl ApiTester { .get_validator_blocks::(slot, &randao_reveal, None) .await .unwrap() - .data + .into_data() .deconstruct() .0; @@ -3254,7 +3258,7 @@ impl ApiTester { ) { // Compare fork name to ForkVersionedResponse rather than metadata consensus_version, which // is deserialized to a dummy value. - assert_eq!(Some(metadata.consensus_version), response.version); + assert_eq!(metadata.consensus_version, response.version); assert_eq!(ForkName::Base, response.metadata.consensus_version); assert_eq!( metadata.execution_payload_blinded, @@ -3380,7 +3384,7 @@ impl ApiTester { ) .await .unwrap() - .data + .into_data() .deconstruct() .0; assert_eq!(block.slot(), slot); @@ -3494,7 +3498,7 @@ impl ApiTester { .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() - .data; + .into_data(); let signed_block = block.sign(&sk, &fork, genesis_validators_root, &self.chain.spec); @@ -3509,7 +3513,7 @@ impl ApiTester { .await .unwrap() .unwrap() - .data; + .into_data(); assert_eq!(head_block.clone_as_blinded(), signed_block); @@ -3582,7 +3586,7 @@ impl ApiTester { .await .unwrap() .unwrap() - .data; + .into_data(); let signed_block = signed_block_contents.signed_block(); assert_eq!(head_block, **signed_block); @@ -3605,7 +3609,7 @@ impl ApiTester { ) .await .unwrap() - .data; + .into_data(); assert_eq!(blinded_block.slot(), slot); self.chain.slot_clock.set_slot(slot.as_u64() + 1); } @@ -3749,7 +3753,7 @@ impl ApiTester { .await .unwrap() .unwrap() - .data; + .into_data(); let expected = attestation; assert_eq!(result, expected); @@ -4323,7 +4327,7 @@ impl ApiTester { .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() - .data + .into_data() .body() .execution_payload() .unwrap() @@ -4369,7 +4373,7 @@ impl ApiTester { .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() - .data + .into_data() .body() .execution_payload() .unwrap() @@ -4413,7 +4417,7 @@ impl ApiTester { .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() - .data + .into_data() .body() .execution_payload() .unwrap() @@ -4487,7 +4491,7 @@ impl ApiTester { .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() - .data + .into_data() .body() .execution_payload() .unwrap() @@ -4573,7 +4577,7 @@ impl ApiTester { .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() - .data + .into_data() .body() .execution_payload() .unwrap() @@ -4665,7 +4669,7 @@ impl ApiTester { .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() - .data + .into_data() .body() .execution_payload() .unwrap() @@ -4755,7 +4759,7 @@ impl ApiTester { .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() - .data + .into_data() .body() .execution_payload() .unwrap() @@ -4844,7 +4848,7 @@ impl ApiTester { .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() - .data + .into_data() .body() .execution_payload() .unwrap() @@ -4919,7 +4923,7 @@ impl ApiTester { .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() - .data + .into_data() .body() .execution_payload() .unwrap() @@ -4982,7 +4986,7 @@ impl ApiTester { .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() - .data + .into_data() .body() .execution_payload() .unwrap() @@ -5058,7 +5062,7 @@ impl ApiTester { .get_validator_blinded_blocks::(next_slot, &randao_reveal, None) .await .unwrap() - .data + .into_data() .body() .execution_payload() .unwrap() @@ -5089,7 +5093,7 @@ impl ApiTester { .get_validator_blinded_blocks::(next_slot, &randao_reveal, None) .await .unwrap() - .data + .into_data() .body() .execution_payload() .unwrap() @@ -5197,7 +5201,7 @@ impl ApiTester { .get_validator_blinded_blocks::(next_slot, &randao_reveal, None) .await .unwrap() - .data + .into_data() .body() .execution_payload() .unwrap() @@ -5238,7 +5242,7 @@ impl ApiTester { .get_validator_blinded_blocks::(next_slot, &randao_reveal, None) .await .unwrap() - .data + .into_data() .body() .execution_payload() .unwrap() @@ -5354,7 +5358,7 @@ impl ApiTester { .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() - .data + .into_data() .body() .execution_payload() .unwrap() @@ -5435,7 +5439,7 @@ impl ApiTester { .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() - .data + .into_data() .body() .execution_payload() .unwrap() @@ -5503,7 +5507,7 @@ impl ApiTester { .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() - .data + .into_data() .body() .execution_payload() .unwrap() @@ -5571,7 +5575,7 @@ impl ApiTester { .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() - .data + .into_data() .body() .execution_payload() .unwrap() @@ -5638,7 +5642,7 @@ impl ApiTester { .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() - .data + .into_data() .body() .execution_payload() .unwrap() @@ -5709,7 +5713,7 @@ impl ApiTester { .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() - .data + .into_data() .body() .execution_payload() .unwrap() diff --git a/beacon_node/tests/test.rs b/beacon_node/tests/test.rs index 0d448e6c069..ab78b65ae93 100644 --- a/beacon_node/tests/test.rs +++ b/beacon_node/tests/test.rs @@ -41,7 +41,7 @@ fn http_server_genesis_state() { .block_on(remote_node.get_debug_beacon_states(StateId::Slot(Slot::new(0)))) .expect("should fetch state from http api") .unwrap() - .data; + .into_data(); let mut db_state = node .client diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index d1b0dc65c47..d7c25ecb820 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -16,7 +16,7 @@ pub mod types; use self::mixin::{RequestAccept, ResponseOptional}; use self::types::{Error as ResponseError, *}; -use ::types::fork_versioned_response::ExecutionOptimisticFinalizedForkVersionedResponse; +use ::types::beacon_response::ExecutionOptimisticFinalizedBeaconResponse; use derivative::Derivative; use either::Either; use futures::Stream; @@ -283,6 +283,52 @@ impl BeaconNodeHttpClient { } } + pub async fn get_fork_contextual( + &self, + url: U, + ctx_constructor: impl Fn(ForkName) -> Ctx, + ) -> Result>, Error> + where + U: IntoUrl, + T: ContextDeserialize<'static, Ctx>, + Meta: DeserializeOwned, + Ctx: Clone, + { + let response = self + .get_response(url, |b| b.accept(Accept::Json)) + .await + .optional()?; + + let Some(resp) = response else { + return Ok(None); + }; + + let bytes = resp.bytes().await?; + + #[derive(serde::Deserialize)] + struct Helper { + version: ForkName, + #[serde(flatten)] + metadata: serde_json::Value, + data: serde_json::Value, + } + + let helper: Helper = serde_json::from_slice(&bytes).map_err(Error::InvalidJson)?; + + let metadata: Meta = serde_json::from_value(helper.metadata).map_err(Error::InvalidJson)?; + + let ctx = ctx_constructor(helper.version); + + let data: T = ContextDeserialize::context_deserialize(helper.data, ctx) + .map_err(Error::InvalidJson)?; + + Ok(Some(ForkVersionedResponse { + version: helper.version, + metadata, + data, + })) + } + /// Perform a HTTP GET request using an 'accept' header, returning `None` on a 404 error. pub async fn get_bytes_opt_accept_header( &self, @@ -850,7 +896,7 @@ impl BeaconNodeHttpClient { &self, start_period: u64, count: u64, - ) -> Result>>>, Error> { + ) -> Result>>>, Error> { let mut path = self.eth_path(V1)?; path.path_segments_mut() @@ -865,7 +911,14 @@ impl BeaconNodeHttpClient { path.query_pairs_mut() .append_pair("count", &count.to_string()); - self.get_opt(path).await + self.get_opt(path).await.map(|opt| { + opt.map(|updates: Vec<_>| { + updates + .into_iter() + .map(BeaconResponse::ForkVersioned) + .collect() + }) + }) } /// `GET beacon/light_client/bootstrap` @@ -874,7 +927,7 @@ impl BeaconNodeHttpClient { pub async fn get_light_client_bootstrap( &self, block_root: Hash256, - ) -> Result>>, Error> { + ) -> Result>>, Error> { let mut path = self.eth_path(V1)?; path.path_segments_mut() @@ -884,7 +937,9 @@ impl BeaconNodeHttpClient { .push("bootstrap") .push(&format!("{:?}", block_root)); - self.get_opt(path).await + self.get_opt(path) + .await + .map(|opt| opt.map(BeaconResponse::ForkVersioned)) } /// `GET beacon/light_client/optimistic_update` @@ -892,7 +947,7 @@ impl BeaconNodeHttpClient { /// Returns `Ok(None)` on a 404 error. pub async fn get_beacon_light_client_optimistic_update( &self, - ) -> Result>>, Error> { + ) -> Result>>, Error> { let mut path = self.eth_path(V1)?; path.path_segments_mut() @@ -901,7 +956,9 @@ impl BeaconNodeHttpClient { .push("light_client") .push("optimistic_update"); - self.get_opt(path).await + self.get_opt(path) + .await + .map(|opt| opt.map(BeaconResponse::ForkVersioned)) } /// `GET beacon/light_client/finality_update` @@ -909,7 +966,7 @@ impl BeaconNodeHttpClient { /// Returns `Ok(None)` on a 404 error. pub async fn get_beacon_light_client_finality_update( &self, - ) -> Result>>, Error> { + ) -> Result>>, Error> { let mut path = self.eth_path(V1)?; path.path_segments_mut() @@ -918,7 +975,9 @@ impl BeaconNodeHttpClient { .push("light_client") .push("finality_update"); - self.get_opt(path).await + self.get_opt(path) + .await + .map(|opt| opt.map(BeaconResponse::ForkVersioned)) } /// `GET beacon/headers?slot,parent_root` @@ -981,8 +1040,14 @@ impl BeaconNodeHttpClient { .push("beacon") .push("blocks"); - self.post_with_timeout(path, block_contents, self.timeouts.proposal) - .await?; + let fork_name = block_contents.signed_block().fork_name_unchecked(); + self.post_generic_with_consensus_version( + path, + block_contents, + Some(self.timeouts.proposal), + fork_name, + ) + .await?; Ok(()) } @@ -1200,16 +1265,12 @@ impl BeaconNodeHttpClient { pub async fn get_beacon_blocks( &self, block_id: BlockId, - ) -> Result< - Option>>, - Error, - > { + ) -> Result>>, Error> + { let path = self.get_beacon_blocks_path(block_id)?; - let Some(response) = self.get_response(path, |b| b).await.optional()? else { - return Ok(None); - }; - - Ok(Some(response.json().await?)) + self.get_opt(path) + .await + .map(|opt| opt.map(BeaconResponse::ForkVersioned)) } /// `GET v1/beacon/blob_sidecars/{block_id}` @@ -1219,8 +1280,8 @@ impl BeaconNodeHttpClient { &self, block_id: BlockId, indices: Option<&[u64]>, - ) -> Result>>, Error> - { + spec: &ChainSpec, + ) -> Result>>, Error> { let mut path = self.get_blobs_path(block_id)?; if let Some(indices) = indices { let indices_string = indices @@ -1232,11 +1293,11 @@ impl BeaconNodeHttpClient { .append_pair("indices", &indices_string); } - let Some(response) = self.get_response(path, |b| b).await.optional()? else { - return Ok(None); - }; - - Ok(Some(response.json().await?)) + self.get_fork_contextual(path, |fork| { + (fork, spec.max_blobs_per_block_by_fork(fork) as usize) + }) + .await + .map(|opt| opt.map(BeaconResponse::ForkVersioned)) } /// `GET v1/beacon/blinded_blocks/{block_id}` @@ -1246,15 +1307,13 @@ impl BeaconNodeHttpClient { &self, block_id: BlockId, ) -> Result< - Option>>, + Option>>, Error, > { let path = self.get_beacon_blinded_blocks_path(block_id)?; - let Some(response) = self.get_response(path, |b| b).await.optional()? else { - return Ok(None); - }; - - Ok(Some(response.json().await?)) + self.get_opt(path) + .await + .map(|opt| opt.map(BeaconResponse::ForkVersioned)) } /// `GET v1/beacon/blocks` (LEGACY) @@ -1263,7 +1322,7 @@ impl BeaconNodeHttpClient { pub async fn get_beacon_blocks_v1( &self, block_id: BlockId, - ) -> Result>>, Error> { + ) -> Result>>, Error> { let mut path = self.eth_path(V1)?; path.path_segments_mut() @@ -1272,7 +1331,9 @@ impl BeaconNodeHttpClient { .push("blocks") .push(&block_id.to_string()); - self.get_opt(path).await + self.get_opt(path) + .await + .map(|opt| opt.map(BeaconResponse::Unversioned)) } /// `GET beacon/blocks` as SSZ @@ -1353,7 +1414,7 @@ impl BeaconNodeHttpClient { pub async fn get_beacon_blocks_attestations_v2( &self, block_id: BlockId, - ) -> Result>>>, Error> + ) -> Result>>>, Error> { let mut path = self.eth_path(V2)?; @@ -1364,7 +1425,9 @@ impl BeaconNodeHttpClient { .push(&block_id.to_string()) .push("attestations"); - self.get_opt(path).await + self.get_opt(path) + .await + .map(|opt| opt.map(BeaconResponse::ForkVersioned)) } /// `POST v1/beacon/pool/attestations` @@ -1456,7 +1519,7 @@ impl BeaconNodeHttpClient { &self, slot: Option, committee_index: Option, - ) -> Result>>, Error> { + ) -> Result>>, Error> { let mut path = self.eth_path(V2)?; path.path_segments_mut() @@ -1475,7 +1538,7 @@ impl BeaconNodeHttpClient { .append_pair("committee_index", &index.to_string()); } - self.get(path).await + self.get(path).await.map(BeaconResponse::ForkVersioned) } /// `POST v1/beacon/pool/attester_slashings` @@ -1534,7 +1597,7 @@ impl BeaconNodeHttpClient { /// `GET v2/beacon/pool/attester_slashings` pub async fn get_beacon_pool_attester_slashings_v2( &self, - ) -> Result>>, Error> { + ) -> Result>>, Error> { let mut path = self.eth_path(V2)?; path.path_segments_mut() @@ -1543,7 +1606,7 @@ impl BeaconNodeHttpClient { .push("pool") .push("attester_slashings"); - self.get(path).await + self.get(path).await.map(BeaconResponse::ForkVersioned) } /// `POST beacon/pool/proposer_slashings` @@ -1964,10 +2027,11 @@ impl BeaconNodeHttpClient { pub async fn get_debug_beacon_states( &self, state_id: StateId, - ) -> Result>>, Error> - { + ) -> Result>>, Error> { let path = self.get_debug_beacon_states_path(state_id)?; - self.get_opt(path).await + self.get_opt(path) + .await + .map(|opt| opt.map(BeaconResponse::ForkVersioned)) } /// `GET debug/beacon/states/{state_id}` @@ -2051,7 +2115,7 @@ impl BeaconNodeHttpClient { slot: Slot, randao_reveal: &SignatureBytes, graffiti: Option<&Graffiti>, - ) -> Result>, Error> { + ) -> Result>, Error> { self.get_validator_blocks_modular(slot, randao_reveal, graffiti, SkipRandaoVerification::No) .await } @@ -2063,12 +2127,12 @@ impl BeaconNodeHttpClient { randao_reveal: &SignatureBytes, graffiti: Option<&Graffiti>, skip_randao_verification: SkipRandaoVerification, - ) -> Result>, Error> { + ) -> Result>, Error> { let path = self .get_validator_blocks_path::(slot, randao_reveal, graffiti, skip_randao_verification) .await?; - self.get(path).await + self.get(path).await.map(BeaconResponse::ForkVersioned) } /// returns `GET v2/validator/blocks/{slot}` URL path @@ -2324,7 +2388,7 @@ impl BeaconNodeHttpClient { slot: Slot, randao_reveal: &SignatureBytes, graffiti: Option<&Graffiti>, - ) -> Result>, Error> { + ) -> Result>, Error> { self.get_validator_blinded_blocks_modular( slot, randao_reveal, @@ -2373,7 +2437,7 @@ impl BeaconNodeHttpClient { randao_reveal: &SignatureBytes, graffiti: Option<&Graffiti>, skip_randao_verification: SkipRandaoVerification, - ) -> Result>, Error> { + ) -> Result>, Error> { let path = self .get_validator_blinded_blocks_path::( slot, @@ -2383,7 +2447,7 @@ impl BeaconNodeHttpClient { ) .await?; - self.get(path).await + self.get(path).await.map(BeaconResponse::ForkVersioned) } /// `GET v2/validator/blinded_blocks/{slot}` in ssz format @@ -2472,7 +2536,7 @@ impl BeaconNodeHttpClient { slot: Slot, attestation_data_root: Hash256, committee_index: CommitteeIndex, - ) -> Result>>, Error> { + ) -> Result>>, Error> { let mut path = self.eth_path(V2)?; path.path_segments_mut() @@ -2490,6 +2554,7 @@ impl BeaconNodeHttpClient { self.get_opt_with_timeout(path, self.timeouts.attestation) .await + .map(|opt| opt.map(BeaconResponse::ForkVersioned)) } /// `GET validator/sync_committee_contribution` diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 0f9fa7f2c2d..f895e5cb8fa 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -10,7 +10,6 @@ use mediatype::{names, MediaType, MediaTypeList}; use multiaddr::Multiaddr; use reqwest::header::HeaderMap; use serde::{Deserialize, Deserializer, Serialize}; -use serde_json::Value; use serde_utils::quoted_u64::Quoted; use ssz::{Decode, DecodeError}; use ssz_derive::{Decode, Encode}; @@ -1050,51 +1049,56 @@ pub struct SseExtendedPayloadAttributesGeneric { pub type SseExtendedPayloadAttributes = SseExtendedPayloadAttributesGeneric; pub type VersionedSsePayloadAttributes = ForkVersionedResponse; -impl ForkVersionDeserialize for SsePayloadAttributes { - fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( - value: serde_json::value::Value, - fork_name: ForkName, - ) -> Result { - match fork_name { - ForkName::Bellatrix => serde_json::from_value(value) - .map(Self::V1) - .map_err(serde::de::Error::custom), - ForkName::Capella => serde_json::from_value(value) - .map(Self::V2) - .map_err(serde::de::Error::custom), - ForkName::Deneb => serde_json::from_value(value) - .map(Self::V3) - .map_err(serde::de::Error::custom), - ForkName::Electra => serde_json::from_value(value) - .map(Self::V3) - .map_err(serde::de::Error::custom), - ForkName::Fulu => serde_json::from_value(value) - .map(Self::V3) - .map_err(serde::de::Error::custom), - ForkName::Base | ForkName::Altair => Err(serde::de::Error::custom(format!( - "SsePayloadAttributes deserialization for {fork_name} not implemented" - ))), - } +impl<'de> ContextDeserialize<'de, ForkName> for SsePayloadAttributes { + fn context_deserialize(deserializer: D, context: ForkName) -> Result + where + D: Deserializer<'de>, + { + let convert_err = |e| { + serde::de::Error::custom(format!( + "SsePayloadAttributes failed to deserialize: {:?}", + e + )) + }; + Ok(match context { + ForkName::Base | ForkName::Altair => { + return Err(serde::de::Error::custom(format!( + "SsePayloadAttributes failed to deserialize: unsupported fork '{}'", + context + ))) + } + ForkName::Bellatrix => { + Self::V1(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Capella => { + Self::V2(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Deneb | ForkName::Electra | ForkName::Fulu => { + Self::V3(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + }) } } -impl ForkVersionDeserialize for SseExtendedPayloadAttributes { - fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( - value: serde_json::value::Value, - fork_name: ForkName, - ) -> Result { - let helper: SseExtendedPayloadAttributesGeneric = - serde_json::from_value(value).map_err(serde::de::Error::custom)?; +impl<'de> ContextDeserialize<'de, ForkName> for SseExtendedPayloadAttributes { + fn context_deserialize(deserializer: D, context: ForkName) -> Result + where + D: Deserializer<'de>, + { + let helper = + SseExtendedPayloadAttributesGeneric::::deserialize(deserializer)?; + Ok(Self { proposal_slot: helper.proposal_slot, proposer_index: helper.proposer_index, parent_block_root: helper.parent_block_root, parent_block_number: helper.parent_block_number, parent_block_hash: helper.parent_block_hash, - payload_attributes: SsePayloadAttributes::deserialize_by_fork::( + payload_attributes: SsePayloadAttributes::context_deserialize( helper.payload_attributes, - fork_name, - )?, + context, + ) + .map_err(serde::de::Error::custom)?, }) } } @@ -1112,8 +1116,8 @@ pub enum EventKind { ChainReorg(SseChainReorg), ContributionAndProof(Box>), LateHead(SseLateHead), - LightClientFinalityUpdate(Box>), - LightClientOptimisticUpdate(Box>), + LightClientFinalityUpdate(Box>>), + LightClientOptimisticUpdate(Box>>), #[cfg(feature = "lighthouse")] BlockReward(BlockReward), PayloadAttributes(VersionedSsePayloadAttributes), @@ -1193,22 +1197,24 @@ impl EventKind { ServerError::InvalidServerSentEvent(format!("Payload Attributes: {:?}", e)) })?, )), - "light_client_finality_update" => Ok(EventKind::LightClientFinalityUpdate( - serde_json::from_str(data).map_err(|e| { + "light_client_finality_update" => Ok(EventKind::LightClientFinalityUpdate(Box::new( + BeaconResponse::ForkVersioned(serde_json::from_str(data).map_err(|e| { ServerError::InvalidServerSentEvent(format!( "Light Client Finality Update: {:?}", e )) - })?, - )), - "light_client_optimistic_update" => Ok(EventKind::LightClientOptimisticUpdate( - serde_json::from_str(data).map_err(|e| { - ServerError::InvalidServerSentEvent(format!( - "Light Client Optimistic Update: {:?}", - e - )) - })?, - )), + })?), + ))), + "light_client_optimistic_update" => { + Ok(EventKind::LightClientOptimisticUpdate(Box::new( + BeaconResponse::ForkVersioned(serde_json::from_str(data).map_err(|e| { + ServerError::InvalidServerSentEvent(format!( + "Light Client Optimistic Update: {:?}", + e + )) + })?), + ))) + } #[cfg(feature = "lighthouse")] "block_reward" => Ok(EventKind::BlockReward(serde_json::from_str(data).map_err( |e| ServerError::InvalidServerSentEvent(format!("Block Reward: {:?}", e)), @@ -1597,7 +1603,7 @@ mod tests { } } -#[derive(Debug, Encode, Serialize, Deserialize)] +#[derive(Debug, Encode, Serialize)] #[serde(untagged)] #[serde(bound = "E: EthSpec")] #[ssz(enum_behaviour = "transparent")] @@ -1610,7 +1616,7 @@ pub type JsonProduceBlockV3Response = ForkVersionedResponse, ProduceBlockV3Metadata>; /// A wrapper over a [`BeaconBlock`] or a [`BlockContents`]. -#[derive(Debug, Encode, Serialize, Deserialize)] +#[derive(Debug, Encode, Serialize)] #[serde(untagged)] #[serde(bound = "E: EthSpec")] #[ssz(enum_behaviour = "transparent")] @@ -1724,18 +1730,18 @@ impl FullBlockContents { } } -impl ForkVersionDeserialize for FullBlockContents { - fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( - value: serde_json::value::Value, - fork_name: ForkName, - ) -> Result { - if fork_name.deneb_enabled() { +impl<'de, E: EthSpec> ContextDeserialize<'de, ForkName> for FullBlockContents { + fn context_deserialize(deserializer: D, context: ForkName) -> Result + where + D: Deserializer<'de>, + { + if context.deneb_enabled() { Ok(FullBlockContents::BlockContents( - BlockContents::deserialize_by_fork::<'de, D>(value, fork_name)?, + BlockContents::context_deserialize::(deserializer, context)?, )) } else { Ok(FullBlockContents::Block( - BeaconBlock::deserialize_by_fork::<'de, D>(value, fork_name)?, + BeaconBlock::context_deserialize::(deserializer, context)?, )) } } @@ -1802,7 +1808,7 @@ impl TryFrom<&HeaderMap> for ProduceBlockV3Metadata { } /// A wrapper over a [`SignedBeaconBlock`] or a [`SignedBlockContents`]. -#[derive(Clone, Debug, Encode, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Encode, Serialize)] #[serde(untagged)] #[serde(bound = "E: EthSpec")] #[ssz(enum_behaviour = "transparent")] @@ -1811,6 +1817,26 @@ pub enum PublishBlockRequest { Block(Arc>), } +impl<'de, E: EthSpec> ContextDeserialize<'de, ForkName> for PublishBlockRequest { + fn context_deserialize(deserializer: D, context: ForkName) -> Result + where + D: Deserializer<'de>, + { + let value = + serde_json::Value::deserialize(deserializer).map_err(serde::de::Error::custom)?; + + SignedBlockContents::::context_deserialize(&value, context) + .map(PublishBlockRequest::BlockContents) + .or_else(|_| { + Arc::>::context_deserialize(&value, context) + .map(PublishBlockRequest::Block) + }) + .map_err(|_| { + serde::de::Error::custom("could not match any variant of PublishBlockRequest") + }) + } +} + impl PublishBlockRequest { pub fn new( block: Arc>, @@ -1887,7 +1913,7 @@ impl From> for PublishBlockRequest { } } -#[derive(Debug, Clone, Serialize, Deserialize, Encode)] +#[derive(Debug, Clone, PartialEq, Serialize, Encode)] #[serde(bound = "E: EthSpec")] pub struct SignedBlockContents { pub signed_block: Arc>, @@ -1896,7 +1922,33 @@ pub struct SignedBlockContents { pub blobs: BlobsList, } -#[derive(Debug, Clone, Serialize, Deserialize, Encode)] +impl<'de, E: EthSpec> ContextDeserialize<'de, ForkName> for SignedBlockContents { + fn context_deserialize(deserializer: D, context: ForkName) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + #[serde(bound = "E: EthSpec")] + struct Helper { + signed_block: serde_json::Value, + kzg_proofs: KzgProofs, + #[serde(with = "ssz_types::serde_utils::list_of_hex_fixed_vec")] + blobs: BlobsList, + } + let helper = Helper::::deserialize(deserializer).map_err(serde::de::Error::custom)?; + + let block = SignedBeaconBlock::context_deserialize(helper.signed_block, context) + .map_err(serde::de::Error::custom)?; + + Ok(Self { + signed_block: Arc::new(block), + kzg_proofs: helper.kzg_proofs, + blobs: helper.blobs, + }) + } +} + +#[derive(Debug, Clone, Serialize, Encode)] #[serde(bound = "E: EthSpec")] pub struct BlockContents { pub block: BeaconBlock, @@ -1905,11 +1957,11 @@ pub struct BlockContents { pub blobs: BlobsList, } -impl ForkVersionDeserialize for BlockContents { - fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( - value: serde_json::value::Value, - fork_name: ForkName, - ) -> Result { +impl<'de, E: EthSpec> ContextDeserialize<'de, ForkName> for BlockContents { + fn context_deserialize(deserializer: D, context: ForkName) -> Result + where + D: Deserializer<'de>, + { #[derive(Deserialize)] #[serde(bound = "E: EthSpec")] struct Helper { @@ -1918,10 +1970,13 @@ impl ForkVersionDeserialize for BlockContents { #[serde(with = "ssz_types::serde_utils::list_of_hex_fixed_vec")] blobs: BlobsList, } - let helper: Helper = serde_json::from_value(value).map_err(serde::de::Error::custom)?; + let helper = Helper::::deserialize(deserializer).map_err(serde::de::Error::custom)?; + + let block = BeaconBlock::context_deserialize(helper.block, context) + .map_err(serde::de::Error::custom)?; Ok(Self { - block: BeaconBlock::deserialize_by_fork::<'de, D>(helper.block, fork_name)?, + block, kzg_proofs: helper.kzg_proofs, blobs: helper.blobs, }) @@ -1993,22 +2048,22 @@ impl FullPayloadContents { } } -impl ForkVersionDeserialize for FullPayloadContents { - fn deserialize_by_fork<'de, D: Deserializer<'de>>( - value: Value, - fork_name: ForkName, - ) -> Result { - if fork_name.deneb_enabled() { - ExecutionPayloadAndBlobs::deserialize_by_fork::<'de, D>(value, fork_name) +impl<'de, E: EthSpec> ContextDeserialize<'de, ForkName> for FullPayloadContents { + fn context_deserialize(deserializer: D, context: ForkName) -> Result + where + D: Deserializer<'de>, + { + if context.deneb_enabled() { + ExecutionPayloadAndBlobs::context_deserialize::(deserializer, context) .map(Self::PayloadAndBlobs) .map_err(serde::de::Error::custom) - } else if fork_name.bellatrix_enabled() { - ExecutionPayload::deserialize_by_fork::<'de, D>(value, fork_name) + } else if context.bellatrix_enabled() { + ExecutionPayload::context_deserialize::(deserializer, context) .map(Self::Payload) .map_err(serde::de::Error::custom) } else { Err(serde::de::Error::custom(format!( - "FullPayloadContents deserialization for {fork_name} not implemented" + "FullPayloadContents deserialization for {context} not implemented" ))) } } @@ -2021,23 +2076,25 @@ pub struct ExecutionPayloadAndBlobs { pub blobs_bundle: BlobsBundle, } -impl ForkVersionDeserialize for ExecutionPayloadAndBlobs { - fn deserialize_by_fork<'de, D: Deserializer<'de>>( - value: Value, - fork_name: ForkName, - ) -> Result { +impl<'de, E: EthSpec> ContextDeserialize<'de, ForkName> for ExecutionPayloadAndBlobs { + fn context_deserialize(deserializer: D, context: ForkName) -> Result + where + D: Deserializer<'de>, + { #[derive(Deserialize)] #[serde(bound = "E: EthSpec")] struct Helper { execution_payload: serde_json::Value, blobs_bundle: BlobsBundle, } - let helper: Helper = serde_json::from_value(value).map_err(serde::de::Error::custom)?; + let helper = Helper::::deserialize(deserializer).map_err(serde::de::Error::custom)?; + Ok(Self { - execution_payload: ExecutionPayload::deserialize_by_fork::<'de, D>( + execution_payload: ExecutionPayload::context_deserialize( helper.execution_payload, - fork_name, - )?, + context, + ) + .map_err(serde::de::Error::custom)?, blobs_bundle: helper.blobs_bundle, }) } @@ -2186,6 +2243,73 @@ mod test { assert_eq!(serde_json::to_string(&y).unwrap(), pubkey_str); } + #[test] + fn test_publish_block_request_context_deserialize() { + let round_trip_test = |request: PublishBlockRequest| { + let fork_name = request.signed_block().fork_name_unchecked(); + let json_str = serde_json::to_string(&request).unwrap(); + let mut de = serde_json::Deserializer::from_str(&json_str); + let deserialized_request = + PublishBlockRequest::::context_deserialize(&mut de, fork_name) + .unwrap(); + assert_eq!(request, deserialized_request); + }; + + let rng = &mut XorShiftRng::from_seed([42; 16]); + for fork_name in ForkName::list_all() { + let signed_beacon_block = + map_fork_name!(fork_name, SignedBeaconBlock, <_>::random_for_test(rng)); + let request = if fork_name.deneb_enabled() { + let kzg_proofs = KzgProofs::::random_for_test(rng); + let blobs = BlobsList::::random_for_test(rng); + let block_contents = SignedBlockContents { + signed_block: Arc::new(signed_beacon_block), + kzg_proofs, + blobs, + }; + PublishBlockRequest::BlockContents(block_contents) + } else { + PublishBlockRequest::Block(Arc::new(signed_beacon_block)) + }; + round_trip_test(request); + println!("fork_name: {:?} PASSED", fork_name); + } + } + + #[test] + fn test_signed_block_contents_context_deserialize() { + let round_trip_test = |contents: SignedBlockContents| { + let fork_name = contents.signed_block.fork_name_unchecked(); + let json_str = serde_json::to_string(&contents).unwrap(); + let mut de = serde_json::Deserializer::from_str(&json_str); + let deserialized_contents = + SignedBlockContents::::context_deserialize(&mut de, fork_name) + .unwrap(); + assert_eq!(contents, deserialized_contents); + }; + + let mut fork_name = ForkName::Deneb; + let rng = &mut XorShiftRng::from_seed([42; 16]); + loop { + let signed_beacon_block = + map_fork_name!(fork_name, SignedBeaconBlock, <_>::random_for_test(rng)); + let kzg_proofs = KzgProofs::::random_for_test(rng); + let blobs = BlobsList::::random_for_test(rng); + let block_contents = SignedBlockContents { + signed_block: Arc::new(signed_beacon_block), + kzg_proofs, + blobs, + }; + round_trip_test(block_contents); + println!("fork_name: {:?} PASSED", fork_name); + if let Some(next_fork_name) = fork_name.next_fork() { + fork_name = next_fork_name; + } else { + break; + } + } + } + #[test] fn test_execution_payload_execution_payload_deserialize_by_fork() { let rng = &mut XorShiftRng::from_seed([42; 16]); @@ -2278,14 +2402,13 @@ mod test { fn generic_deserialize_by_fork< 'de, D: Deserializer<'de>, - O: ForkVersionDeserialize + PartialEq + Debug, + O: ContextDeserialize<'de, ForkName> + PartialEq + Debug, >( deserializer: D, original: O, fork_name: ForkName, ) { - let val = Value::deserialize(deserializer).unwrap(); - let roundtrip = O::deserialize_by_fork::<'de, D>(val, fork_name).unwrap(); + let roundtrip = O::context_deserialize::(deserializer, fork_name).unwrap(); assert_eq!(original, roundtrip); } } diff --git a/consensus/context_deserialize/Cargo.toml b/consensus/context_deserialize/Cargo.toml new file mode 100644 index 00000000000..30dae761362 --- /dev/null +++ b/consensus/context_deserialize/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "context_deserialize" +version = "0.1.0" +edition = "2021" + +[dependencies] +milhouse = { workspace = true } +serde = { workspace = true } +ssz_types = { workspace = true } diff --git a/consensus/context_deserialize/src/impls.rs b/consensus/context_deserialize/src/impls.rs new file mode 100644 index 00000000000..803619365f1 --- /dev/null +++ b/consensus/context_deserialize/src/impls.rs @@ -0,0 +1,103 @@ +use crate::ContextDeserialize; +use serde::de::{Deserialize, DeserializeSeed, Deserializer, SeqAccess, Visitor}; +use std::marker::PhantomData; +use std::sync::Arc; + +impl<'de, C, T> ContextDeserialize<'de, T> for Arc +where + C: ContextDeserialize<'de, T>, +{ + fn context_deserialize(deserializer: D, context: T) -> Result + where + D: Deserializer<'de>, + { + Ok(Arc::new(C::context_deserialize(deserializer, context)?)) + } +} + +impl<'de, T, C> ContextDeserialize<'de, C> for Vec +where + T: ContextDeserialize<'de, C>, + C: Clone, +{ + fn context_deserialize(deserializer: D, context: C) -> Result + where + D: Deserializer<'de>, + { + // Our Visitor, which owns one copy of the context T + struct ContextVisitor { + context: T, + _marker: PhantomData, + } + + impl<'de, C, T> Visitor<'de> for ContextVisitor + where + C: ContextDeserialize<'de, T>, + T: Clone, + { + type Value = Vec; + + fn expecting(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { + fmt.write_str("a sequence of context‐deserialized elements") + } + + fn visit_seq(self, mut seq: A) -> Result, A::Error> + where + A: SeqAccess<'de>, + { + let mut out = Vec::with_capacity(seq.size_hint().unwrap_or(0)); + // for each element, we clone the context and hand it to the seed + while let Some(elem) = seq.next_element_seed(ContextSeed { + context: self.context.clone(), + _marker: PhantomData, + })? { + out.push(elem); + } + Ok(out) + } + } + + // A little seed that hands the deserializer + context into C::context_deserialize + struct ContextSeed { + context: C, + _marker: PhantomData, + } + + impl<'de, T, C> DeserializeSeed<'de> for ContextSeed + where + T: ContextDeserialize<'de, C>, + C: Clone, + { + type Value = T; + + fn deserialize(self, deserializer: D) -> Result + where + D: Deserializer<'de>, + { + T::context_deserialize(deserializer, self.context) + } + } + + deserializer.deserialize_seq(ContextVisitor { + context, + _marker: PhantomData, + }) + } +} + +macro_rules! trivial_deserialize { + ($($t:ty),* $(,)?) => { + $( + impl<'de, T> ContextDeserialize<'de, T> for $t { + fn context_deserialize(deserializer: D, _context: T) -> Result + where + D: Deserializer<'de>, + { + <$t>::deserialize(deserializer) + } + } + )* + }; +} + +trivial_deserialize!(bool, u8, u16, u32, u64, u128, i8, i16, i32, i64, i128, f32, f64); diff --git a/consensus/context_deserialize/src/lib.rs b/consensus/context_deserialize/src/lib.rs new file mode 100644 index 00000000000..9de819247bd --- /dev/null +++ b/consensus/context_deserialize/src/lib.rs @@ -0,0 +1,13 @@ +pub mod impls; +pub mod milhouse; +pub mod ssz_impls; + +extern crate serde; +use serde::de::Deserializer; + +/// General-purpose deserialization trait that accepts extra context `C`. +pub trait ContextDeserialize<'de, C>: Sized { + fn context_deserialize(deserializer: D, context: C) -> Result + where + D: Deserializer<'de>; +} diff --git a/consensus/context_deserialize/src/milhouse.rs b/consensus/context_deserialize/src/milhouse.rs new file mode 100644 index 00000000000..3b86f067a3e --- /dev/null +++ b/consensus/context_deserialize/src/milhouse.rs @@ -0,0 +1,45 @@ +use crate::ContextDeserialize; +use milhouse::{List, Value, Vector}; +use serde::de::Deserializer; +use ssz_types::typenum::Unsigned; + +impl<'de, C, T, N> ContextDeserialize<'de, C> for List +where + T: ContextDeserialize<'de, C> + Value, + N: Unsigned, + C: Clone, +{ + fn context_deserialize(deserializer: D, context: C) -> Result + where + D: Deserializer<'de>, + { + // First deserialize as a Vec. + // This is not the most efficient implementation as it allocates a temporary Vec. In future + // we could write a more performant implementation using `List::builder()`. + let vec = Vec::::context_deserialize(deserializer, context)?; + + // Then convert to List, which will check the length. + List::new(vec) + .map_err(|e| serde::de::Error::custom(format!("Failed to create List: {:?}", e))) + } +} + +impl<'de, C, T, N> ContextDeserialize<'de, C> for Vector +where + T: ContextDeserialize<'de, C> + Value, + N: Unsigned, + C: Clone, +{ + fn context_deserialize(deserializer: D, context: C) -> Result + where + D: Deserializer<'de>, + { + // First deserialize as a List + let list = List::::context_deserialize(deserializer, context)?; + + // Then convert to Vector, which will check the length + Vector::try_from(list).map_err(|e| { + serde::de::Error::custom(format!("Failed to convert List to Vector: {:?}", e)) + }) + } +} diff --git a/consensus/context_deserialize/src/ssz_impls.rs b/consensus/context_deserialize/src/ssz_impls.rs new file mode 100644 index 00000000000..e989d67b295 --- /dev/null +++ b/consensus/context_deserialize/src/ssz_impls.rs @@ -0,0 +1,48 @@ +use crate::serde::de::Error; +use crate::ContextDeserialize; +use serde::de::Deserializer; +use serde::Deserialize; +use ssz_types::length::{Fixed, Variable}; +use ssz_types::typenum::Unsigned; +use ssz_types::{Bitfield, FixedVector}; + +impl<'de, C, T, N> ContextDeserialize<'de, C> for FixedVector +where + T: ContextDeserialize<'de, C>, + N: Unsigned, + C: Clone, +{ + fn context_deserialize(deserializer: D, context: C) -> Result + where + D: Deserializer<'de>, + { + let vec = Vec::::context_deserialize(deserializer, context)?; + FixedVector::new(vec).map_err(|e| D::Error::custom(format!("{:?}", e))) + } +} + +impl<'de, C, N> ContextDeserialize<'de, C> for Bitfield> +where + N: Unsigned + Clone, +{ + fn context_deserialize(deserializer: D, _context: C) -> Result + where + D: Deserializer<'de>, + { + Bitfield::>::deserialize(deserializer) + .map_err(|e| D::Error::custom(format!("{:?}", e))) + } +} + +impl<'de, C, N> ContextDeserialize<'de, C> for Bitfield> +where + N: Unsigned + Clone, +{ + fn context_deserialize(deserializer: D, _context: C) -> Result + where + D: Deserializer<'de>, + { + Bitfield::>::deserialize(deserializer) + .map_err(|e| D::Error::custom(format!("{:?}", e))) + } +} diff --git a/consensus/context_deserialize_derive/Cargo.toml b/consensus/context_deserialize_derive/Cargo.toml new file mode 100644 index 00000000000..eedae30cdfe --- /dev/null +++ b/consensus/context_deserialize_derive/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "context_deserialize_derive" +version = "0.1.0" +edition = "2021" + +[lib] +proc-macro = true + +[dependencies] +quote = { workspace = true } +syn = { workspace = true } + +[dev-dependencies] +context_deserialize = { path = "../context_deserialize" } +serde = { workspace = true } +serde_json = "1.0" diff --git a/consensus/context_deserialize_derive/src/lib.rs b/consensus/context_deserialize_derive/src/lib.rs new file mode 100644 index 00000000000..0b73a43b0a4 --- /dev/null +++ b/consensus/context_deserialize_derive/src/lib.rs @@ -0,0 +1,118 @@ +extern crate proc_macro; +extern crate quote; +extern crate syn; + +use proc_macro::TokenStream; +use quote::quote; +use syn::{ + parse_macro_input, AttributeArgs, DeriveInput, GenericParam, LifetimeDef, Meta, NestedMeta, + WhereClause, +}; + +#[proc_macro_attribute] +pub fn context_deserialize(attr: TokenStream, item: TokenStream) -> TokenStream { + let args = parse_macro_input!(attr as AttributeArgs); + let input = parse_macro_input!(item as DeriveInput); + let ident = &input.ident; + + let mut ctx_types = Vec::new(); + let mut explicit_where: Option = None; + + for meta in args { + match meta { + NestedMeta::Meta(Meta::Path(p)) => { + ctx_types.push(p); + } + NestedMeta::Meta(Meta::NameValue(nv)) if nv.path.is_ident("bound") => { + if let syn::Lit::Str(lit_str) = &nv.lit { + let where_string = format!("where {}", lit_str.value()); + match syn::parse_str::(&where_string) { + Ok(where_clause) => { + explicit_where = Some(where_clause); + } + Err(err) => { + return syn::Error::new_spanned( + lit_str, + format!("Invalid where clause '{}': {}", lit_str.value(), err), + ) + .to_compile_error() + .into(); + } + } + } else { + return syn::Error::new_spanned( + &nv, + "Expected a string literal for `bound` value", + ) + .to_compile_error() + .into(); + } + } + _ => { + return syn::Error::new_spanned( + &meta, + "Expected paths or `bound = \"...\"` in #[context_deserialize(...)]", + ) + .to_compile_error() + .into(); + } + } + } + + if ctx_types.is_empty() { + return quote! { + compile_error!("Usage: #[context_deserialize(Type1, Type2, ..., bound = \"...\")]"); + } + .into(); + } + + let original_generics = input.generics.clone(); + + // Clone and clean generics for impl use (remove default params) + let mut impl_generics = input.generics.clone(); + for param in impl_generics.params.iter_mut() { + if let GenericParam::Type(ty) = param { + ty.eq_token = None; + ty.default = None; + } + } + + // Ensure 'de lifetime exists in impl generics + let has_de = impl_generics + .lifetimes() + .any(|LifetimeDef { lifetime, .. }| lifetime.ident == "de"); + + if !has_de { + impl_generics.params.insert(0, syn::parse_quote! { 'de }); + } + + let (_, ty_generics, _) = original_generics.split_for_impl(); + let (impl_gens, _, _) = impl_generics.split_for_impl(); + + // Generate: no `'de` applied to the type name + let mut impls = quote! {}; + for ctx in ctx_types { + impls.extend(quote! { + impl #impl_gens context_deserialize::ContextDeserialize<'de, #ctx> + for #ident #ty_generics + #explicit_where + { + fn context_deserialize( + deserializer: D, + _context: #ctx, + ) -> Result + where + D: serde::de::Deserializer<'de>, + { + ::deserialize(deserializer) + } + } + }); + } + + quote! { + #input + #impls + } + .into() +} diff --git a/consensus/context_deserialize_derive/tests/context_deserialize_derive.rs b/consensus/context_deserialize_derive/tests/context_deserialize_derive.rs new file mode 100644 index 00000000000..d6883400e0b --- /dev/null +++ b/consensus/context_deserialize_derive/tests/context_deserialize_derive.rs @@ -0,0 +1,94 @@ +use context_deserialize::ContextDeserialize; +use context_deserialize_derive::context_deserialize; +use serde::{Deserialize, Serialize}; + +#[test] +fn test_context_deserialize_derive() { + type TestContext = (); + + #[context_deserialize(TestContext)] + #[derive(Debug, PartialEq, Serialize, Deserialize)] + struct Test { + field: String, + } + + let test = Test { + field: "test".to_string(), + }; + let serialized = serde_json::to_string(&test).unwrap(); + let deserialized = + Test::context_deserialize(&mut serde_json::Deserializer::from_str(&serialized), ()) + .unwrap(); + assert_eq!(test, deserialized); +} + +#[test] +fn test_context_deserialize_derive_multiple_types() { + #[allow(dead_code)] + struct TestContext1(u64); + #[allow(dead_code)] + struct TestContext2(String); + + // This will derive: + // - ContextDeserialize for Test + // - ContextDeserialize for Test + // by just leveraging the Deserialize impl + #[context_deserialize(TestContext1, TestContext2)] + #[derive(Debug, PartialEq, Serialize, Deserialize)] + struct Test { + field: String, + } + + let test = Test { + field: "test".to_string(), + }; + let serialized = serde_json::to_string(&test).unwrap(); + let deserialized = Test::context_deserialize( + &mut serde_json::Deserializer::from_str(&serialized), + TestContext1(1), + ) + .unwrap(); + assert_eq!(test, deserialized); + + let deserialized = Test::context_deserialize( + &mut serde_json::Deserializer::from_str(&serialized), + TestContext2("2".to_string()), + ) + .unwrap(); + + assert_eq!(test, deserialized); +} + +#[test] +fn test_context_deserialize_derive_bound() { + use std::fmt::Debug; + + struct TestContext; + + #[derive(Debug, PartialEq, Serialize, Deserialize)] + struct Inner { + value: u64, + } + + #[context_deserialize( + TestContext, + bound = "T: Serialize + for<'a> Deserialize<'a> + Debug + PartialEq" + )] + #[derive(Debug, PartialEq, Serialize, Deserialize)] + struct Wrapper { + inner: T, + } + + let val = Wrapper { + inner: Inner { value: 42 }, + }; + + let serialized = serde_json::to_string(&val).unwrap(); + let deserialized = Wrapper::::context_deserialize( + &mut serde_json::Deserializer::from_str(&serialized), + TestContext, + ) + .unwrap(); + + assert_eq!(val, deserialized); +} diff --git a/consensus/types/Cargo.toml b/consensus/types/Cargo.toml index 013230f1581..b58d4ef96f3 100644 --- a/consensus/types/Cargo.toml +++ b/consensus/types/Cargo.toml @@ -17,6 +17,8 @@ arbitrary = { workspace = true, features = ["derive"] } bls = { workspace = true, features = ["arbitrary"] } compare_fields = { workspace = true } compare_fields_derive = { workspace = true } +context_deserialize = { workspace = true } +context_deserialize_derive = { workspace = true } derivative = { workspace = true } eth2_interop_keypairs = { path = "../../common/eth2_interop_keypairs" } ethereum_hashing = { workspace = true } diff --git a/consensus/types/src/aggregate_and_proof.rs b/consensus/types/src/aggregate_and_proof.rs index 6edd8d38925..a280afeaae3 100644 --- a/consensus/types/src/aggregate_and_proof.rs +++ b/consensus/types/src/aggregate_and_proof.rs @@ -1,8 +1,9 @@ use super::{AttestationBase, AttestationElectra, AttestationRef}; use super::{ - ChainSpec, Domain, EthSpec, Fork, Hash256, PublicKey, SecretKey, SelectionProof, Signature, - SignedRoot, + ChainSpec, Domain, EthSpec, Fork, ForkName, Hash256, PublicKey, SecretKey, SelectionProof, + Signature, SignedRoot, }; +use crate::context_deserialize; use crate::test_utils::TestRandom; use crate::Attestation; use serde::{Deserialize, Serialize}; @@ -26,6 +27,7 @@ use tree_hash_derive::TreeHash; TestRandom, TreeHash, ), + context_deserialize(ForkName), serde(bound = "E: EthSpec"), arbitrary(bound = "E: EthSpec"), ), diff --git a/consensus/types/src/attestation.rs b/consensus/types/src/attestation.rs index e2973132b04..286e4622f84 100644 --- a/consensus/types/src/attestation.rs +++ b/consensus/types/src/attestation.rs @@ -1,8 +1,9 @@ +use crate::context_deserialize; use crate::slot_data::SlotData; use crate::{test_utils::TestRandom, Hash256, Slot}; -use crate::{Checkpoint, ForkVersionDeserialize}; +use crate::{Checkpoint, ContextDeserialize, ForkName}; use derivative::Derivative; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::BitVector; use std::collections::HashSet; @@ -47,6 +48,7 @@ impl From for Error { arbitrary::Arbitrary, TreeHash, ), + context_deserialize(ForkName), derivative(PartialEq, Hash(bound = "E: EthSpec")), serde(bound = "E: EthSpec", deny_unknown_fields), arbitrary(bound = "E: EthSpec"), @@ -532,45 +534,44 @@ impl<'a, E: EthSpec> From> for AttestationRef<'a, E> } } -impl ForkVersionDeserialize for Attestation { - fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( - value: serde_json::Value, - fork_name: crate::ForkName, - ) -> Result { - if fork_name.electra_enabled() { - let attestation: AttestationElectra = - serde_json::from_value(value).map_err(serde::de::Error::custom)?; - Ok(Attestation::Electra(attestation)) +impl<'de, E: EthSpec> ContextDeserialize<'de, ForkName> for Attestation { + fn context_deserialize(deserializer: D, context: ForkName) -> Result + where + D: Deserializer<'de>, + { + if context.electra_enabled() { + AttestationElectra::::deserialize(deserializer) + .map_err(serde::de::Error::custom) + .map(Attestation::Electra) } else { - let attestation: AttestationBase = - serde_json::from_value(value).map_err(serde::de::Error::custom)?; - Ok(Attestation::Base(attestation)) + AttestationBase::::deserialize(deserializer) + .map_err(serde::de::Error::custom) + .map(Attestation::Base) } } } -impl ForkVersionDeserialize for Vec> { - fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( - value: serde_json::Value, - fork_name: crate::ForkName, - ) -> Result { - if fork_name.electra_enabled() { - let attestations: Vec> = - serde_json::from_value(value).map_err(serde::de::Error::custom)?; - Ok(attestations - .into_iter() - .map(Attestation::Electra) - .collect::>()) +/* +impl<'de, E: EthSpec> ContextDeserialize<'de, ForkName> for Vec> { + fn context_deserialize( + deserializer: D, + context: ForkName, + ) -> Result + where + D: Deserializer<'de>, + { + if context.electra_enabled() { + >>::deserialize(deserializer) + .map_err(serde::de::Error::custom) + .map(|vec| vec.into_iter().map(Attestation::Electra).collect::>()) } else { - let attestations: Vec> = - serde_json::from_value(value).map_err(serde::de::Error::custom)?; - Ok(attestations - .into_iter() - .map(Attestation::Base) - .collect::>()) + >>::deserialize(deserializer) + .map_err(serde::de::Error::custom) + .map(|vec| vec.into_iter().map(Attestation::Base).collect::>()) } } } +*/ #[derive( Debug, @@ -585,6 +586,7 @@ impl ForkVersionDeserialize for Vec> { TreeHash, PartialEq, )] +#[context_deserialize(ForkName)] pub struct SingleAttestation { #[serde(with = "serde_utils::quoted_u64")] pub committee_index: u64, diff --git a/consensus/types/src/attestation_data.rs b/consensus/types/src/attestation_data.rs index 7578981f514..d0d4dcc5538 100644 --- a/consensus/types/src/attestation_data.rs +++ b/consensus/types/src/attestation_data.rs @@ -1,12 +1,11 @@ -use crate::test_utils::TestRandom; -use crate::{Checkpoint, Hash256, SignedRoot, Slot}; - use crate::slot_data::SlotData; +use crate::test_utils::TestRandom; +use crate::{Checkpoint, ForkName, Hash256, SignedRoot, Slot}; +use context_deserialize_derive::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; - /// The data upon which an attestation is based. /// /// Spec v0.12.1 @@ -25,6 +24,7 @@ use tree_hash_derive::TreeHash; TestRandom, Default, )] +#[context_deserialize(ForkName)] pub struct AttestationData { pub slot: Slot, #[serde(with = "serde_utils::quoted_u64")] diff --git a/consensus/types/src/attester_slashing.rs b/consensus/types/src/attester_slashing.rs index f6aa654d445..8fb5862f219 100644 --- a/consensus/types/src/attester_slashing.rs +++ b/consensus/types/src/attester_slashing.rs @@ -1,10 +1,12 @@ +use crate::context_deserialize; use crate::indexed_attestation::{ IndexedAttestationBase, IndexedAttestationElectra, IndexedAttestationRef, }; use crate::{test_utils::TestRandom, EthSpec}; +use crate::{ContextDeserialize, ForkName}; use derivative::Derivative; use rand::{Rng, RngCore}; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; use ssz_derive::{Decode, Encode}; use superstruct::superstruct; use test_random_derive::TestRandom; @@ -25,6 +27,7 @@ use tree_hash_derive::TreeHash; TestRandom, arbitrary::Arbitrary ), + context_deserialize(ForkName), derivative(PartialEq, Eq, Hash(bound = "E: EthSpec")), serde(bound = "E: EthSpec"), arbitrary(bound = "E: EthSpec") @@ -171,25 +174,27 @@ impl TestRandom for AttesterSlashing { } } -impl crate::ForkVersionDeserialize for Vec> { - fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( - value: serde_json::Value, - fork_name: crate::ForkName, - ) -> Result { - if fork_name.electra_enabled() { - let slashings: Vec> = - serde_json::from_value(value).map_err(serde::de::Error::custom)?; - Ok(slashings - .into_iter() - .map(AttesterSlashing::Electra) - .collect::>()) +impl<'de, E: EthSpec> ContextDeserialize<'de, ForkName> for Vec> { + fn context_deserialize(deserializer: D, context: ForkName) -> Result + where + D: Deserializer<'de>, + { + if context.electra_enabled() { + >>::deserialize(deserializer) + .map_err(serde::de::Error::custom) + .map(|vec| { + vec.into_iter() + .map(AttesterSlashing::Electra) + .collect::>() + }) } else { - let slashings: Vec> = - serde_json::from_value(value).map_err(serde::de::Error::custom)?; - Ok(slashings - .into_iter() - .map(AttesterSlashing::Base) - .collect::>()) + >>::deserialize(deserializer) + .map_err(serde::de::Error::custom) + .map(|vec| { + vec.into_iter() + .map(AttesterSlashing::Base) + .collect::>() + }) } } } diff --git a/consensus/types/src/beacon_block.rs b/consensus/types/src/beacon_block.rs index 6ea897cf1a7..385cd0fcf51 100644 --- a/consensus/types/src/beacon_block.rs +++ b/consensus/types/src/beacon_block.rs @@ -2,7 +2,7 @@ use crate::attestation::AttestationBase; use crate::test_utils::TestRandom; use crate::*; use derivative::Derivative; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; use ssz::{Decode, DecodeError}; use ssz_derive::{Decode, Encode}; use std::fmt; @@ -765,23 +765,21 @@ impl From>> } } -impl> ForkVersionDeserialize +impl<'de, E: EthSpec, Payload: AbstractExecPayload> ContextDeserialize<'de, ForkName> for BeaconBlock { - fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( - value: serde_json::value::Value, - fork_name: ForkName, - ) -> Result { + fn context_deserialize(deserializer: D, context: ForkName) -> Result + where + D: Deserializer<'de>, + { Ok(map_fork_name!( - fork_name, + context, Self, - serde_json::from_value(value).map_err(|e| serde::de::Error::custom(format!( - "BeaconBlock failed to deserialize: {:?}", - e - )))? + serde::Deserialize::deserialize(deserializer)? )) } } + pub enum BlockImportSource { Gossip, Lookup, diff --git a/consensus/types/src/beacon_block_body.rs b/consensus/types/src/beacon_block_body.rs index 10c1a11edec..4440c5cf256 100644 --- a/consensus/types/src/beacon_block_body.rs +++ b/consensus/types/src/beacon_block_body.rs @@ -3,7 +3,7 @@ use crate::*; use derivative::Derivative; use merkle_proof::{MerkleTree, MerkleTreeError}; use metastruct::metastruct; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; use ssz_derive::{Decode, Encode}; use std::marker::PhantomData; use superstruct::superstruct; @@ -48,6 +48,7 @@ pub const BLOB_KZG_COMMITMENTS_INDEX: usize = 11; deny_unknown_fields ), arbitrary(bound = "E: EthSpec, Payload: AbstractExecPayload"), + context_deserialize(ForkName), ), specific_variant_attributes( Base(metastruct(mappings(beacon_block_body_base_fields(groups(fields))))), @@ -61,10 +62,11 @@ pub const BLOB_KZG_COMMITMENTS_INDEX: usize = 11; cast_error(ty = "Error", expr = "Error::IncorrectStateVariant"), partial_getter_error(ty = "Error", expr = "Error::IncorrectStateVariant") )] -#[derive(Debug, Clone, Serialize, Deserialize, Derivative, arbitrary::Arbitrary)] +#[derive(Debug, Clone, Serialize, Deserialize, Derivative, TreeHash, arbitrary::Arbitrary)] #[derivative(PartialEq, Hash(bound = "E: EthSpec"))] #[serde(untagged)] #[serde(bound = "E: EthSpec, Payload: AbstractExecPayload")] +#[tree_hash(enum_behaviour = "transparent")] #[arbitrary(bound = "E: EthSpec, Payload: AbstractExecPayload")] pub struct BeaconBlockBody = FullPayload> { pub randao_reveal: Signature, @@ -980,6 +982,21 @@ impl From>> } } +impl<'de, E: EthSpec, Payload: AbstractExecPayload> ContextDeserialize<'de, ForkName> + for BeaconBlockBody +{ + fn context_deserialize(deserializer: D, context: ForkName) -> Result + where + D: Deserializer<'de>, + { + Ok(map_fork_name!( + context, + Self, + serde::Deserialize::deserialize(deserializer)? + )) + } +} + /// Util method helpful for logging. pub fn format_kzg_commitments(commitments: &[KzgCommitment]) -> String { let commitment_strings: Vec = commitments.iter().map(|x| x.to_string()).collect(); diff --git a/consensus/types/src/beacon_block_header.rs b/consensus/types/src/beacon_block_header.rs index b382359313c..8416f975db2 100644 --- a/consensus/types/src/beacon_block_header.rs +++ b/consensus/types/src/beacon_block_header.rs @@ -1,6 +1,7 @@ use crate::test_utils::TestRandom; use crate::*; +use context_deserialize_derive::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; @@ -24,6 +25,7 @@ use tree_hash_derive::TreeHash; TreeHash, TestRandom, )] +#[context_deserialize(ForkName)] pub struct BeaconBlockHeader { pub slot: Slot, #[serde(with = "serde_utils::quoted_u64")] diff --git a/consensus/types/src/beacon_response.rs b/consensus/types/src/beacon_response.rs new file mode 100644 index 00000000000..2e458543649 --- /dev/null +++ b/consensus/types/src/beacon_response.rs @@ -0,0 +1,239 @@ +use crate::{ContextDeserialize, ForkName}; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Deserializer, Serialize}; +use serde_json::value::Value; + +pub trait ForkVersionDecode: Sized { + /// SSZ decode with explicit fork variant. + fn from_ssz_bytes_by_fork(bytes: &[u8], fork_name: ForkName) -> Result; +} + +/// The metadata of type M should be set to `EmptyMetadata` if you don't care about adding fields other than +/// version. If you *do* care about adding other fields you can mix in any type that implements +/// `Deserialize`. +#[derive(Debug, PartialEq, Clone, Serialize)] +pub struct ForkVersionedResponse { + pub version: ForkName, + #[serde(flatten)] + pub metadata: M, + pub data: T, +} + +// Used for responses to V1 endpoints that don't have a version field. +/// The metadata of type M should be set to `EmptyMetadata` if you don't care about adding fields other than +/// version. If you *do* care about adding other fields you can mix in any type that implements +/// `Deserialize`. +#[derive(Debug, PartialEq, Clone, Serialize)] +pub struct UnversionedResponse { + pub metadata: M, + pub data: T, +} + +#[derive(Debug, PartialEq, Clone, Serialize)] +#[serde(untagged)] +pub enum BeaconResponse { + ForkVersioned(ForkVersionedResponse), + Unversioned(UnversionedResponse), +} + +impl BeaconResponse { + pub fn version(&self) -> Option { + match self { + BeaconResponse::ForkVersioned(response) => Some(response.version), + BeaconResponse::Unversioned(_) => None, + } + } + + pub fn data(&self) -> &T { + match self { + BeaconResponse::ForkVersioned(response) => &response.data, + BeaconResponse::Unversioned(response) => &response.data, + } + } + + pub fn metadata(&self) -> &M { + match self { + BeaconResponse::ForkVersioned(response) => &response.metadata, + BeaconResponse::Unversioned(response) => &response.metadata, + } + } +} + +/// Metadata type similar to unit (i.e. `()`) but deserializes from a map (`serde_json::Value`). +/// +/// Unfortunately the braces are semantically significant, i.e. `struct EmptyMetadata;` does not +/// work. +#[derive(Debug, PartialEq, Clone, Default, Deserialize, Serialize)] +pub struct EmptyMetadata {} + +/// Fork versioned response with extra information about finalization & optimistic execution. +pub type ExecutionOptimisticFinalizedBeaconResponse = + BeaconResponse; + +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] +pub struct ExecutionOptimisticFinalizedMetadata { + pub execution_optimistic: Option, + pub finalized: Option, +} + +impl<'de, T, M> Deserialize<'de> for ForkVersionedResponse +where + T: ContextDeserialize<'de, ForkName>, + M: DeserializeOwned, +{ + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper { + version: ForkName, + #[serde(flatten)] + metadata: Value, + data: Value, + } + + let helper = Helper::deserialize(deserializer)?; + + // Deserialize metadata + let metadata = serde_json::from_value(helper.metadata).map_err(serde::de::Error::custom)?; + + // Deserialize `data` using ContextDeserialize + let data = T::context_deserialize(helper.data, helper.version) + .map_err(serde::de::Error::custom)?; + + Ok(ForkVersionedResponse { + version: helper.version, + metadata, + data, + }) + } +} + +impl<'de, T, M> Deserialize<'de> for UnversionedResponse +where + T: DeserializeOwned, + M: DeserializeOwned, +{ + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper { + #[serde(flatten)] + metadata: M, + data: T, + } + + let helper = Helper::deserialize(deserializer)?; + + Ok(UnversionedResponse { + metadata: helper.metadata, + data: helper.data, + }) + } +} + +impl BeaconResponse { + pub fn map_data(self, f: impl FnOnce(T) -> U) -> BeaconResponse { + match self { + BeaconResponse::ForkVersioned(response) => { + BeaconResponse::ForkVersioned(response.map_data(f)) + } + BeaconResponse::Unversioned(response) => { + BeaconResponse::Unversioned(response.map_data(f)) + } + } + } + + pub fn into_data(self) -> T { + match self { + BeaconResponse::ForkVersioned(response) => response.data, + BeaconResponse::Unversioned(response) => response.data, + } + } +} + +impl UnversionedResponse { + pub fn map_data(self, f: impl FnOnce(T) -> U) -> UnversionedResponse { + let UnversionedResponse { metadata, data } = self; + UnversionedResponse { + metadata, + data: f(data), + } + } +} + +impl ForkVersionedResponse { + /// Apply a function to the inner `data`, potentially changing its type. + pub fn map_data(self, f: impl FnOnce(T) -> U) -> ForkVersionedResponse { + let ForkVersionedResponse { + version, + metadata, + data, + } = self; + ForkVersionedResponse { + version, + metadata, + data: f(data), + } + } +} + +impl From> for BeaconResponse { + fn from(response: ForkVersionedResponse) -> Self { + BeaconResponse::ForkVersioned(response) + } +} + +impl From> for BeaconResponse { + fn from(response: UnversionedResponse) -> Self { + BeaconResponse::Unversioned(response) + } +} + +#[cfg(test)] +mod fork_version_response_tests { + use crate::{ + ExecutionPayload, ExecutionPayloadBellatrix, ForkName, ForkVersionedResponse, + MainnetEthSpec, + }; + use serde_json::json; + + #[test] + fn fork_versioned_response_deserialize_correct_fork() { + type E = MainnetEthSpec; + + let response_json = + serde_json::to_string(&json!(ForkVersionedResponse::> { + version: ForkName::Bellatrix, + metadata: Default::default(), + data: ExecutionPayload::Bellatrix(ExecutionPayloadBellatrix::default()), + })) + .unwrap(); + + let result: Result>, _> = + serde_json::from_str(&response_json); + + assert!(result.is_ok()); + } + + #[test] + fn fork_versioned_response_deserialize_incorrect_fork() { + type E = MainnetEthSpec; + + let response_json = + serde_json::to_string(&json!(ForkVersionedResponse::> { + version: ForkName::Capella, + metadata: Default::default(), + data: ExecutionPayload::Bellatrix(ExecutionPayloadBellatrix::default()), + })) + .unwrap(); + + let result: Result>, _> = + serde_json::from_str(&response_json); + + assert!(result.is_err()); + } +} diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index 4aed79898d3..ce41eddc179 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -1,6 +1,7 @@ use self::committee_cache::get_active_validator_indices; use crate::historical_summary::HistoricalSummary; use crate::test_utils::TestRandom; +use crate::ContextDeserialize; use crate::FixedBytesExtended; use crate::*; use compare_fields::CompareFields; @@ -11,7 +12,7 @@ use int_to_bytes::{int_to_bytes4, int_to_bytes8}; use metastruct::{metastruct, NumFields}; pub use pubkey_cache::PubkeyCache; use safe_arith::{ArithError, SafeArith}; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; use ssz::{ssz_encode, Decode, DecodeError, Encode}; use ssz_derive::{Decode, Encode}; use std::hash::Hash; @@ -2748,18 +2749,15 @@ impl CompareFields for BeaconState { } } -impl ForkVersionDeserialize for BeaconState { - fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( - value: serde_json::value::Value, - fork_name: ForkName, - ) -> Result { +impl<'de, E: EthSpec> ContextDeserialize<'de, ForkName> for BeaconState { + fn context_deserialize(deserializer: D, context: ForkName) -> Result + where + D: Deserializer<'de>, + { Ok(map_fork_name!( - fork_name, + context, Self, - serde_json::from_value(value).map_err(|e| serde::de::Error::custom(format!( - "BeaconState failed to deserialize: {:?}", - e - )))? + serde::Deserialize::deserialize(deserializer)? )) } } diff --git a/consensus/types/src/blob_sidecar.rs b/consensus/types/src/blob_sidecar.rs index ff4555747c6..f7a5725c5a0 100644 --- a/consensus/types/src/blob_sidecar.rs +++ b/consensus/types/src/blob_sidecar.rs @@ -1,9 +1,10 @@ +use crate::context_deserialize; use crate::test_utils::TestRandom; use crate::{ beacon_block_body::BLOB_KZG_COMMITMENTS_INDEX, AbstractExecPayload, BeaconBlockHeader, - BeaconStateError, Blob, ChainSpec, Epoch, EthSpec, FixedVector, ForkName, - ForkVersionDeserialize, Hash256, KzgProofs, RuntimeFixedVector, RuntimeVariableList, - SignedBeaconBlock, SignedBeaconBlockHeader, Slot, VariableList, + BeaconStateError, Blob, ChainSpec, Epoch, EthSpec, FixedVector, ForkName, Hash256, KzgProofs, + RuntimeFixedVector, RuntimeVariableList, SignedBeaconBlock, SignedBeaconBlockHeader, Slot, + VariableList, }; use bls::Signature; use derivative::Derivative; @@ -25,6 +26,7 @@ use tree_hash_derive::TreeHash; #[derive( Serialize, Deserialize, Encode, Decode, TreeHash, Copy, Clone, Debug, PartialEq, Eq, Hash, )] +#[context_deserialize(ForkName)] pub struct BlobIdentifier { pub block_root: Hash256, pub index: u64, @@ -54,6 +56,7 @@ impl Ord for BlobIdentifier { Derivative, arbitrary::Arbitrary, )] +#[context_deserialize(ForkName)] #[serde(bound = "E: EthSpec")] #[arbitrary(bound = "E: EthSpec")] #[derivative(PartialEq, Eq, Hash(bound = "E: EthSpec"))] @@ -296,12 +299,3 @@ pub type BlobSidecarList = RuntimeVariableList>>; /// Alias for a non length-constrained list of `BlobSidecar`s. pub type FixedBlobSidecarList = RuntimeFixedVector>>>; pub type BlobsList = VariableList, ::MaxBlobCommitmentsPerBlock>; - -impl ForkVersionDeserialize for BlobSidecarList { - fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( - value: serde_json::value::Value, - _: ForkName, - ) -> Result { - serde_json::from_value::>(value).map_err(serde::de::Error::custom) - } -} diff --git a/consensus/types/src/bls_to_execution_change.rs b/consensus/types/src/bls_to_execution_change.rs index 07d71b360f9..b333862220d 100644 --- a/consensus/types/src/bls_to_execution_change.rs +++ b/consensus/types/src/bls_to_execution_change.rs @@ -19,6 +19,7 @@ use tree_hash_derive::TreeHash; TreeHash, TestRandom, )] +#[context_deserialize(ForkName)] pub struct BlsToExecutionChange { #[serde(with = "serde_utils::quoted_u64")] pub validator_index: u64, diff --git a/consensus/types/src/builder_bid.rs b/consensus/types/src/builder_bid.rs index 5c146c41542..f8d61a0da5b 100644 --- a/consensus/types/src/builder_bid.rs +++ b/consensus/types/src/builder_bid.rs @@ -1,9 +1,10 @@ use crate::beacon_block_body::KzgCommitments; use crate::{ - test_utils::TestRandom, ChainSpec, EthSpec, ExecutionPayloadHeaderBellatrix, - ExecutionPayloadHeaderCapella, ExecutionPayloadHeaderDeneb, ExecutionPayloadHeaderElectra, - ExecutionPayloadHeaderFulu, ExecutionPayloadHeaderRef, ExecutionPayloadHeaderRefMut, - ExecutionRequests, ForkName, ForkVersionDecode, ForkVersionDeserialize, SignedRoot, Uint256, + test_utils::TestRandom, ChainSpec, ContextDeserialize, EthSpec, + ExecutionPayloadHeaderBellatrix, ExecutionPayloadHeaderCapella, ExecutionPayloadHeaderDeneb, + ExecutionPayloadHeaderElectra, ExecutionPayloadHeaderFulu, ExecutionPayloadHeaderRef, + ExecutionPayloadHeaderRefMut, ExecutionRequests, ForkName, ForkVersionDecode, SignedRoot, + Uint256, }; use bls::PublicKeyBytes; use bls::Signature; @@ -127,46 +128,58 @@ impl ForkVersionDecode for SignedBuilderBid { } } -impl ForkVersionDeserialize for BuilderBid { - fn deserialize_by_fork<'de, D: Deserializer<'de>>( - value: serde_json::value::Value, - fork_name: ForkName, - ) -> Result { +impl<'de, E: EthSpec> ContextDeserialize<'de, ForkName> for BuilderBid { + fn context_deserialize(deserializer: D, context: ForkName) -> Result + where + D: Deserializer<'de>, + { let convert_err = |e| serde::de::Error::custom(format!("BuilderBid failed to deserialize: {:?}", e)); - - Ok(match fork_name { + Ok(match context { ForkName::Bellatrix => { - Self::Bellatrix(serde_json::from_value(value).map_err(convert_err)?) + Self::Bellatrix(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Capella => { + Self::Capella(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Deneb => { + Self::Deneb(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Electra => { + Self::Electra(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Fulu => { + Self::Fulu(Deserialize::deserialize(deserializer).map_err(convert_err)?) } - ForkName::Capella => Self::Capella(serde_json::from_value(value).map_err(convert_err)?), - ForkName::Deneb => Self::Deneb(serde_json::from_value(value).map_err(convert_err)?), - ForkName::Electra => Self::Electra(serde_json::from_value(value).map_err(convert_err)?), - ForkName::Fulu => Self::Fulu(serde_json::from_value(value).map_err(convert_err)?), ForkName::Base | ForkName::Altair => { return Err(serde::de::Error::custom(format!( "BuilderBid failed to deserialize: unsupported fork '{}'", - fork_name + context ))); } }) } } -impl ForkVersionDeserialize for SignedBuilderBid { - fn deserialize_by_fork<'de, D: Deserializer<'de>>( - value: serde_json::value::Value, - fork_name: ForkName, - ) -> Result { +impl<'de, E: EthSpec> ContextDeserialize<'de, ForkName> for SignedBuilderBid { + fn context_deserialize(deserializer: D, context: ForkName) -> Result + where + D: Deserializer<'de>, + { #[derive(Deserialize)] struct Helper { - pub message: serde_json::Value, - pub signature: Signature, + message: serde_json::Value, + signature: Signature, } - let helper: Helper = serde_json::from_value(value).map_err(serde::de::Error::custom)?; - Ok(Self { - message: BuilderBid::deserialize_by_fork::<'de, D>(helper.message, fork_name)?, + let helper = Helper::deserialize(deserializer)?; + + // Deserialize `data` using ContextDeserialize + let message = BuilderBid::::context_deserialize(helper.message, context) + .map_err(serde::de::Error::custom)?; + + Ok(SignedBuilderBid { + message, signature: helper.signature, }) } diff --git a/consensus/types/src/checkpoint.rs b/consensus/types/src/checkpoint.rs index 044fc57f22a..c3cb1d5c36d 100644 --- a/consensus/types/src/checkpoint.rs +++ b/consensus/types/src/checkpoint.rs @@ -1,5 +1,6 @@ use crate::test_utils::TestRandom; -use crate::{Epoch, Hash256}; +use crate::{Epoch, ForkName, Hash256}; +use context_deserialize_derive::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; @@ -24,6 +25,7 @@ use tree_hash_derive::TreeHash; TreeHash, TestRandom, )] +#[context_deserialize(ForkName)] pub struct Checkpoint { pub epoch: Epoch, pub root: Hash256, diff --git a/consensus/types/src/consolidation_request.rs b/consensus/types/src/consolidation_request.rs index e2df0bb9726..c7375dab844 100644 --- a/consensus/types/src/consolidation_request.rs +++ b/consensus/types/src/consolidation_request.rs @@ -1,4 +1,5 @@ -use crate::{test_utils::TestRandom, Address, PublicKeyBytes, SignedRoot}; +use crate::context_deserialize; +use crate::{test_utils::TestRandom, Address, ForkName, PublicKeyBytes, SignedRoot}; use serde::{Deserialize, Serialize}; use ssz::Encode; use ssz_derive::{Decode, Encode}; @@ -19,6 +20,7 @@ use tree_hash_derive::TreeHash; TreeHash, TestRandom, )] +#[context_deserialize(ForkName)] pub struct ConsolidationRequest { pub source_address: Address, pub source_pubkey: PublicKeyBytes, diff --git a/consensus/types/src/contribution_and_proof.rs b/consensus/types/src/contribution_and_proof.rs index 321c12d2206..e918beacb00 100644 --- a/consensus/types/src/contribution_and_proof.rs +++ b/consensus/types/src/contribution_and_proof.rs @@ -1,7 +1,8 @@ use super::{ - ChainSpec, EthSpec, Fork, Hash256, SecretKey, Signature, SignedRoot, SyncCommitteeContribution, - SyncSelectionProof, + ChainSpec, EthSpec, Fork, ForkName, Hash256, SecretKey, Signature, SignedRoot, + SyncCommitteeContribution, SyncSelectionProof, }; +use crate::context_deserialize; use crate::test_utils::TestRandom; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; @@ -23,6 +24,7 @@ use tree_hash_derive::TreeHash; )] #[serde(bound = "E: EthSpec")] #[arbitrary(bound = "E: EthSpec")] +#[context_deserialize(ForkName)] pub struct ContributionAndProof { /// The index of the validator that created the sync contribution. #[serde(with = "serde_utils::quoted_u64")] diff --git a/consensus/types/src/data_column_sidecar.rs b/consensus/types/src/data_column_sidecar.rs index d2802670b65..5ec2b28b2bf 100644 --- a/consensus/types/src/data_column_sidecar.rs +++ b/consensus/types/src/data_column_sidecar.rs @@ -1,7 +1,8 @@ use crate::beacon_block_body::{KzgCommitments, BLOB_KZG_COMMITMENTS_INDEX}; +use crate::context_deserialize; use crate::test_utils::TestRandom; use crate::{ - BeaconBlockHeader, BeaconStateError, Epoch, EthSpec, Hash256, RuntimeVariableList, + BeaconBlockHeader, BeaconStateError, Epoch, EthSpec, ForkName, Hash256, RuntimeVariableList, SignedBeaconBlockHeader, Slot, }; use bls::Signature; @@ -84,6 +85,7 @@ pub type DataColumnSidecarList = Vec>>; #[serde(bound = "E: EthSpec")] #[arbitrary(bound = "E: EthSpec")] #[derivative(PartialEq, Eq, Hash(bound = "E: EthSpec"))] +#[context_deserialize(ForkName)] pub struct DataColumnSidecar { #[serde(with = "serde_utils::quoted_u64")] pub index: ColumnIndex, diff --git a/consensus/types/src/deposit.rs b/consensus/types/src/deposit.rs index c818c7d8081..8b4b6af95dd 100644 --- a/consensus/types/src/deposit.rs +++ b/consensus/types/src/deposit.rs @@ -1,3 +1,4 @@ +use crate::context_deserialize; use crate::test_utils::TestRandom; use crate::*; use serde::{Deserialize, Serialize}; @@ -24,6 +25,7 @@ pub const DEPOSIT_TREE_DEPTH: usize = 32; TreeHash, TestRandom, )] +#[context_deserialize(ForkName)] pub struct Deposit { pub proof: FixedVector, pub data: DepositData, diff --git a/consensus/types/src/deposit_data.rs b/consensus/types/src/deposit_data.rs index f62829e7953..d29e8c8d14a 100644 --- a/consensus/types/src/deposit_data.rs +++ b/consensus/types/src/deposit_data.rs @@ -1,6 +1,5 @@ use crate::test_utils::TestRandom; use crate::*; - use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; @@ -22,6 +21,7 @@ use tree_hash_derive::TreeHash; TreeHash, TestRandom, )] +#[context_deserialize(ForkName)] pub struct DepositData { pub pubkey: PublicKeyBytes, pub withdrawal_credentials: Hash256, diff --git a/consensus/types/src/deposit_message.rs b/consensus/types/src/deposit_message.rs index 6184d0aeb32..5c2a0b7c2bd 100644 --- a/consensus/types/src/deposit_message.rs +++ b/consensus/types/src/deposit_message.rs @@ -21,6 +21,7 @@ use tree_hash_derive::TreeHash; TreeHash, TestRandom, )] +#[context_deserialize(ForkName)] pub struct DepositMessage { pub pubkey: PublicKeyBytes, pub withdrawal_credentials: Hash256, diff --git a/consensus/types/src/deposit_request.rs b/consensus/types/src/deposit_request.rs index a21760551b5..141258b5ab6 100644 --- a/consensus/types/src/deposit_request.rs +++ b/consensus/types/src/deposit_request.rs @@ -1,5 +1,6 @@ +use crate::context_deserialize; use crate::test_utils::TestRandom; -use crate::{Hash256, PublicKeyBytes}; +use crate::{ForkName, Hash256, PublicKeyBytes}; use bls::SignatureBytes; use serde::{Deserialize, Serialize}; use ssz::Encode; @@ -20,6 +21,7 @@ use tree_hash_derive::TreeHash; TreeHash, TestRandom, )] +#[context_deserialize(ForkName)] pub struct DepositRequest { pub pubkey: PublicKeyBytes, pub withdrawal_credentials: Hash256, diff --git a/consensus/types/src/eth1_data.rs b/consensus/types/src/eth1_data.rs index e2c4e511ef3..7bd0d3228d7 100644 --- a/consensus/types/src/eth1_data.rs +++ b/consensus/types/src/eth1_data.rs @@ -1,6 +1,7 @@ use super::Hash256; +use crate::context_deserialize; use crate::test_utils::TestRandom; - +use crate::ForkName; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; @@ -24,6 +25,7 @@ use tree_hash_derive::TreeHash; TreeHash, TestRandom, )] +#[context_deserialize(ForkName)] pub struct Eth1Data { pub deposit_root: Hash256, #[serde(with = "serde_utils::quoted_u64")] diff --git a/consensus/types/src/execution_payload.rs b/consensus/types/src/execution_payload.rs index 5d756c8529f..b4b06081505 100644 --- a/consensus/types/src/execution_payload.rs +++ b/consensus/types/src/execution_payload.rs @@ -1,6 +1,6 @@ use crate::{test_utils::TestRandom, *}; use derivative::Derivative; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; @@ -30,6 +30,7 @@ pub type Withdrawals = VariableList::MaxWithdrawal Derivative, arbitrary::Arbitrary ), + context_deserialize(ForkName), derivative(PartialEq, Hash(bound = "E: EthSpec")), serde(bound = "E: EthSpec", deny_unknown_fields), arbitrary(bound = "E: EthSpec") @@ -133,28 +134,35 @@ impl ExecutionPayload { } } -impl ForkVersionDeserialize for ExecutionPayload { - fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( - value: serde_json::value::Value, - fork_name: ForkName, - ) -> Result { +impl<'de, E: EthSpec> ContextDeserialize<'de, ForkName> for ExecutionPayload { + fn context_deserialize(deserializer: D, context: ForkName) -> Result + where + D: Deserializer<'de>, + { let convert_err = |e| { serde::de::Error::custom(format!("ExecutionPayload failed to deserialize: {:?}", e)) }; - - Ok(match fork_name { - ForkName::Bellatrix => { - Self::Bellatrix(serde_json::from_value(value).map_err(convert_err)?) - } - ForkName::Capella => Self::Capella(serde_json::from_value(value).map_err(convert_err)?), - ForkName::Deneb => Self::Deneb(serde_json::from_value(value).map_err(convert_err)?), - ForkName::Electra => Self::Electra(serde_json::from_value(value).map_err(convert_err)?), - ForkName::Fulu => Self::Fulu(serde_json::from_value(value).map_err(convert_err)?), + Ok(match context { ForkName::Base | ForkName::Altair => { return Err(serde::de::Error::custom(format!( "ExecutionPayload failed to deserialize: unsupported fork '{}'", - fork_name - ))); + context + ))) + } + ForkName::Bellatrix => { + Self::Bellatrix(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Capella => { + Self::Capella(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Deneb => { + Self::Deneb(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Electra => { + Self::Electra(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Fulu => { + Self::Fulu(Deserialize::deserialize(deserializer).map_err(convert_err)?) } }) } diff --git a/consensus/types/src/execution_payload_header.rs b/consensus/types/src/execution_payload_header.rs index 3012041b8b1..a16f29819d9 100644 --- a/consensus/types/src/execution_payload_header.rs +++ b/consensus/types/src/execution_payload_header.rs @@ -1,6 +1,6 @@ use crate::{test_utils::TestRandom, *}; use derivative::Derivative; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; @@ -25,7 +25,8 @@ use tree_hash_derive::TreeHash; ), derivative(PartialEq, Hash(bound = "E: EthSpec")), serde(bound = "E: EthSpec", deny_unknown_fields), - arbitrary(bound = "E: EthSpec") + arbitrary(bound = "E: EthSpec"), + context_deserialize(ForkName), ), ref_attributes( derive(PartialEq, TreeHash, Debug), @@ -472,31 +473,38 @@ impl TryFrom> for ExecutionPayloadHeaderFu } } -impl ForkVersionDeserialize for ExecutionPayloadHeader { - fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( - value: serde_json::value::Value, - fork_name: ForkName, - ) -> Result { +impl<'de, E: EthSpec> ContextDeserialize<'de, ForkName> for ExecutionPayloadHeader { + fn context_deserialize(deserializer: D, context: ForkName) -> Result + where + D: Deserializer<'de>, + { let convert_err = |e| { serde::de::Error::custom(format!( "ExecutionPayloadHeader failed to deserialize: {:?}", e )) }; - - Ok(match fork_name { - ForkName::Bellatrix => { - Self::Bellatrix(serde_json::from_value(value).map_err(convert_err)?) - } - ForkName::Capella => Self::Capella(serde_json::from_value(value).map_err(convert_err)?), - ForkName::Deneb => Self::Deneb(serde_json::from_value(value).map_err(convert_err)?), - ForkName::Electra => Self::Electra(serde_json::from_value(value).map_err(convert_err)?), - ForkName::Fulu => Self::Fulu(serde_json::from_value(value).map_err(convert_err)?), + Ok(match context { ForkName::Base | ForkName::Altair => { return Err(serde::de::Error::custom(format!( "ExecutionPayloadHeader failed to deserialize: unsupported fork '{}'", - fork_name - ))); + context + ))) + } + ForkName::Bellatrix => { + Self::Bellatrix(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Capella => { + Self::Capella(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Deneb => { + Self::Deneb(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Electra => { + Self::Electra(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Fulu => { + Self::Fulu(Deserialize::deserialize(deserializer).map_err(convert_err)?) } }) } diff --git a/consensus/types/src/execution_requests.rs b/consensus/types/src/execution_requests.rs index 223c6444ccf..2fec3b5f665 100644 --- a/consensus/types/src/execution_requests.rs +++ b/consensus/types/src/execution_requests.rs @@ -1,5 +1,6 @@ +use crate::context_deserialize; use crate::test_utils::TestRandom; -use crate::{ConsolidationRequest, DepositRequest, EthSpec, Hash256, WithdrawalRequest}; +use crate::{ConsolidationRequest, DepositRequest, EthSpec, ForkName, Hash256, WithdrawalRequest}; use alloy_primitives::Bytes; use derivative::Derivative; use ethereum_hashing::{DynamicContext, Sha256Context}; @@ -33,6 +34,7 @@ pub type ConsolidationRequests = #[serde(bound = "E: EthSpec")] #[arbitrary(bound = "E: EthSpec")] #[derivative(PartialEq, Eq, Hash(bound = "E: EthSpec"))] +#[context_deserialize(ForkName)] pub struct ExecutionRequests { pub deposits: DepositRequests, pub withdrawals: WithdrawalRequests, diff --git a/consensus/types/src/fork.rs b/consensus/types/src/fork.rs index b23113f4363..239ffe33c0e 100644 --- a/consensus/types/src/fork.rs +++ b/consensus/types/src/fork.rs @@ -1,5 +1,6 @@ use crate::test_utils::TestRandom; -use crate::Epoch; +use crate::{Epoch, ForkName}; +use context_deserialize_derive::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; @@ -23,6 +24,7 @@ use tree_hash_derive::TreeHash; TreeHash, TestRandom, )] +#[context_deserialize(ForkName)] pub struct Fork { #[serde(with = "serde_utils::bytes_4_hex")] pub previous_version: [u8; 4], diff --git a/consensus/types/src/fork_data.rs b/consensus/types/src/fork_data.rs index 52ce57a2a94..1ac91084d22 100644 --- a/consensus/types/src/fork_data.rs +++ b/consensus/types/src/fork_data.rs @@ -1,5 +1,6 @@ use crate::test_utils::TestRandom; -use crate::{Hash256, SignedRoot}; +use crate::{ForkName, Hash256, SignedRoot}; +use context_deserialize_derive::context_deserialize; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; @@ -22,6 +23,7 @@ use tree_hash_derive::TreeHash; TreeHash, TestRandom, )] +#[context_deserialize(ForkName)] pub struct ForkData { #[serde(with = "serde_utils::bytes_4_hex")] pub current_version: [u8; 4], diff --git a/consensus/types/src/fork_versioned_response.rs b/consensus/types/src/fork_versioned_response.rs deleted file mode 100644 index 7e4efd05d66..00000000000 --- a/consensus/types/src/fork_versioned_response.rs +++ /dev/null @@ -1,152 +0,0 @@ -use crate::ForkName; -use serde::de::DeserializeOwned; -use serde::{Deserialize, Deserializer, Serialize}; -use serde_json::value::Value; -use std::sync::Arc; - -pub trait ForkVersionDecode: Sized { - /// SSZ decode with explicit fork variant. - fn from_ssz_bytes_by_fork(bytes: &[u8], fork_name: ForkName) -> Result; -} - -pub trait ForkVersionDeserialize: Sized + DeserializeOwned { - fn deserialize_by_fork<'de, D: Deserializer<'de>>( - value: Value, - fork_name: ForkName, - ) -> Result; -} - -/// Deserialize is only implemented for types that implement ForkVersionDeserialize. -/// -/// The metadata of type M should be set to `EmptyMetadata` if you don't care about adding fields other than -/// version. If you *do* care about adding other fields you can mix in any type that implements -/// `Deserialize`. -#[derive(Debug, PartialEq, Clone, Serialize)] -pub struct ForkVersionedResponse { - #[serde(skip_serializing_if = "Option::is_none")] - pub version: Option, - #[serde(flatten)] - pub metadata: M, - pub data: T, -} - -/// Metadata type similar to unit (i.e. `()`) but deserializes from a map (`serde_json::Value`). -/// -/// Unfortunately the braces are semantically significant, i.e. `struct EmptyMetadata;` does not -/// work. -#[derive(Debug, PartialEq, Clone, Default, Deserialize, Serialize)] -pub struct EmptyMetadata {} - -/// Fork versioned response with extra information about finalization & optimistic execution. -pub type ExecutionOptimisticFinalizedForkVersionedResponse = - ForkVersionedResponse; - -#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] -pub struct ExecutionOptimisticFinalizedMetadata { - pub execution_optimistic: Option, - pub finalized: Option, -} - -impl<'de, F, M> serde::Deserialize<'de> for ForkVersionedResponse -where - F: ForkVersionDeserialize, - M: DeserializeOwned, -{ - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - #[derive(Deserialize)] - struct Helper { - version: Option, - #[serde(flatten)] - metadata: serde_json::Value, - data: serde_json::Value, - } - - let helper = Helper::deserialize(deserializer)?; - let data = match helper.version { - Some(fork_name) => F::deserialize_by_fork::<'de, D>(helper.data, fork_name)?, - None => serde_json::from_value(helper.data).map_err(serde::de::Error::custom)?, - }; - let metadata = serde_json::from_value(helper.metadata).map_err(serde::de::Error::custom)?; - - Ok(ForkVersionedResponse { - version: helper.version, - metadata, - data, - }) - } -} - -impl ForkVersionDeserialize for Arc { - fn deserialize_by_fork<'de, D: Deserializer<'de>>( - value: Value, - fork_name: ForkName, - ) -> Result { - Ok(Arc::new(F::deserialize_by_fork::<'de, D>( - value, fork_name, - )?)) - } -} - -impl ForkVersionedResponse { - /// Apply a function to the inner `data`, potentially changing its type. - pub fn map_data(self, f: impl FnOnce(T) -> U) -> ForkVersionedResponse { - let ForkVersionedResponse { - version, - metadata, - data, - } = self; - ForkVersionedResponse { - version, - metadata, - data: f(data), - } - } -} - -#[cfg(test)] -mod fork_version_response_tests { - use crate::{ - ExecutionPayload, ExecutionPayloadBellatrix, ForkName, ForkVersionedResponse, - MainnetEthSpec, - }; - use serde_json::json; - - #[test] - fn fork_versioned_response_deserialize_correct_fork() { - type E = MainnetEthSpec; - - let response_json = - serde_json::to_string(&json!(ForkVersionedResponse::> { - version: Some(ForkName::Bellatrix), - metadata: Default::default(), - data: ExecutionPayload::Bellatrix(ExecutionPayloadBellatrix::default()), - })) - .unwrap(); - - let result: Result>, _> = - serde_json::from_str(&response_json); - - assert!(result.is_ok()); - } - - #[test] - fn fork_versioned_response_deserialize_incorrect_fork() { - type E = MainnetEthSpec; - - let response_json = - serde_json::to_string(&json!(ForkVersionedResponse::> { - version: Some(ForkName::Capella), - metadata: Default::default(), - data: ExecutionPayload::Bellatrix(ExecutionPayloadBellatrix::default()), - })) - .unwrap(); - - let result: Result>, _> = - serde_json::from_str(&response_json); - - assert!(result.is_err()); - } -} diff --git a/consensus/types/src/historical_batch.rs b/consensus/types/src/historical_batch.rs index 7bac9699eb6..3a02810bba3 100644 --- a/consensus/types/src/historical_batch.rs +++ b/consensus/types/src/historical_batch.rs @@ -22,6 +22,7 @@ use tree_hash_derive::TreeHash; arbitrary::Arbitrary, )] #[arbitrary(bound = "E: EthSpec")] +#[context_deserialize(ForkName)] pub struct HistoricalBatch { #[test_random(default)] pub block_roots: Vector, diff --git a/consensus/types/src/historical_summary.rs b/consensus/types/src/historical_summary.rs index 8c82d52b810..7ad423dadec 100644 --- a/consensus/types/src/historical_summary.rs +++ b/consensus/types/src/historical_summary.rs @@ -1,5 +1,6 @@ +use crate::context_deserialize; use crate::test_utils::TestRandom; -use crate::{BeaconState, EthSpec, Hash256}; +use crate::{BeaconState, EthSpec, ForkName, Hash256}; use compare_fields_derive::CompareFields; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; @@ -28,6 +29,7 @@ use tree_hash_derive::TreeHash; Default, arbitrary::Arbitrary, )] +#[context_deserialize(ForkName)] pub struct HistoricalSummary { block_summary_root: Hash256, state_summary_root: Hash256, diff --git a/consensus/types/src/indexed_attestation.rs b/consensus/types/src/indexed_attestation.rs index f3243a9f05e..ea65d785041 100644 --- a/consensus/types/src/indexed_attestation.rs +++ b/consensus/types/src/indexed_attestation.rs @@ -1,4 +1,7 @@ -use crate::{test_utils::TestRandom, AggregateSignature, AttestationData, EthSpec, VariableList}; +use crate::context_deserialize; +use crate::{ + test_utils::TestRandom, AggregateSignature, AttestationData, EthSpec, ForkName, VariableList, +}; use core::slice::Iter; use derivative::Derivative; use serde::{Deserialize, Serialize}; @@ -29,6 +32,7 @@ use tree_hash_derive::TreeHash; arbitrary::Arbitrary, TreeHash, ), + context_deserialize(ForkName), derivative(PartialEq, Hash(bound = "E: EthSpec")), serde(bound = "E: EthSpec", deny_unknown_fields), arbitrary(bound = "E: EthSpec"), diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 70f07f01090..f0555a06d6d 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -22,6 +22,7 @@ pub mod beacon_block; pub mod beacon_block_body; pub mod beacon_block_header; pub mod beacon_committee; +pub mod beacon_response; pub mod beacon_state; pub mod bls_to_execution_change; pub mod builder_bid; @@ -44,7 +45,6 @@ pub mod execution_payload_header; pub mod fork; pub mod fork_data; pub mod fork_name; -pub mod fork_versioned_response; pub mod graffiti; pub mod historical_batch; pub mod historical_summary; @@ -138,6 +138,9 @@ pub use crate::beacon_block_body::{ }; pub use crate::beacon_block_header::BeaconBlockHeader; pub use crate::beacon_committee::{BeaconCommittee, OwnedBeaconCommittee}; +pub use crate::beacon_response::{ + BeaconResponse, ForkVersionDecode, ForkVersionedResponse, UnversionedResponse, +}; pub use crate::beacon_state::{Error as BeaconStateError, *}; pub use crate::blob_sidecar::{BlobIdentifier, BlobSidecar, BlobSidecarList, BlobsList}; pub use crate::bls_to_execution_change::BlsToExecutionChange; @@ -178,9 +181,6 @@ pub use crate::fork::Fork; pub use crate::fork_context::ForkContext; pub use crate::fork_data::ForkData; pub use crate::fork_name::{ForkName, InconsistentFork}; -pub use crate::fork_versioned_response::{ - ForkVersionDecode, ForkVersionDeserialize, ForkVersionedResponse, -}; pub use crate::graffiti::{Graffiti, GRAFFITI_BYTES_LEN}; pub use crate::historical_batch::HistoricalBatch; pub use crate::indexed_attestation::{ @@ -287,6 +287,8 @@ pub use bls::{ AggregatePublicKey, AggregateSignature, Keypair, PublicKey, PublicKeyBytes, SecretKey, Signature, SignatureBytes, }; +pub use context_deserialize::ContextDeserialize; +pub use context_deserialize_derive::context_deserialize; pub use kzg::{KzgCommitment, KzgProof, VERSIONED_HASH_VERSION_KZG}; pub use milhouse::{self, List, Vector}; pub use ssz_types::{typenum, typenum::Unsigned, BitList, BitVector, FixedVector, VariableList}; diff --git a/consensus/types/src/light_client_bootstrap.rs b/consensus/types/src/light_client_bootstrap.rs index aa0d8836d14..e82b34cc8cd 100644 --- a/consensus/types/src/light_client_bootstrap.rs +++ b/consensus/types/src/light_client_bootstrap.rs @@ -1,12 +1,12 @@ +use crate::context_deserialize; use crate::{ - light_client_update::*, test_utils::TestRandom, BeaconState, ChainSpec, EthSpec, FixedVector, - ForkName, ForkVersionDeserialize, Hash256, LightClientHeader, LightClientHeaderAltair, + light_client_update::*, test_utils::TestRandom, BeaconState, ChainSpec, ContextDeserialize, + EthSpec, FixedVector, ForkName, Hash256, LightClientHeader, LightClientHeaderAltair, LightClientHeaderCapella, LightClientHeaderDeneb, LightClientHeaderElectra, LightClientHeaderFulu, SignedBlindedBeaconBlock, Slot, SyncCommittee, }; use derivative::Derivative; use serde::{Deserialize, Deserializer, Serialize}; -use serde_json::Value; use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; use std::sync::Arc; @@ -34,6 +34,7 @@ use tree_hash_derive::TreeHash; ), serde(bound = "E: EthSpec", deny_unknown_fields), arbitrary(bound = "E: EthSpec"), + context_deserialize(ForkName), ) )] #[derive( @@ -213,20 +214,40 @@ impl LightClientBootstrap { } } -impl ForkVersionDeserialize for LightClientBootstrap { - fn deserialize_by_fork<'de, D: Deserializer<'de>>( - value: Value, - fork_name: ForkName, - ) -> Result { - if fork_name.altair_enabled() { - Ok(serde_json::from_value::>(value) - .map_err(serde::de::Error::custom))? - } else { - Err(serde::de::Error::custom(format!( - "LightClientBootstrap failed to deserialize: unsupported fork '{}'", - fork_name - ))) - } +impl<'de, E: EthSpec> ContextDeserialize<'de, ForkName> for LightClientBootstrap { + fn context_deserialize(deserializer: D, context: ForkName) -> Result + where + D: Deserializer<'de>, + { + let convert_err = |e| { + serde::de::Error::custom(format!( + "LightClientBootstrap failed to deserialize: {:?}", + e + )) + }; + Ok(match context { + ForkName::Base => { + return Err(serde::de::Error::custom(format!( + "LightClientBootstrap failed to deserialize: unsupported fork '{}'", + context + ))) + } + ForkName::Altair | ForkName::Bellatrix => { + Self::Altair(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Capella => { + Self::Capella(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Deneb => { + Self::Deneb(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Electra => { + Self::Electra(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Fulu => { + Self::Fulu(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + }) } } diff --git a/consensus/types/src/light_client_finality_update.rs b/consensus/types/src/light_client_finality_update.rs index ee3b53c853e..9189dcd0a0b 100644 --- a/consensus/types/src/light_client_finality_update.rs +++ b/consensus/types/src/light_client_finality_update.rs @@ -1,13 +1,13 @@ use super::{EthSpec, FixedVector, Hash256, LightClientHeader, Slot, SyncAggregate}; +use crate::context_deserialize; use crate::ChainSpec; use crate::{ - light_client_update::*, test_utils::TestRandom, ForkName, ForkVersionDeserialize, + light_client_update::*, test_utils::TestRandom, ContextDeserialize, ForkName, LightClientHeaderAltair, LightClientHeaderCapella, LightClientHeaderDeneb, LightClientHeaderElectra, LightClientHeaderFulu, SignedBlindedBeaconBlock, }; use derivative::Derivative; use serde::{Deserialize, Deserializer, Serialize}; -use serde_json::Value; use ssz::{Decode, Encode}; use ssz_derive::Decode; use ssz_derive::Encode; @@ -33,11 +33,10 @@ use tree_hash_derive::TreeHash; ), serde(bound = "E: EthSpec", deny_unknown_fields), arbitrary(bound = "E: EthSpec"), + context_deserialize(ForkName), ) )] -#[derive( - Debug, Clone, Serialize, Encode, TreeHash, Deserialize, arbitrary::Arbitrary, PartialEq, -)] +#[derive(Debug, Clone, Serialize, Encode, TreeHash, arbitrary::Arbitrary, PartialEq)] #[serde(untagged)] #[tree_hash(enum_behaviour = "transparent")] #[ssz(enum_behaviour = "transparent")] @@ -233,20 +232,40 @@ impl LightClientFinalityUpdate { } } -impl ForkVersionDeserialize for LightClientFinalityUpdate { - fn deserialize_by_fork<'de, D: Deserializer<'de>>( - value: Value, - fork_name: ForkName, - ) -> Result { - if fork_name.altair_enabled() { - serde_json::from_value::>(value) - .map_err(serde::de::Error::custom) - } else { - Err(serde::de::Error::custom(format!( - "LightClientFinalityUpdate failed to deserialize: unsupported fork '{}'", - fork_name - ))) - } +impl<'de, E: EthSpec> ContextDeserialize<'de, ForkName> for LightClientFinalityUpdate { + fn context_deserialize(deserializer: D, context: ForkName) -> Result + where + D: Deserializer<'de>, + { + let convert_err = |e| { + serde::de::Error::custom(format!( + "LightClientFinalityUpdate failed to deserialize: {:?}", + e + )) + }; + Ok(match context { + ForkName::Base => { + return Err(serde::de::Error::custom(format!( + "LightClientFinalityUpdate failed to deserialize: unsupported fork '{}'", + context + ))) + } + ForkName::Altair | ForkName::Bellatrix => { + Self::Altair(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Capella => { + Self::Capella(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Deneb => { + Self::Deneb(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Electra => { + Self::Electra(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Fulu => { + Self::Fulu(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + }) } } diff --git a/consensus/types/src/light_client_header.rs b/consensus/types/src/light_client_header.rs index 0be26a70360..36f2932ecd5 100644 --- a/consensus/types/src/light_client_header.rs +++ b/consensus/types/src/light_client_header.rs @@ -1,6 +1,5 @@ +use crate::context_deserialize; use crate::ChainSpec; -use crate::ForkName; -use crate::ForkVersionDeserialize; use crate::{light_client_update::*, BeaconBlockBody}; use crate::{ test_utils::TestRandom, EthSpec, ExecutionPayloadHeaderCapella, ExecutionPayloadHeaderDeneb, @@ -8,8 +7,9 @@ use crate::{ SignedBlindedBeaconBlock, }; use crate::{BeaconBlockHeader, ExecutionPayloadHeader}; +use crate::{ContextDeserialize, ForkName}; use derivative::Derivative; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; use ssz::Decode; use ssz_derive::{Decode, Encode}; use std::marker::PhantomData; @@ -35,11 +35,10 @@ use tree_hash_derive::TreeHash; ), serde(bound = "E: EthSpec", deny_unknown_fields), arbitrary(bound = "E: EthSpec"), + context_deserialize(ForkName), ) )] -#[derive( - Debug, Clone, Serialize, TreeHash, Encode, Deserialize, arbitrary::Arbitrary, PartialEq, -)] +#[derive(Debug, Clone, Serialize, TreeHash, Encode, arbitrary::Arbitrary, PartialEq)] #[serde(untagged)] #[tree_hash(enum_behaviour = "transparent")] #[ssz(enum_behaviour = "transparent")] @@ -334,31 +333,40 @@ impl Default for LightClientHeaderFulu { } } -impl ForkVersionDeserialize for LightClientHeader { - fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( - value: serde_json::value::Value, - fork_name: ForkName, - ) -> Result { - match fork_name { - ForkName::Altair | ForkName::Bellatrix => serde_json::from_value(value) - .map(|light_client_header| Self::Altair(light_client_header)) - .map_err(serde::de::Error::custom), - ForkName::Capella => serde_json::from_value(value) - .map(|light_client_header| Self::Capella(light_client_header)) - .map_err(serde::de::Error::custom), - ForkName::Deneb => serde_json::from_value(value) - .map(|light_client_header| Self::Deneb(light_client_header)) - .map_err(serde::de::Error::custom), - ForkName::Electra => serde_json::from_value(value) - .map(|light_client_header| Self::Electra(light_client_header)) - .map_err(serde::de::Error::custom), - ForkName::Fulu => serde_json::from_value(value) - .map(|light_client_header| Self::Fulu(light_client_header)) - .map_err(serde::de::Error::custom), - ForkName::Base => Err(serde::de::Error::custom(format!( - "LightClientHeader deserialization for {fork_name} not implemented" - ))), - } +impl<'de, E: EthSpec> ContextDeserialize<'de, ForkName> for LightClientHeader { + fn context_deserialize(deserializer: D, context: ForkName) -> Result + where + D: Deserializer<'de>, + { + let convert_err = |e| { + serde::de::Error::custom(format!( + "LightClientFinalityUpdate failed to deserialize: {:?}", + e + )) + }; + Ok(match context { + ForkName::Base => { + return Err(serde::de::Error::custom(format!( + "LightClientFinalityUpdate failed to deserialize: unsupported fork '{}'", + context + ))) + } + ForkName::Altair | ForkName::Bellatrix => { + Self::Altair(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Capella => { + Self::Capella(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Deneb => { + Self::Deneb(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Electra => { + Self::Electra(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Fulu => { + Self::Fulu(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + }) } } diff --git a/consensus/types/src/light_client_optimistic_update.rs b/consensus/types/src/light_client_optimistic_update.rs index fcf357757b1..5701ebd8751 100644 --- a/consensus/types/src/light_client_optimistic_update.rs +++ b/consensus/types/src/light_client_optimistic_update.rs @@ -1,4 +1,5 @@ -use super::{EthSpec, ForkName, ForkVersionDeserialize, LightClientHeader, Slot, SyncAggregate}; +use super::{ContextDeserialize, EthSpec, ForkName, LightClientHeader, Slot, SyncAggregate}; +use crate::context_deserialize; use crate::test_utils::TestRandom; use crate::{ light_client_update::*, ChainSpec, LightClientHeaderAltair, LightClientHeaderCapella, @@ -7,7 +8,6 @@ use crate::{ }; use derivative::Derivative; use serde::{Deserialize, Deserializer, Serialize}; -use serde_json::Value; use ssz::{Decode, Encode}; use ssz_derive::Decode; use ssz_derive::Encode; @@ -36,11 +36,10 @@ use tree_hash_derive::TreeHash; ), serde(bound = "E: EthSpec", deny_unknown_fields), arbitrary(bound = "E: EthSpec"), + context_deserialize(ForkName), ) )] -#[derive( - Debug, Clone, Serialize, Encode, TreeHash, Deserialize, arbitrary::Arbitrary, PartialEq, -)] +#[derive(Debug, Clone, Serialize, Encode, TreeHash, arbitrary::Arbitrary, PartialEq)] #[serde(untagged)] #[tree_hash(enum_behaviour = "transparent")] #[ssz(enum_behaviour = "transparent")] @@ -206,22 +205,40 @@ impl LightClientOptimisticUpdate { } } -impl ForkVersionDeserialize for LightClientOptimisticUpdate { - fn deserialize_by_fork<'de, D: Deserializer<'de>>( - value: Value, - fork_name: ForkName, - ) -> Result { - if fork_name.altair_enabled() { - Ok( - serde_json::from_value::>(value) - .map_err(serde::de::Error::custom), - )? - } else { - Err(serde::de::Error::custom(format!( - "LightClientOptimisticUpdate failed to deserialize: unsupported fork '{}'", - fork_name - ))) - } +impl<'de, E: EthSpec> ContextDeserialize<'de, ForkName> for LightClientOptimisticUpdate { + fn context_deserialize(deserializer: D, context: ForkName) -> Result + where + D: Deserializer<'de>, + { + let convert_err = |e| { + serde::de::Error::custom(format!( + "LightClientOptimisticUpdate failed to deserialize: {:?}", + e + )) + }; + Ok(match context { + ForkName::Base => { + return Err(serde::de::Error::custom(format!( + "LightClientOptimisticUpdate failed to deserialize: unsupported fork '{}'", + context + ))) + } + ForkName::Altair | ForkName::Bellatrix => { + Self::Altair(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Capella => { + Self::Capella(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Deneb => { + Self::Deneb(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Electra => { + Self::Electra(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Fulu => { + Self::Fulu(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + }) } } diff --git a/consensus/types/src/light_client_update.rs b/consensus/types/src/light_client_update.rs index e1fce479757..92aeeb33bb9 100644 --- a/consensus/types/src/light_client_update.rs +++ b/consensus/types/src/light_client_update.rs @@ -1,8 +1,9 @@ use super::{EthSpec, FixedVector, Hash256, Slot, SyncAggregate, SyncCommittee}; +use crate::context_deserialize; use crate::light_client_header::LightClientHeaderElectra; use crate::LightClientHeader; use crate::{ - beacon_state, test_utils::TestRandom, ChainSpec, Epoch, ForkName, ForkVersionDeserialize, + beacon_state, test_utils::TestRandom, ChainSpec, ContextDeserialize, Epoch, ForkName, LightClientHeaderAltair, LightClientHeaderCapella, LightClientHeaderDeneb, LightClientHeaderFulu, SignedBlindedBeaconBlock, }; @@ -10,7 +11,6 @@ use derivative::Derivative; use safe_arith::ArithError; use safe_arith::SafeArith; use serde::{Deserialize, Deserializer, Serialize}; -use serde_json::Value; use ssz::{Decode, Encode}; use ssz_derive::Decode; use ssz_derive::Encode; @@ -117,11 +117,10 @@ impl From for Error { ), serde(bound = "E: EthSpec", deny_unknown_fields), arbitrary(bound = "E: EthSpec"), + context_deserialize(ForkName), ) )] -#[derive( - Debug, Clone, Serialize, Encode, TreeHash, Deserialize, arbitrary::Arbitrary, PartialEq, -)] +#[derive(Debug, Clone, Serialize, Encode, TreeHash, arbitrary::Arbitrary, PartialEq)] #[serde(untagged)] #[tree_hash(enum_behaviour = "transparent")] #[ssz(enum_behaviour = "transparent")] @@ -180,19 +179,37 @@ pub struct LightClientUpdate { pub signature_slot: Slot, } -impl ForkVersionDeserialize for LightClientUpdate { - fn deserialize_by_fork<'de, D: Deserializer<'de>>( - value: Value, - fork_name: ForkName, - ) -> Result { - match fork_name { - ForkName::Base => Err(serde::de::Error::custom(format!( - "LightClientUpdate failed to deserialize: unsupported fork '{}'", - fork_name - ))), - _ => Ok(serde_json::from_value::>(value) - .map_err(serde::de::Error::custom))?, - } +impl<'de, E: EthSpec> ContextDeserialize<'de, ForkName> for LightClientUpdate { + fn context_deserialize(deserializer: D, context: ForkName) -> Result + where + D: Deserializer<'de>, + { + let convert_err = |e| { + serde::de::Error::custom(format!("LightClientUpdate failed to deserialize: {:?}", e)) + }; + Ok(match context { + ForkName::Base => { + return Err(serde::de::Error::custom(format!( + "LightClientUpdate failed to deserialize: unsupported fork '{}'", + context + ))) + } + ForkName::Altair | ForkName::Bellatrix => { + Self::Altair(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Capella => { + Self::Capella(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Deneb => { + Self::Deneb(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Electra => { + Self::Electra(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + ForkName::Fulu => { + Self::Fulu(Deserialize::deserialize(deserializer).map_err(convert_err)?) + } + }) } } diff --git a/consensus/types/src/pending_attestation.rs b/consensus/types/src/pending_attestation.rs index 0bccab50795..b7b4a19f4bb 100644 --- a/consensus/types/src/pending_attestation.rs +++ b/consensus/types/src/pending_attestation.rs @@ -1,6 +1,6 @@ +use crate::context_deserialize; use crate::test_utils::TestRandom; -use crate::{AttestationData, BitList, EthSpec}; - +use crate::{AttestationData, BitList, EthSpec, ForkName}; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; @@ -22,6 +22,7 @@ use tree_hash_derive::TreeHash; arbitrary::Arbitrary, )] #[arbitrary(bound = "E: EthSpec")] +#[context_deserialize(ForkName)] pub struct PendingAttestation { pub aggregation_bits: BitList, pub data: AttestationData, diff --git a/consensus/types/src/pending_consolidation.rs b/consensus/types/src/pending_consolidation.rs index 6e0b74a7383..9a513f2744a 100644 --- a/consensus/types/src/pending_consolidation.rs +++ b/consensus/types/src/pending_consolidation.rs @@ -1,4 +1,6 @@ +use crate::context_deserialize; use crate::test_utils::TestRandom; +use crate::ForkName; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; @@ -18,6 +20,7 @@ use tree_hash_derive::TreeHash; TreeHash, TestRandom, )] +#[context_deserialize(ForkName)] pub struct PendingConsolidation { #[serde(with = "serde_utils::quoted_u64")] pub source_index: u64, diff --git a/consensus/types/src/pending_deposit.rs b/consensus/types/src/pending_deposit.rs index 3bee86417de..970c3264677 100644 --- a/consensus/types/src/pending_deposit.rs +++ b/consensus/types/src/pending_deposit.rs @@ -18,6 +18,7 @@ use tree_hash_derive::TreeHash; TreeHash, TestRandom, )] +#[context_deserialize(ForkName)] pub struct PendingDeposit { pub pubkey: PublicKeyBytes, pub withdrawal_credentials: Hash256, diff --git a/consensus/types/src/pending_partial_withdrawal.rs b/consensus/types/src/pending_partial_withdrawal.rs index 846dd973602..ca490328592 100644 --- a/consensus/types/src/pending_partial_withdrawal.rs +++ b/consensus/types/src/pending_partial_withdrawal.rs @@ -1,5 +1,6 @@ +use crate::context_deserialize; use crate::test_utils::TestRandom; -use crate::Epoch; +use crate::{Epoch, ForkName}; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; @@ -19,6 +20,7 @@ use tree_hash_derive::TreeHash; TreeHash, TestRandom, )] +#[context_deserialize(ForkName)] pub struct PendingPartialWithdrawal { #[serde(with = "serde_utils::quoted_u64")] pub validator_index: u64, diff --git a/consensus/types/src/proposer_slashing.rs b/consensus/types/src/proposer_slashing.rs index ee55d62c201..7b03dbb83ed 100644 --- a/consensus/types/src/proposer_slashing.rs +++ b/consensus/types/src/proposer_slashing.rs @@ -1,5 +1,6 @@ +use crate::context_deserialize; use crate::test_utils::TestRandom; -use crate::SignedBeaconBlockHeader; +use crate::{ForkName, SignedBeaconBlockHeader}; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; @@ -23,6 +24,7 @@ use tree_hash_derive::TreeHash; TreeHash, TestRandom, )] +#[context_deserialize(ForkName)] pub struct ProposerSlashing { pub signed_header_1: SignedBeaconBlockHeader, pub signed_header_2: SignedBeaconBlockHeader, diff --git a/consensus/types/src/runtime_var_list.rs b/consensus/types/src/runtime_var_list.rs index d6b1c10e993..454c8b9e188 100644 --- a/consensus/types/src/runtime_var_list.rs +++ b/consensus/types/src/runtime_var_list.rs @@ -1,5 +1,7 @@ +use crate::ContextDeserialize; use derivative::Derivative; -use serde::{Deserialize, Serialize}; +use serde::de::Error as DeError; +use serde::{Deserialize, Deserializer, Serialize}; use ssz::Decode; use ssz_types::Error; use std::ops::{Deref, Index, IndexMut}; @@ -217,6 +219,28 @@ where } } +impl<'de, C, T> ContextDeserialize<'de, (C, usize)> for RuntimeVariableList +where + T: ContextDeserialize<'de, C>, + C: Clone, +{ + fn context_deserialize(deserializer: D, context: (C, usize)) -> Result + where + D: Deserializer<'de>, + { + // first parse out a Vec using the Vec impl you already have + let vec: Vec = Vec::context_deserialize(deserializer, context.0)?; + if vec.len() > context.1 { + return Err(DeError::custom(format!( + "RuntimeVariableList lengh {} exceeds max_len {}", + vec.len(), + context.1 + ))); + } + Ok(RuntimeVariableList::from_vec(vec, context.1)) + } +} + #[cfg(test)] mod test { use super::*; diff --git a/consensus/types/src/signed_aggregate_and_proof.rs b/consensus/types/src/signed_aggregate_and_proof.rs index 26eca19bf15..7b1f97e5218 100644 --- a/consensus/types/src/signed_aggregate_and_proof.rs +++ b/consensus/types/src/signed_aggregate_and_proof.rs @@ -2,11 +2,11 @@ use super::{ AggregateAndProof, AggregateAndProofBase, AggregateAndProofElectra, AggregateAndProofRef, }; use super::{ - AttestationRef, ChainSpec, Domain, EthSpec, Fork, Hash256, SecretKey, SelectionProof, - Signature, SignedRoot, + Attestation, AttestationRef, ChainSpec, Domain, EthSpec, Fork, ForkName, Hash256, SecretKey, + SelectionProof, Signature, SignedRoot, }; +use crate::context_deserialize; use crate::test_utils::TestRandom; -use crate::Attestation; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use superstruct::superstruct; @@ -32,6 +32,7 @@ use tree_hash_derive::TreeHash; TestRandom, TreeHash, ), + context_deserialize(ForkName), serde(bound = "E: EthSpec"), arbitrary(bound = "E: EthSpec"), ), diff --git a/consensus/types/src/signed_beacon_block.rs b/consensus/types/src/signed_beacon_block.rs index 6379d73ec0d..85bed35a19c 100644 --- a/consensus/types/src/signed_beacon_block.rs +++ b/consensus/types/src/signed_beacon_block.rs @@ -3,7 +3,7 @@ use crate::test_utils::TestRandom; use crate::*; use derivative::Derivative; use merkle_proof::MerkleTree; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; use ssz_derive::{Decode, Encode}; use std::fmt; use superstruct::superstruct; @@ -703,20 +703,17 @@ impl SignedBeaconBlock { } } -impl> ForkVersionDeserialize +impl<'de, E: EthSpec, Payload: AbstractExecPayload> ContextDeserialize<'de, ForkName> for SignedBeaconBlock { - fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( - value: serde_json::value::Value, - fork_name: ForkName, - ) -> Result { + fn context_deserialize(deserializer: D, context: ForkName) -> Result + where + D: Deserializer<'de>, + { Ok(map_fork_name!( - fork_name, + context, Self, - serde_json::from_value(value).map_err(|e| serde::de::Error::custom(format!( - "SignedBeaconBlock failed to deserialize: {:?}", - e - )))? + serde::Deserialize::deserialize(deserializer)? )) } } diff --git a/consensus/types/src/signed_beacon_block_header.rs b/consensus/types/src/signed_beacon_block_header.rs index 3d4269a2cef..9106fa83724 100644 --- a/consensus/types/src/signed_beacon_block_header.rs +++ b/consensus/types/src/signed_beacon_block_header.rs @@ -1,5 +1,6 @@ +use crate::context_deserialize; use crate::{ - test_utils::TestRandom, BeaconBlockHeader, ChainSpec, Domain, EthSpec, Fork, Hash256, + test_utils::TestRandom, BeaconBlockHeader, ChainSpec, Domain, EthSpec, Fork, ForkName, Hash256, PublicKey, Signature, SignedRoot, }; use serde::{Deserialize, Serialize}; @@ -24,6 +25,7 @@ use tree_hash_derive::TreeHash; TreeHash, TestRandom, )] +#[context_deserialize(ForkName)] pub struct SignedBeaconBlockHeader { pub message: BeaconBlockHeader, pub signature: Signature, diff --git a/consensus/types/src/signed_bls_to_execution_change.rs b/consensus/types/src/signed_bls_to_execution_change.rs index a7bfd7c2710..383663e36b8 100644 --- a/consensus/types/src/signed_bls_to_execution_change.rs +++ b/consensus/types/src/signed_bls_to_execution_change.rs @@ -19,6 +19,7 @@ use tree_hash_derive::TreeHash; TreeHash, TestRandom, )] +#[context_deserialize(ForkName)] pub struct SignedBlsToExecutionChange { pub message: BlsToExecutionChange, pub signature: Signature, diff --git a/consensus/types/src/signed_contribution_and_proof.rs b/consensus/types/src/signed_contribution_and_proof.rs index 068fd980ae6..42115bfbc06 100644 --- a/consensus/types/src/signed_contribution_and_proof.rs +++ b/consensus/types/src/signed_contribution_and_proof.rs @@ -1,7 +1,8 @@ use super::{ - ChainSpec, ContributionAndProof, Domain, EthSpec, Fork, Hash256, SecretKey, Signature, - SignedRoot, SyncCommitteeContribution, SyncSelectionProof, + ChainSpec, ContributionAndProof, Domain, EthSpec, Fork, ForkName, Hash256, SecretKey, + Signature, SignedRoot, SyncCommitteeContribution, SyncSelectionProof, }; +use crate::context_deserialize; use crate::test_utils::TestRandom; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; @@ -24,6 +25,7 @@ use tree_hash_derive::TreeHash; )] #[serde(bound = "E: EthSpec")] #[arbitrary(bound = "E: EthSpec")] +#[context_deserialize(ForkName)] pub struct SignedContributionAndProof { /// The `ContributionAndProof` that was signed. pub message: ContributionAndProof, diff --git a/consensus/types/src/signed_voluntary_exit.rs b/consensus/types/src/signed_voluntary_exit.rs index 30eda117919..b6451d3ab5e 100644 --- a/consensus/types/src/signed_voluntary_exit.rs +++ b/consensus/types/src/signed_voluntary_exit.rs @@ -1,4 +1,5 @@ -use crate::{test_utils::TestRandom, VoluntaryExit}; +use crate::context_deserialize; +use crate::{test_utils::TestRandom, ForkName, VoluntaryExit}; use bls::Signature; use serde::{Deserialize, Serialize}; @@ -22,6 +23,7 @@ use tree_hash_derive::TreeHash; TreeHash, TestRandom, )] +#[context_deserialize(ForkName)] pub struct SignedVoluntaryExit { pub message: VoluntaryExit, pub signature: Signature, diff --git a/consensus/types/src/signing_data.rs b/consensus/types/src/signing_data.rs index f30d5fdfcb4..aa25ecffd92 100644 --- a/consensus/types/src/signing_data.rs +++ b/consensus/types/src/signing_data.rs @@ -1,5 +1,6 @@ +use crate::context_deserialize; use crate::test_utils::TestRandom; -use crate::Hash256; +use crate::{ForkName, Hash256}; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; @@ -19,6 +20,7 @@ use tree_hash_derive::TreeHash; TreeHash, TestRandom, )] +#[context_deserialize(ForkName)] pub struct SigningData { pub object_root: Hash256, pub domain: Hash256, diff --git a/consensus/types/src/sync_aggregate.rs b/consensus/types/src/sync_aggregate.rs index 12b91501ae0..4f810db22ae 100644 --- a/consensus/types/src/sync_aggregate.rs +++ b/consensus/types/src/sync_aggregate.rs @@ -1,6 +1,7 @@ use crate::consts::altair::SYNC_COMMITTEE_SUBNET_COUNT; +use crate::context_deserialize; use crate::test_utils::TestRandom; -use crate::{AggregateSignature, BitVector, EthSpec, SyncCommitteeContribution}; +use crate::{AggregateSignature, BitVector, EthSpec, ForkName, SyncCommitteeContribution}; use derivative::Derivative; use safe_arith::{ArithError, SafeArith}; use serde::{Deserialize, Serialize}; @@ -36,6 +37,7 @@ impl From for Error { #[derivative(PartialEq, Hash(bound = "E: EthSpec"))] #[serde(bound = "E: EthSpec")] #[arbitrary(bound = "E: EthSpec")] +#[context_deserialize(ForkName)] pub struct SyncAggregate { pub sync_committee_bits: BitVector, pub sync_committee_signature: AggregateSignature, diff --git a/consensus/types/src/sync_aggregator_selection_data.rs b/consensus/types/src/sync_aggregator_selection_data.rs index 3da130bb068..a61cd47d04a 100644 --- a/consensus/types/src/sync_aggregator_selection_data.rs +++ b/consensus/types/src/sync_aggregator_selection_data.rs @@ -1,6 +1,6 @@ +use crate::context_deserialize; use crate::test_utils::TestRandom; -use crate::{SignedRoot, Slot}; - +use crate::{ForkName, SignedRoot, Slot}; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; @@ -19,6 +19,7 @@ use tree_hash_derive::TreeHash; TreeHash, TestRandom, )] +#[context_deserialize(ForkName)] pub struct SyncAggregatorSelectionData { pub slot: Slot, #[serde(with = "serde_utils::quoted_u64")] diff --git a/consensus/types/src/sync_committee.rs b/consensus/types/src/sync_committee.rs index 032f0d61f9f..c7ec7bdcc3f 100644 --- a/consensus/types/src/sync_committee.rs +++ b/consensus/types/src/sync_committee.rs @@ -1,5 +1,6 @@ +use crate::context_deserialize; use crate::test_utils::TestRandom; -use crate::{EthSpec, FixedVector, SyncSubnetId}; +use crate::{EthSpec, FixedVector, ForkName, SyncSubnetId}; use bls::PublicKeyBytes; use safe_arith::{ArithError, SafeArith}; use serde::{Deserialize, Serialize}; @@ -38,6 +39,7 @@ impl From for Error { )] #[serde(bound = "E: EthSpec")] #[arbitrary(bound = "E: EthSpec")] +#[context_deserialize(ForkName)] pub struct SyncCommittee { pub pubkeys: FixedVector, pub aggregate_pubkey: PublicKeyBytes, diff --git a/consensus/types/src/sync_committee_contribution.rs b/consensus/types/src/sync_committee_contribution.rs index e160332f455..e2ac414cfa7 100644 --- a/consensus/types/src/sync_committee_contribution.rs +++ b/consensus/types/src/sync_committee_contribution.rs @@ -1,4 +1,5 @@ -use super::{AggregateSignature, EthSpec, SignedRoot}; +use super::{AggregateSignature, EthSpec, ForkName, SignedRoot}; +use crate::context_deserialize; use crate::slot_data::SlotData; use crate::{test_utils::TestRandom, BitVector, Hash256, Slot, SyncCommitteeMessage}; use serde::{Deserialize, Serialize}; @@ -28,6 +29,7 @@ pub enum Error { )] #[serde(bound = "E: EthSpec")] #[arbitrary(bound = "E: EthSpec")] +#[context_deserialize(ForkName)] pub struct SyncCommitteeContribution { pub slot: Slot, pub beacon_block_root: Hash256, diff --git a/consensus/types/src/sync_committee_message.rs b/consensus/types/src/sync_committee_message.rs index d7d309cd567..4b442b30539 100644 --- a/consensus/types/src/sync_committee_message.rs +++ b/consensus/types/src/sync_committee_message.rs @@ -1,7 +1,9 @@ -use crate::test_utils::TestRandom; -use crate::{ChainSpec, Domain, EthSpec, Fork, Hash256, SecretKey, Signature, SignedRoot, Slot}; - +use crate::context_deserialize; use crate::slot_data::SlotData; +use crate::test_utils::TestRandom; +use crate::{ + ChainSpec, Domain, EthSpec, Fork, ForkName, Hash256, SecretKey, Signature, SignedRoot, Slot, +}; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; @@ -20,6 +22,7 @@ use tree_hash_derive::TreeHash; TreeHash, TestRandom, )] +#[context_deserialize(ForkName)] pub struct SyncCommitteeMessage { pub slot: Slot, pub beacon_block_root: Hash256, diff --git a/consensus/types/src/validator.rs b/consensus/types/src/validator.rs index 027958b178c..165f477ff46 100644 --- a/consensus/types/src/validator.rs +++ b/consensus/types/src/validator.rs @@ -1,3 +1,4 @@ +use crate::context_deserialize; use crate::{ test_utils::TestRandom, Address, BeaconState, ChainSpec, Checkpoint, Epoch, EthSpec, FixedBytesExtended, ForkName, Hash256, PublicKeyBytes, @@ -23,6 +24,7 @@ use tree_hash_derive::TreeHash; TestRandom, TreeHash, )] +#[context_deserialize(ForkName)] pub struct Validator { pub pubkey: PublicKeyBytes, pub withdrawal_credentials: Hash256, diff --git a/consensus/types/src/voluntary_exit.rs b/consensus/types/src/voluntary_exit.rs index 787bf234d88..75260add4b5 100644 --- a/consensus/types/src/voluntary_exit.rs +++ b/consensus/types/src/voluntary_exit.rs @@ -1,3 +1,4 @@ +use crate::context_deserialize; use crate::{ test_utils::TestRandom, ChainSpec, Domain, Epoch, ForkName, Hash256, SecretKey, SignedRoot, SignedVoluntaryExit, @@ -24,6 +25,7 @@ use tree_hash_derive::TreeHash; TreeHash, TestRandom, )] +#[context_deserialize(ForkName)] pub struct VoluntaryExit { /// Earliest epoch when voluntary exit can be processed. pub epoch: Epoch, diff --git a/consensus/types/src/withdrawal.rs b/consensus/types/src/withdrawal.rs index 7f98ff1e60a..9ca50fccfbd 100644 --- a/consensus/types/src/withdrawal.rs +++ b/consensus/types/src/withdrawal.rs @@ -19,6 +19,7 @@ use tree_hash_derive::TreeHash; TreeHash, TestRandom, )] +#[context_deserialize(ForkName)] pub struct Withdrawal { #[serde(with = "serde_utils::quoted_u64")] pub index: u64, diff --git a/consensus/types/src/withdrawal_request.rs b/consensus/types/src/withdrawal_request.rs index 1296426ac05..57c6e798eb4 100644 --- a/consensus/types/src/withdrawal_request.rs +++ b/consensus/types/src/withdrawal_request.rs @@ -1,5 +1,6 @@ +use crate::context_deserialize; use crate::test_utils::TestRandom; -use crate::{Address, PublicKeyBytes}; +use crate::{Address, ForkName, PublicKeyBytes}; use serde::{Deserialize, Serialize}; use ssz::Encode; use ssz_derive::{Decode, Encode}; @@ -20,6 +21,7 @@ use tree_hash_derive::TreeHash; TreeHash, TestRandom, )] +#[context_deserialize(ForkName)] pub struct WithdrawalRequest { #[serde(with = "serde_utils::address_hex")] pub source_address: Address, diff --git a/lcli/src/block_root.rs b/lcli/src/block_root.rs index 80087fd6d4a..3c07d4f9ef0 100644 --- a/lcli/src/block_root.rs +++ b/lcli/src/block_root.rs @@ -79,7 +79,7 @@ pub fn run( .await .map_err(|e| format!("Failed to download block: {:?}", e))? .ok_or_else(|| format!("Unable to locate block at {:?}", block_id))? - .data; + .into_data(); Ok::<_, String>(block) }) .map_err(|e| format!("Failed to complete task: {:?}", e))? diff --git a/lcli/src/http_sync.rs b/lcli/src/http_sync.rs index 1ef40e63978..cb6a9d2b1d9 100644 --- a/lcli/src/http_sync.rs +++ b/lcli/src/http_sync.rs @@ -123,11 +123,11 @@ async fn get_block_from_source( .unwrap() .unwrap(); let blobs_from_source = source - .get_blobs::(block_id, None) + .get_blobs::(block_id, None, spec) .await .unwrap() .unwrap() - .data; + .into_data(); let (kzg_proofs, blobs): (Vec<_>, Vec<_>) = blobs_from_source .iter() diff --git a/lcli/src/skip_slots.rs b/lcli/src/skip_slots.rs index 834123e9391..9456f345703 100644 --- a/lcli/src/skip_slots.rs +++ b/lcli/src/skip_slots.rs @@ -102,7 +102,7 @@ pub fn run( }) .map_err(|e| format!("Failed to complete task: {:?}", e))? .ok_or_else(|| format!("Unable to locate state at {:?}", state_id))? - .data; + .into_data(); let state_root = match state_id { StateId::Root(root) => Some(root), _ => None, diff --git a/lcli/src/state_root.rs b/lcli/src/state_root.rs index b2308999d42..7b10ab9362f 100644 --- a/lcli/src/state_root.rs +++ b/lcli/src/state_root.rs @@ -50,7 +50,7 @@ pub fn run( }) .map_err(|e| format!("Failed to complete task: {:?}", e))? .ok_or_else(|| format!("Unable to locate state at {:?}", state_id))? - .data + .into_data() } _ => return Err("must supply either --state-path or --beacon-url".into()), }; diff --git a/lcli/src/transition_blocks.rs b/lcli/src/transition_blocks.rs index 4831f864913..2226105c341 100644 --- a/lcli/src/transition_blocks.rs +++ b/lcli/src/transition_blocks.rs @@ -154,7 +154,7 @@ pub fn run( .await .map_err(|e| format!("Failed to download block: {:?}", e))? .ok_or_else(|| format!("Unable to locate block at {:?}", block_id))? - .data; + .into_data(); if block.slot() == inner_spec.genesis_slot { return Err("Cannot run on the genesis block".to_string()); @@ -165,7 +165,7 @@ pub fn run( .await .map_err(|e| format!("Failed to download parent block: {:?}", e))? .ok_or_else(|| format!("Unable to locate parent block at {:?}", block_id))? - .data; + .into_data(); let state_root = parent_block.state_root(); let state_id = StateId::Root(state_root); @@ -174,7 +174,7 @@ pub fn run( .await .map_err(|e| format!("Failed to download state: {:?}", e))? .ok_or_else(|| format!("Unable to locate state at {:?}", state_id))? - .data; + .into_data(); Ok((pre_state, Some(state_root), block)) }) diff --git a/testing/ef_tests/src/cases/light_client_verify_is_better_update.rs b/testing/ef_tests/src/cases/light_client_verify_is_better_update.rs index de281d906c1..b2afc047c5c 100644 --- a/testing/ef_tests/src/cases/light_client_verify_is_better_update.rs +++ b/testing/ef_tests/src/cases/light_client_verify_is_better_update.rs @@ -3,8 +3,7 @@ use decode::ssz_decode_light_client_update; use serde::Deserialize; use types::{LightClientUpdate, Slot}; -#[derive(Debug, Clone, Deserialize)] -#[serde(deny_unknown_fields)] +#[derive(Debug, Clone)] pub struct LightClientVerifyIsBetterUpdate { light_client_updates: Vec>, } diff --git a/testing/simulator/src/checks.rs b/testing/simulator/src/checks.rs index cd0e2e726e5..1b2d4024d11 100644 --- a/testing/simulator/src/checks.rs +++ b/testing/simulator/src/checks.rs @@ -97,7 +97,7 @@ async fn verify_validator_count( let vc = remote_node .get_debug_beacon_states::(StateId::Head) .await - .map(|body| body.unwrap().data) + .map(|body| body.unwrap().into_data()) .map_err(|e| format!("Get state root via http failed: {:?}", e))? .validators() .len(); @@ -197,7 +197,7 @@ pub async fn verify_full_sync_aggregates_up_to( slot ) }) - .data + .data() .message() .body() .sync_aggregate() @@ -235,7 +235,7 @@ pub async fn verify_transition_block_finalized( let execution_block_hash: ExecutionBlockHash = remote_node .get_beacon_blocks::(BlockId::Finalized) .await - .map(|body| body.unwrap().data) + .map(|body| body.unwrap().into_data()) .map_err(|e| format!("Get state root via http failed: {:?}", e))? .message() .execution_payload() @@ -308,7 +308,7 @@ pub(crate) async fn verify_light_client_updates( .await .map_err(|e| format!("Error while getting light client updates: {:?}", e))? .ok_or(format!("Light client optimistic update not found {slot:?}"))? - .data + .data() .signature_slot(); let signature_slot_distance = slot - signature_slot; if signature_slot_distance > light_client_update_slot_tolerance { @@ -337,7 +337,7 @@ pub(crate) async fn verify_light_client_updates( .await .map_err(|e| format!("Error while getting light client updates: {:?}", e))? .ok_or(format!("Light client finality update not found {slot:?}"))? - .data + .data() .signature_slot(); let signature_slot_distance = slot - signature_slot; if signature_slot_distance > light_client_update_slot_tolerance { @@ -385,7 +385,7 @@ pub async fn ensure_node_synced_up_to_slot( .ok() .flatten() .ok_or(format!("No head block exists on node {node_index}"))? - .data; + .into_data(); // Check the head block is synced with the rest of the network. if head.slot() >= upto_slot { @@ -422,7 +422,7 @@ pub async fn verify_full_blob_production_up_to( // the `verify_full_block_production_up_to` function. if block.is_some() { remote_node - .get_blobs::(BlockId::Slot(Slot::new(slot)), None) + .get_blobs::(BlockId::Slot(Slot::new(slot)), None, &E::default_spec()) .await .map_err(|e| format!("Failed to get blobs at slot {slot:?}: {e:?}"))? .ok_or_else(|| format!("No blobs available at slot {slot:?}"))?; diff --git a/validator_client/beacon_node_fallback/src/lib.rs b/validator_client/beacon_node_fallback/src/lib.rs index 8d022f8e758..e11cc97e791 100644 --- a/validator_client/beacon_node_fallback/src/lib.rs +++ b/validator_client/beacon_node_fallback/src/lib.rs @@ -482,12 +482,26 @@ impl BeaconNodeFallback { for (result, node) in results { if let Err(e) = result { - if *e != CandidateError::PreGenesis { - warn!( - error = ?e, - endpoint = %node, - "A connected beacon node errored during routine health check" - ); + match e { + // Avoid spamming warns before genesis. + CandidateError::PreGenesis => {} + // Uninitialized *should* only occur during start-up before the + // slot clock has been initialized. + // Seeing this log in any other circumstance would indicate a serious bug. + CandidateError::Uninitialized => { + debug!( + error = ?e, + endpoint = %node, + "A connected beacon node is uninitialized" + ); + } + _ => { + warn!( + error = ?e, + endpoint = %node, + "A connected beacon node errored during routine health check" + ); + } } } } diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index c1e96a28087..f7765677065 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -569,7 +569,7 @@ impl AttestationService