diff --git a/crates/cluster/src/definition.rs b/crates/cluster/src/definition.rs index ae4e4633..60763062 100644 --- a/crates/cluster/src/definition.rs +++ b/crates/cluster/src/definition.rs @@ -857,9 +857,12 @@ pub struct ValidatorAddresses { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct DefinitionV1x0or1 { /// Human-readable cosmetic identifier. Max 256 chars. + #[serde(default)] pub name: String, /// Charon nodes in the cluster and their operators. /// Max 256 operators. + #[serde(default)] + #[serde_as(as = "DefaultOnNull")] pub operators: Vec, /// Human-readable random unique identifier. Max 64 chars. pub uuid: String, @@ -868,6 +871,7 @@ pub struct DefinitionV1x0or1 { /// Human-readable timestamp of this definition. Max 32 /// chars. Note that this was added in v1.1.0, so may be empty for older /// versions. + #[serde(default)] pub timestamp: String, /// Number of DVs to be created in the cluster lock /// file. @@ -877,9 +881,11 @@ pub struct DefinitionV1x0or1 { pub threshold: u64, /// Fee recipient address for the /// validator. + #[serde(default)] pub fee_recipient_address: String, /// Withdrawal address for the /// validator. + #[serde(default)] pub withdrawal_address: String, /// DKG algorithm to use for key generation. Max 32 chars. pub dkg_algorithm: String, @@ -970,9 +976,12 @@ impl TryFrom for Definition { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct DefinitionV1x2or3 { /// Human-readable cosmetic identifier. Max 256 chars. + #[serde(default)] pub name: String, /// Charon nodes in the cluster and their operators. /// Max 256 operators. + #[serde(default)] + #[serde_as(as = "DefaultOnNull")] pub operators: Vec, /// Human-readable random unique identifier. Max 64 chars. pub uuid: String, @@ -981,6 +990,7 @@ pub struct DefinitionV1x2or3 { /// Human-readable timestamp of this definition. Max 32 /// chars. Note that this was added in v1.1.0, so may be empty for older /// versions. + #[serde(default)] pub timestamp: String, /// Number of DVs to be created in the cluster lock /// file. @@ -990,9 +1000,11 @@ pub struct DefinitionV1x2or3 { pub threshold: u64, /// Fee recipient address for the /// validator. + #[serde(default)] pub fee_recipient_address: String, /// Withdrawal address for the /// validator. + #[serde(default)] pub withdrawal_address: String, /// DKGAlgorithm to use for key generation. Max 32 chars. pub dkg_algorithm: String, @@ -1083,12 +1095,15 @@ impl TryFrom for Definition { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct DefinitionV1x4 { /// Human-readable cosmetic identifier. Max 256 chars. + #[serde(default)] pub name: String, /// Creator identifies the creator of a cluster definition. They may also be /// an operator. pub creator: Creator, /// Operators define the charon nodes in the cluster and their operators. /// Max 256 operators. + #[serde(default)] + #[serde_as(as = "DefaultOnNull")] pub operators: Vec, /// Human-readable random unique identifier. Max 64 chars. pub uuid: String, @@ -1097,6 +1112,7 @@ pub struct DefinitionV1x4 { /// Human-readable timestamp of this definition. Max 32 /// chars. Note that this was added in v1.1.0, so may be empty for older /// versions. + #[serde(default)] pub timestamp: String, /// Number of DVs to be created in the cluster lock /// file. @@ -1202,12 +1218,15 @@ impl TryFrom for Definition { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct DefinitionV1x5to7 { /// Human-readable cosmetic identifier. Max 256 chars. + #[serde(default)] pub name: String, /// Creator identifies the creator of a cluster definition. They may also be /// an operator. pub creator: Creator, /// Charon nodes in the cluster and their operators. /// Max 256 operators. + #[serde(default)] + #[serde_as(as = "DefaultOnNull")] pub operators: Vec, /// Human-readable random unique identifier. Max 64 chars. pub uuid: String, @@ -1216,6 +1235,7 @@ pub struct DefinitionV1x5to7 { /// Human-readable timestamp of this definition. Max 32 /// chars. Note that this was added in v1.1.0, so may be empty for older /// versions. + #[serde(default)] pub timestamp: String, /// Number of DVs to be created in the cluster lock /// file. @@ -1224,7 +1244,8 @@ pub struct DefinitionV1x5to7 { /// for number of nodes/peers. pub threshold: u64, /// Addresses of each validator. - #[serde(rename = "validators")] + #[serde(rename = "validators", default)] + #[serde_as(as = "DefaultOnNull")] pub validator_addresses: Vec, /// DKG algorithm to use for key generation. Max 32 chars. pub dkg_algorithm: String, @@ -1301,12 +1322,18 @@ impl From for Definition { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct DefinitionV1x8 { /// Name is a human-readable cosmetic identifier. Max 256 chars. + // charon marshals `name` with `omitempty`, so it is absent when empty. + #[serde(default)] pub name: String, /// Creator identifies the creator of a cluster definition. They may also be /// an operator. pub creator: Creator, /// Operators define the charon nodes in the cluster and their operators. /// Max 256 operators. + // charon marshals a nil `operators` slice as JSON `null`, and older tools + // may omit the key entirely, so accept both. + #[serde(default)] + #[serde_as(as = "DefaultOnNull")] pub operators: Vec, /// UUID is a human-readable random unique identifier. Max 64 chars. pub uuid: String, @@ -1315,6 +1342,8 @@ pub struct DefinitionV1x8 { /// Timestamp is the human-readable timestamp of this definition. Max 32 /// chars. Note that this was added in v1.1.0, so may be empty for older /// versions. + // charon marshals `timestamp` with `omitempty`, so it is absent when empty. + #[serde(default)] pub timestamp: String, /// NumValidators is the number of DVs to be created in the cluster lock /// file. @@ -1323,7 +1352,10 @@ pub struct DefinitionV1x8 { /// for number of nodes/peers. pub threshold: u64, /// ValidatorAddresses define addresses of each validator. - #[serde(rename = "validators")] + // charon marshals a nil `validators` slice as JSON `null`, and older tools + // may omit the key entirely, so accept both. + #[serde(default, rename = "validators")] + #[serde_as(as = "DefaultOnNull")] pub validator_addresses: Vec, /// DKGAlgorithm to use for key generation. Max 32 chars. pub dkg_algorithm: String, @@ -1405,12 +1437,18 @@ impl From for Definition { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct DefinitionV1x9 { /// Name is a human-readable cosmetic identifier. Max 256 chars. + // charon marshals `name` with `omitempty`, so it is absent when empty. + #[serde(default)] pub name: String, /// Creator identifies the creator of a cluster definition. They may also be /// an operator. pub creator: Creator, /// Operators define the charon nodes in the cluster and their operators. /// Max 256 operators. + // charon marshals a nil `operators` slice as JSON `null`, and older tools + // may omit the key entirely, so accept both. + #[serde(default)] + #[serde_as(as = "DefaultOnNull")] pub operators: Vec, /// UUID is a human-readable random unique identifier. Max 64 chars. pub uuid: String, @@ -1419,6 +1457,8 @@ pub struct DefinitionV1x9 { /// Timestamp is the human-readable timestamp of this definition. Max 32 /// chars. Note that this was added in v1.1.0, so may be empty for older /// versions. + // charon marshals `timestamp` with `omitempty`, so it is absent when empty. + #[serde(default)] pub timestamp: String, /// NumValidators is the number of DVs to be created in the cluster lock /// file. @@ -1427,7 +1467,10 @@ pub struct DefinitionV1x9 { /// for number of nodes/peers. pub threshold: u64, /// ValidatorAddresses define addresses of each validator. - #[serde(rename = "validators")] + // charon marshals a nil `validators` slice as JSON `null`, and older tools + // may omit the key entirely, so accept both. + #[serde(default, rename = "validators")] + #[serde_as(as = "DefaultOnNull")] pub validator_addresses: Vec, /// DKGAlgorithm to use for key generation. Max 32 chars. pub dkg_algorithm: String, @@ -1520,6 +1563,10 @@ pub struct DefinitionV1x10 { pub creator: Creator, /// Charon nodes in the cluster and their operators. /// Max 256 operators. + // charon marshals a nil `operators` slice as JSON `null`, and older tools + // may omit the key entirely, so accept both. + #[serde(default)] + #[serde_as(as = "DefaultOnNull")] pub operators: Vec, /// Human-readable random unique identifier. Max 64 chars. pub uuid: String, @@ -1528,6 +1575,8 @@ pub struct DefinitionV1x10 { /// Human-readable timestamp of this definition. Max 32 /// chars. Note that this was added in v1.1.0, so may be empty for older /// versions. + // charon marshals `timestamp` with `omitempty`, so it is absent when empty. + #[serde(default)] pub timestamp: String, /// Number of DVs to be created in the cluster lock /// file. @@ -1536,7 +1585,10 @@ pub struct DefinitionV1x10 { /// for number of nodes/peers. pub threshold: u64, /// Addresses of each validator. - #[serde(rename = "validators")] + // charon marshals a nil `validators` slice as JSON `null`, and older tools + // may omit the key entirely, so accept both. + #[serde(default, rename = "validators")] + #[serde_as(as = "DefaultOnNull")] pub validator_addresses: Vec, /// DKG algorithm to use for key generation. Max 32 chars. pub dkg_algorithm: String, @@ -1985,6 +2037,41 @@ mod tests { assert!(definition.verify_hashes().is_ok()); } + /// charon marshals nil `operators`/`validators` slices as JSON `null` and + /// omits `name`/`timestamp` (both `omitempty`). Every supported definition + /// version must accept those shapes rather than fail deserialization. + #[test] + fn definition_accepts_null_slices_and_absent_omitempty_fields() { + for fixture in [ + include_str!("testdata/cluster_definition_v1_8_0.json"), + include_str!("testdata/cluster_definition_v1_9_0.json"), + include_str!("testdata/cluster_definition_v1_10_0.json"), + ] { + let mut value: serde_json::Value = serde_json::from_str(fixture).unwrap(); + let obj = value.as_object_mut().unwrap(); + obj.insert("operators".to_owned(), serde_json::Value::Null); + // charon only marshals `validators: null` when there are no + // validators, so keep the count consistent with the deserializer's + // num_validators/validators cross-check. + obj.insert("validators".to_owned(), serde_json::Value::Null); + obj.insert("num_validators".to_owned(), serde_json::json!(0)); + obj.remove("name"); + obj.remove("timestamp"); + + let version = value["version"].as_str().unwrap().to_owned(); + let definition = serde_json::from_value::(value) + .unwrap_or_else(|e| panic!("version {version} must parse: {e}")); + + assert!(definition.operators.is_empty(), "version {version}"); + assert!( + definition.validator_addresses.is_empty(), + "version {version}" + ); + assert_eq!(definition.name, "", "version {version}"); + assert_eq!(definition.timestamp, "", "version {version}"); + } + } + #[test] fn definition_empty_deposit_amounts_serialize_as_null() { let mut definition = Definition { diff --git a/crates/cluster/src/distvalidator.rs b/crates/cluster/src/distvalidator.rs index 78011aa4..00c8cf03 100644 --- a/crates/cluster/src/distvalidator.rs +++ b/crates/cluster/src/distvalidator.rs @@ -327,11 +327,16 @@ pub struct DistValidatorV1x8orLater { /// Public shares are the public keys corresponding to each node's secret /// key share. It can be used to verify a partial signature created by /// any node in the cluster. - #[serde(rename = "public_shares")] + // charon marshals `public_shares` with `omitempty`, so it is absent when + // empty. + #[serde(default, rename = "public_shares")] #[serde_as(as = "Vec")] pub pub_shares: Vec>, /// Deposit data defines the deposit data to activate a validator. + // charon marshals `partial_deposit_data` with `omitempty`, so it is absent + // when empty. + #[serde(default)] pub partial_deposit_data: Vec, /// Builder registration is the pre-generated signed validator builder @@ -360,3 +365,28 @@ impl From for DistValidator { } } } + +#[cfg(test)] +mod tests { + use super::*; + + /// charon marshals `public_shares` and `partial_deposit_data` with + /// `omitempty`, so both keys are absent when empty. Deserialization must + /// tolerate that rather than require the keys. + #[test] + fn dist_validator_v1x8_accepts_absent_omitempty_fields() { + let lock: serde_json::Value = + serde_json::from_str(include_str!("testdata/cluster_lock_v1_10_0.json")).unwrap(); + + let mut validator = lock["distributed_validators"][0].clone(); + let obj = validator.as_object_mut().unwrap(); + obj.remove("public_shares"); + obj.remove("partial_deposit_data"); + + let dist_validator: DistValidatorV1x8orLater = + serde_json::from_value(validator).expect("must parse without omitempty fields"); + + assert!(dist_validator.pub_shares.is_empty()); + assert!(dist_validator.partial_deposit_data.is_empty()); + } +} diff --git a/crates/cluster/src/lock.rs b/crates/cluster/src/lock.rs index 92da90e5..3fa70348 100644 --- a/crates/cluster/src/lock.rs +++ b/crates/cluster/src/lock.rs @@ -19,6 +19,7 @@ use crate::{ use pluto_eth2util::enr::{Record, RecordError}; use pluto_k1util::K1UtilError; use serde_with::{ + DefaultOnNull, base64::{Base64, Standard}, serde_as, }; @@ -445,7 +446,8 @@ pub struct LockV1x0or1 { pub definition: Definition, /// Validators are the distributed validators managed by the cluster. - #[serde(rename = "distributed_validators")] + #[serde(rename = "distributed_validators", default)] + #[serde_as(as = "DefaultOnNull")] pub distributed_validators: Vec, /// Lock hash uniquely identifies a cluster lock. @@ -500,7 +502,8 @@ pub struct LockV1x2to5 { pub definition: Definition, /// Validators are the distributed validators managed by the cluster. - #[serde(rename = "distributed_validators")] + #[serde(rename = "distributed_validators", default)] + #[serde_as(as = "DefaultOnNull")] pub distributed_validators: Vec, /// LockHash uniquely identifies a cluster lock. @@ -555,7 +558,8 @@ pub struct LockV1x6 { pub definition: Definition, /// Validators are the distributed validators managed by the cluster. - #[serde(rename = "distributed_validators")] + #[serde(rename = "distributed_validators", default)] + #[serde_as(as = "DefaultOnNull")] pub distributed_validators: Vec, /// Lock hash uniquely identifies a cluster lock. @@ -610,7 +614,8 @@ pub struct LockV1x7 { pub definition: Definition, /// Validators are the distributed validators managed by the cluster. - #[serde(rename = "distributed_validators")] + #[serde(rename = "distributed_validators", default)] + #[serde_as(as = "DefaultOnNull")] pub distributed_validators: Vec, /// Lock hash uniquely identifies a cluster lock. @@ -671,7 +676,8 @@ pub struct LockV1x8orLater { pub definition: Definition, /// Validators are the distributed validators managed by the cluster. - #[serde(rename = "distributed_validators")] + #[serde(rename = "distributed_validators", default)] + #[serde_as(as = "DefaultOnNull")] pub distributed_validators: Vec, /// Lock hash uniquely identifies a cluster lock. @@ -728,44 +734,54 @@ mod tests { use super::*; fn parse_example_lock(json: &str) -> Lock { - let mut value: serde_json::Value = serde_json::from_str(json).unwrap(); - - if let Some(validators) = value - .get_mut("distributed_validators") - .and_then(serde_json::Value::as_array_mut) - { - for validator in validators { - let Some(deposit_data) = validator - .get_mut("deposit_data") - .and_then(serde_json::Value::as_object_mut) - else { - continue; - }; - - for (field, len_bytes) in [ - ("pubkey", 48usize), - ("withdrawal_credentials", 32usize), - ("signature", 96usize), - ] { - if deposit_data - .get(field) - .and_then(serde_json::Value::as_str) - .is_some_and(str::is_empty) - { - let zeros = "00".repeat(len_bytes); - deposit_data[field] = serde_json::Value::String(format!("0x{zeros}")); - } - } - } - } - - serde_json::from_value(value).unwrap() + // Empty hex strings (charon's `to0xHex(nil)`) for fixed-size deposit + // fields are handled by the `HexBytes` deserializer, so parse directly. + serde_json::from_str(json).unwrap() } async fn test_eth1_client() -> EthClient { EthClient::new("http://127.0.0.1:8545").await.unwrap() } + /// Mirrors charon's `TestExamples`: every checked-in example cluster file — + /// definitions *and* locks — must deserialize with the versioned parsers. + /// These fixtures exercise charon's `null`/omitempty/empty-hex shapes + /// across the full supported version range (v1.0 through v1.10). + #[test] + fn parses_every_example_file() { + let dir = concat!(env!("CARGO_MANIFEST_DIR"), "/src/examples"); + + let mut definitions = 0usize; + let mut locks = 0usize; + for entry in std::fs::read_dir(dir).expect("read examples dir") { + let path = entry.expect("dir entry").path(); + if path.extension().and_then(|e| e.to_str()) != Some("json") { + continue; + } + + let name = path + .file_name() + .and_then(|n| n.to_str()) + .expect("file name") + .to_owned(); + let json = std::fs::read_to_string(&path).expect("read example file"); + + if name.starts_with("cluster-lock") { + serde_json::from_str::(&json) + .unwrap_or_else(|err| panic!("failed to parse {name}: {err}")); + locks += 1; + } else if name.starts_with("cluster-definition") { + serde_json::from_str::(&json) + .unwrap_or_else(|err| panic!("failed to parse {name}: {err}")); + definitions += 1; + } + } + + // Guard against the glob silently matching nothing (e.g. a renamed dir). + assert!(definitions > 0, "no example definitions found"); + assert!(locks > 0, "no example locks found"); + } + #[test] fn lock_v1_10_0() { let lock = serde_json::from_str::(include_str!("testdata/cluster_lock_v1_10_0.json")) diff --git a/crates/cluster/src/operator.rs b/crates/cluster/src/operator.rs index ce0e3dc0..59b63ce4 100644 --- a/crates/cluster/src/operator.rs +++ b/crates/cluster/src/operator.rs @@ -1,7 +1,7 @@ use crate::{definition::DefinitionError, version::ZERO_NONCE}; use pluto_ssz::serde_utils::HexBytes; use serde::{Deserialize, Serialize}; -use serde_with::serde_as; +use serde_with::{DefaultOnNull, serde_as}; /// Operator represents a charon node operator. #[serde_as] @@ -32,11 +32,16 @@ pub struct OperatorV1X1 { enr: String, /// The nonce of the operator (always 0) nonce: u64, - /// The config signature of the operator - #[serde_as(as = "HexBytes")] + /// The config signature of the operator. + /// + /// v1.0/v1.1 marshal this `[]byte` directly, so a nil signature is emitted + /// as JSON `null`; accept that as an empty signature. + #[serde(default)] + #[serde_as(as = "DefaultOnNull")] pub config_signature: Vec, - /// The ENR signature of the operator - #[serde_as(as = "HexBytes")] + /// The ENR signature of the operator (see `config_signature` re: `null`). + #[serde(default)] + #[serde_as(as = "DefaultOnNull")] pub enr_signature: Vec, } @@ -50,11 +55,16 @@ pub struct OperatorV1X2OrLater { address: String, /// The ENR of the operator enr: String, - /// The config signature of the operator - #[serde_as(as = "HexBytes")] + /// The config signature of the operator. + /// + /// Charon's `ethHex` marshals a nil slice as `""` and tolerates `null` on + /// read, so accept both as an empty signature. + #[serde(default)] + #[serde_as(as = "DefaultOnNull")] config_signature: Vec, - /// The ENR signature of the operator - #[serde_as(as = "HexBytes")] + /// The ENR signature of the operator (see `config_signature`). + #[serde(default)] + #[serde_as(as = "DefaultOnNull")] enr_signature: Vec, } diff --git a/crates/ssz/src/serde_utils.rs b/crates/ssz/src/serde_utils.rs index c4baaa9b..a3b7004b 100644 --- a/crates/ssz/src/serde_utils.rs +++ b/crates/ssz/src/serde_utils.rs @@ -110,21 +110,33 @@ where } } -impl<'de, T> DeserializeAs<'de, T> for HexBytes -where - T: TryFrom>, -{ - fn deserialize_as(deserializer: D) -> Result +impl<'de> DeserializeAs<'de, Vec> for HexBytes { + fn deserialize_as(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + decode_0x_hex::(value.as_str()) + } +} + +// Fixed-size byte arrays deserialise the empty string as all-zeros rather than +// rejecting it. Charon's `to0xHex(nil)` marshals a nil/empty byte slice as +// `""`, and an SSZ fixed-size vector (`Bytes48`, `Bytes32`, ...) of a nil slice +// is exactly `N` zero bytes, so `""` and `"0x00..00"` denote the same value. +// The non-empty path still enforces the exact length. +impl<'de, const N: usize> DeserializeAs<'de, [u8; N]> for HexBytes { + fn deserialize_as(deserializer: D) -> Result<[u8; N], D::Error> where D: Deserializer<'de>, { let value = String::deserialize(deserializer)?; let decoded = decode_0x_hex::(value.as_str())?; - decoded.try_into().map_err(|_err: T::Error| { - D::Error::invalid_value( - Unexpected::Str(value.as_str()), - &"hex bytes convertible to target type", - ) + if decoded.is_empty() { + return Ok([0u8; N]); + } + decoded.try_into().map_err(|bytes: Vec| { + D::Error::invalid_length(bytes.len(), &format!("{N} bytes").as_str()) }) } }