diff --git a/ledger/src/shred.rs b/ledger/src/shred.rs index 4468aa172387f2..e971f79fe0571a 100644 --- a/ledger/src/shred.rs +++ b/ledger/src/shred.rs @@ -129,13 +129,12 @@ pub const MAX_CODE_SHREDS_PER_SLOT: usize = MAX_DATA_SHREDS_PER_SLOT; // 2. Merkles are chained // 3. No retransmit signature (only included for last batch) pub const fn get_data_shred_bytes_per_batch_typical() -> u64 { - let capacity = - match merkle::ShredData::const_capacity(PROOF_ENTRIES_FOR_32_32_BATCH, true, false) { - Ok(v) => v, - Err(_proof_size) => { - panic!("this is unreachable"); - } - }; + let capacity = match merkle::ShredData::const_capacity(PROOF_ENTRIES_FOR_32_32_BATCH, false) { + Ok(v) => v, + Err(_proof_size) => { + panic!("this is unreachable"); + } + }; (DATA_SHREDS_PER_FEC_BLOCK * capacity) as u64 } @@ -226,22 +225,12 @@ enum ShredVariant { // proof_size is the number of Merkle proof entries, and is encoded in the // lowest 4 bits of the binary representation. The first 4 bits identify // the shred variant: - // 0b0100_???? MerkleCode // 0b0110_???? MerkleCode chained // 0b0111_???? MerkleCode chained resigned - // 0b1000_???? MerkleData // 0b1001_???? MerkleData chained // 0b1011_???? MerkleData chained resigned - MerkleCode { - proof_size: u8, - chained: bool, - resigned: bool, - }, // 0b01??_???? - MerkleData { - proof_size: u8, - chained: bool, - resigned: bool, - }, // 0b10??_???? + MerkleCode { proof_size: u8, resigned: bool }, // 0b01??_???? + MerkleData { proof_size: u8, resigned: bool }, // 0b10??_???? } /// A common header that is present in data and code shred headers @@ -622,44 +611,20 @@ impl From for u8 { match shred_variant { ShredVariant::MerkleCode { proof_size, - chained: false, - resigned: false, - } => proof_size | 0x40, - ShredVariant::MerkleCode { - proof_size, - chained: true, resigned: false, } => proof_size | 0x60, ShredVariant::MerkleCode { proof_size, - chained: true, resigned: true, } => proof_size | 0x70, ShredVariant::MerkleData { proof_size, - chained: false, - resigned: false, - } => proof_size | 0x80, - ShredVariant::MerkleData { - proof_size, - chained: true, resigned: false, } => proof_size | 0x90, ShredVariant::MerkleData { proof_size, - chained: true, resigned: true, } => proof_size | 0xb0, - ShredVariant::MerkleCode { - proof_size: _, - chained: false, - resigned: true, - } - | ShredVariant::MerkleData { - proof_size: _, - chained: false, - resigned: true, - } => panic!("Invalid shred variant: {shred_variant:?}"), } } } @@ -674,34 +639,20 @@ impl TryFrom for ShredVariant { } else { let proof_size = shred_variant & 0x0F; match shred_variant & 0xF0 { - 0x40 => Ok(ShredVariant::MerkleCode { - proof_size, - chained: false, - resigned: false, - }), 0x60 => Ok(ShredVariant::MerkleCode { proof_size, - chained: true, resigned: false, }), 0x70 => Ok(ShredVariant::MerkleCode { proof_size, - chained: true, resigned: true, }), - 0x80 => Ok(ShredVariant::MerkleData { - proof_size, - chained: false, - resigned: false, - }), 0x90 => Ok(ShredVariant::MerkleData { proof_size, - chained: true, resigned: false, }), 0xb0 => Ok(ShredVariant::MerkleData { proof_size, - chained: true, resigned: true, }), _ => Err(Error::InvalidShredVariant), @@ -858,15 +809,11 @@ where } match shred_variant { - ShredVariant::MerkleCode { chained: false, .. } - | ShredVariant::MerkleData { chained: false, .. } => { - return true; - } - ShredVariant::MerkleCode { chained: true, .. } => { + ShredVariant::MerkleCode { .. } => { stats.num_shreds_merkle_code_chained = stats.num_shreds_merkle_code_chained.saturating_add(1); } - ShredVariant::MerkleData { chained: true, .. } => { + ShredVariant::MerkleData { .. } => { stats.num_shreds_merkle_data_chained = stats.num_shreds_merkle_data_chained.saturating_add(1); } @@ -907,26 +854,22 @@ pub fn max_entries_per_n_shred_last_or_not( num_shreds: u64, is_last_in_slot: bool, ) -> u64 { - // Default 32:32 erasure batches yields 64 shreds; log2(64) = 6. - let merkle_variant_unsigned = Some(( - /*proof_size:*/ 6, /*chained:*/ true, /*resigned:*/ false, - )); - let merkle_variant_signed = Some(( - /*proof_size:*/ 6, /*chained:*/ true, /*resigned:*/ true, - )); - let vec_size = bincode::serialized_size(&vec![entry]).unwrap(); let entry_size = bincode::serialized_size(entry).unwrap(); let count_size = vec_size - entry_size; + // Default 32:32 erasure batches yields 64 shreds; log2(64) = 6. if !is_last_in_slot { // all shreds are unsigned - let shred_data_size = ShredData::capacity(merkle_variant_unsigned).unwrap() as u64; + let shred_data_size = + ShredData::capacity(/*proof_size:*/ 6, /*resigned:*/ false).unwrap() as u64; (shred_data_size * num_shreds - count_size) / entry_size } else { // last FEC SET is signed, all others are unsigned - let shred_data_size_unsigned = ShredData::capacity(merkle_variant_unsigned).unwrap() as u64; - let shred_data_size_signed = ShredData::capacity(merkle_variant_signed).unwrap() as u64; + let shred_data_size_unsigned = + ShredData::capacity(/*proof_size:*/ 6, /*resigned:*/ false).unwrap() as u64; + let shred_data_size_signed = + ShredData::capacity(/*proof_size:*/ 6, /*resigned:*/ true).unwrap() as u64; let shreds_per_fec_block = SHREDS_PER_FEC_BLOCK as u64; (shred_data_size_unsigned * (num_shreds - shreds_per_fec_block) + shred_data_size_signed * shreds_per_fec_block @@ -941,10 +884,7 @@ pub fn max_entries_per_n_shred( shred_data_size: Option, ) -> u64 { // Default 32:32 erasure batches yields 64 shreds; log2(64) = 6. - let merkle_variant = Some(( - /*proof_size:*/ 6, /*chained:*/ true, /*resigned:*/ true, - )); - let data_buffer_size = ShredData::capacity(merkle_variant).unwrap(); + let data_buffer_size = ShredData::capacity(/*proof_size:*/ 6, /*resigned:*/ true).unwrap(); let shred_data_size = shred_data_size.unwrap_or(data_buffer_size) as u64; let vec_size = bincode::serialized_size(&vec![entry]).unwrap(); let entry_size = bincode::serialized_size(entry).unwrap(); @@ -1026,7 +966,7 @@ mod tests { merkle::make_shreds_from_data( &thread_pool, &Keypair::new(), - Some(chained_merkle_root), + chained_merkle_root, &data[..], slot, parent_slot, @@ -1046,7 +986,6 @@ mod tests { signature: Signature::default(), shred_variant: ShredVariant::MerkleCode { proof_size: 0, - chained: true, resigned: false, }, slot: Slot::MAX, @@ -1092,7 +1031,6 @@ mod tests { SIZE_OF_SHRED_VARIANT, bincode::serialized_size(&ShredVariant::MerkleCode { proof_size: 15, - chained: true, resigned: true }) .unwrap() as usize @@ -1618,7 +1556,6 @@ mod tests { assert_eq!( u8::from(ShredVariant::MerkleCode { proof_size, - chained: true, resigned, }), byte @@ -1626,7 +1563,6 @@ mod tests { assert_eq!( ShredType::from(ShredVariant::MerkleCode { proof_size, - chained: true, resigned, }), ShredType::Code @@ -1635,13 +1571,11 @@ mod tests { ShredVariant::try_from(byte).unwrap(), ShredVariant::MerkleCode { proof_size, - chained: true, resigned, }, ); let buf = bincode::serialize(&ShredVariant::MerkleCode { proof_size, - chained: true, resigned, }) .unwrap(); @@ -1650,7 +1584,6 @@ mod tests { bincode::deserialize::(&[byte]).unwrap(), ShredVariant::MerkleCode { proof_size, - chained: true, resigned, } ); @@ -1665,7 +1598,6 @@ mod tests { assert_eq!( u8::from(ShredVariant::MerkleData { proof_size, - chained: true, resigned, }), byte @@ -1673,7 +1605,6 @@ mod tests { assert_eq!( ShredType::from(ShredVariant::MerkleData { proof_size, - chained: true, resigned, }), ShredType::Data @@ -1682,13 +1613,11 @@ mod tests { ShredVariant::try_from(byte).unwrap(), ShredVariant::MerkleData { proof_size, - chained: true, resigned } ); let buf = bincode::serialize(&ShredVariant::MerkleData { proof_size, - chained: true, resigned, }) .unwrap(); @@ -1697,7 +1626,6 @@ mod tests { bincode::deserialize::(&[byte]).unwrap(), ShredVariant::MerkleData { proof_size, - chained: true, resigned } ); diff --git a/ledger/src/shred/merkle.rs b/ledger/src/shred/merkle.rs index a98e1e84aa3cc4..71ad8e4656f560 100644 --- a/ledger/src/shred/merkle.rs +++ b/ledger/src/shred/merkle.rs @@ -164,12 +164,11 @@ impl ShredData { pub(super) fn get_data( shred: &[u8], proof_size: u8, - chained: bool, resigned: bool, size: u16, // DataShredHeader.size ) -> Result<&[u8], Error> { let size = usize::from(size); - let data_buffer_size = Self::capacity(proof_size, chained, resigned)?; + let data_buffer_size = Self::capacity(proof_size, resigned)?; (Self::SIZE_OF_HEADERS..=Self::SIZE_OF_HEADERS + data_buffer_size) .contains(&size) .then(|| shred.get(Self::SIZE_OF_HEADERS..size)) @@ -180,17 +179,11 @@ impl ShredData { }) } - pub(super) fn get_merkle_root( - shred: &[u8], - proof_size: u8, - chained: bool, - resigned: bool, - ) -> Option { + pub(super) fn get_merkle_root(shred: &[u8], proof_size: u8, resigned: bool) -> Option { debug_assert_eq!( shred::layout::get_shred_variant(shred).unwrap(), ShredVariant::MerkleData { proof_size, - chained, resigned, }, ); @@ -202,23 +195,18 @@ impl ShredData { .map(usize::try_from)? .ok()? }; - let proof_offset = Self::get_proof_offset(proof_size, chained, resigned).ok()?; + let proof_offset = Self::get_proof_offset(proof_size, resigned).ok()?; let proof = get_merkle_proof(shred, proof_offset, proof_size).ok()?; let node = get_merkle_node(shred, SIZE_OF_SIGNATURE..proof_offset).ok()?; get_merkle_root(index, node, proof).ok() } - pub(crate) const fn const_capacity( - proof_size: u8, - chained: bool, - resigned: bool, - ) -> Result { - debug_assert!(chained || !resigned); + pub(crate) const fn const_capacity(proof_size: u8, resigned: bool) -> Result { // Merkle proof is generated and signed after coding shreds are // generated. Coding shred headers cannot be erasure coded either. match Self::SIZE_OF_PAYLOAD.checked_sub( Self::SIZE_OF_HEADERS - + if chained { SIZE_OF_MERKLE_ROOT } else { 0 } + + SIZE_OF_MERKLE_ROOT + (proof_size as usize) * SIZE_OF_MERKLE_PROOF_ENTRY + if resigned { SIZE_OF_SIGNATURE } else { 0 }, ) { @@ -234,17 +222,11 @@ impl ShredCode { // Offset into the payload where the erasure coded slice begins. const ERASURE_SHARD_START_OFFSET: usize = Self::SIZE_OF_HEADERS; - pub(super) fn get_merkle_root( - shred: &[u8], - proof_size: u8, - chained: bool, - resigned: bool, - ) -> Option { + pub(super) fn get_merkle_root(shred: &[u8], proof_size: u8, resigned: bool) -> Option { debug_assert_eq!( shred::layout::get_shred_variant(shred).unwrap(), ShredVariant::MerkleCode { proof_size, - chained, resigned, }, ); @@ -260,7 +242,7 @@ impl ShredCode { .ok()?; num_data_shreds.checked_add(position)? }; - let proof_offset = Self::get_proof_offset(proof_size, chained, resigned).ok()?; + let proof_offset = Self::get_proof_offset(proof_size, resigned).ok()?; let proof = get_merkle_proof(shred, proof_offset, proof_size).ok()?; let node = get_merkle_node(shred, SIZE_OF_SIGNATURE..proof_offset).ok()?; get_merkle_root(index, node, proof).ok() @@ -284,18 +266,13 @@ macro_rules! impl_merkle_shred { // ShredCode::capacity(proof_size, chained, resigned).unwrap() // - ShredData::SIZE_OF_HEADERS // + SIZE_OF_SIGNATURE - pub(super) fn capacity( - proof_size: u8, - chained: bool, - resigned: bool, - ) -> Result { - debug_assert!(chained || !resigned); + pub(super) fn capacity(proof_size: u8, resigned: bool) -> Result { // Merkle proof is generated and signed after coding shreds are // generated. Coding shred headers cannot be erasure coded either. Self::SIZE_OF_PAYLOAD .checked_sub( Self::SIZE_OF_HEADERS - + if chained { SIZE_OF_MERKLE_ROOT } else { 0 } + + SIZE_OF_MERKLE_ROOT + usize::from(proof_size) * SIZE_OF_MERKLE_PROOF_ENTRY + if resigned { SIZE_OF_SIGNATURE } else { 0 }, ) @@ -306,43 +283,34 @@ macro_rules! impl_merkle_shred { fn proof_offset(&self) -> Result { let ShredVariant::$variant { proof_size, - chained, resigned, } = self.common_header.shred_variant else { return Err(Error::InvalidShredVariant); }; - Self::get_proof_offset(proof_size, chained, resigned) + Self::get_proof_offset(proof_size, resigned) } - fn get_proof_offset(proof_size: u8, chained: bool, resigned: bool) -> Result { - Ok(Self::SIZE_OF_HEADERS - + Self::capacity(proof_size, chained, resigned)? - + if chained { SIZE_OF_MERKLE_ROOT } else { 0 }) + fn get_proof_offset(proof_size: u8, resigned: bool) -> Result { + Ok(Self::SIZE_OF_HEADERS + Self::capacity(proof_size, resigned)? + SIZE_OF_MERKLE_ROOT) } fn chained_merkle_root_offset(&self) -> Result { let ShredVariant::$variant { proof_size, - chained, resigned, } = self.common_header.shred_variant else { return Err(Error::InvalidShredVariant); }; - Self::get_chained_merkle_root_offset(proof_size, chained, resigned) + Self::get_chained_merkle_root_offset(proof_size, resigned) } pub(super) fn get_chained_merkle_root_offset( proof_size: u8, - chained: bool, resigned: bool, ) -> Result { - if !chained { - return Err(Error::InvalidShredVariant); - } - debug_assert!(chained); - Ok(Self::SIZE_OF_HEADERS + Self::capacity(proof_size, chained, resigned)?) + Ok(Self::SIZE_OF_HEADERS + Self::capacity(proof_size, resigned)?) } pub(super) fn chained_merkle_root(&self) -> Result { @@ -433,24 +401,22 @@ macro_rules! impl_merkle_shred { pub(super) fn retransmitter_signature_offset(&self) -> Result { let ShredVariant::$variant { proof_size, - chained, resigned, } = self.common_header.shred_variant else { return Err(Error::InvalidShredVariant); }; - Self::get_retransmitter_signature_offset(proof_size, chained, resigned) + Self::get_retransmitter_signature_offset(proof_size, resigned) } pub(super) fn get_retransmitter_signature_offset( proof_size: u8, - chained: bool, resigned: bool, ) -> Result { if !resigned { return Err(Error::InvalidShredVariant); } - let proof_offset = Self::get_proof_offset(proof_size, chained, resigned)?; + let proof_offset = Self::get_proof_offset(proof_size, resigned)?; Ok(proof_offset + usize::from(proof_size) * SIZE_OF_MERKLE_PROOF_ENTRY) } @@ -461,13 +427,12 @@ macro_rules! impl_merkle_shred { } let ShredVariant::$variant { proof_size, - chained, resigned, } = self.common_header.shred_variant else { return Err(Error::InvalidShredVariant); }; - let offset = Self::SIZE_OF_HEADERS + Self::capacity(proof_size, chained, resigned)?; + let offset = Self::SIZE_OF_HEADERS + Self::capacity(proof_size, resigned)?; Ok(Self::ERASURE_SHARD_START_OFFSET..offset) } @@ -621,19 +586,12 @@ impl ShredDataTrait for ShredData { fn data(&self) -> Result<&[u8], Error> { let ShredVariant::MerkleData { proof_size, - chained, resigned, } = self.common_header.shred_variant else { return Err(Error::InvalidShredVariant); }; - Self::get_data( - &self.payload, - proof_size, - chained, - resigned, - self.data_header.size, - ) + Self::get_data(&self.payload, proof_size, resigned, self.data_header.size) } } @@ -703,12 +661,11 @@ pub(super) fn recover( ) }; debug_assert_matches!(common_header.shred_variant, ShredVariant::MerkleCode { .. }); - let (proof_size, chained, resigned) = match common_header.shred_variant { + let (proof_size, resigned) = match common_header.shred_variant { ShredVariant::MerkleCode { proof_size, - chained, resigned, - } => (proof_size, chained, resigned), + } => (proof_size, resigned), ShredVariant::MerkleData { .. } => { return Err(Error::InvalidShredVariant); } @@ -733,7 +690,6 @@ pub(super) fn recover( shred_variant == &ShredVariant::MerkleData { proof_size, - chained, resigned, } } @@ -746,7 +702,6 @@ pub(super) fn recover( shred_variant == &ShredVariant::MerkleCode { proof_size, - chained, resigned, } && num_data_shreds == coding_header.num_data_shreds @@ -928,7 +883,6 @@ fn make_stub_shred( }; let shred_variant = ShredVariant::MerkleData { proof_size, - chained: chained_merkle_root.is_some(), resigned: retransmitter_signature.is_some(), }; let index = common_header.fec_set_index @@ -977,8 +931,8 @@ fn make_shreds_data<'a>( debug_assert_matches!(common_header.shred_variant, ShredVariant::MerkleData { .. }); chunks.into_iter().map(move |chunk| { debug_assert_matches!(common_header.shred_variant, - ShredVariant::MerkleData { proof_size, chained, resigned } - if chunk.len() <= ShredData::capacity(proof_size, chained, resigned).unwrap() + ShredVariant::MerkleData { proof_size, resigned } + if chunk.len() <= ShredData::capacity(proof_size, resigned).unwrap() ); let size = ShredData::SIZE_OF_HEADERS + chunk.len(); let mut payload = vec![0u8; ShredData::SIZE_OF_PAYLOAD]; @@ -1022,8 +976,7 @@ fn make_shreds_code_header_only( pub(crate) fn make_shreds_from_data( thread_pool: &ThreadPool, keypair: &Keypair, - // The Merkle root of the previous erasure batch if chained. - chained_merkle_root: Option, + chained_merkle_root: Hash, data: &[u8], // Serialized &[Entry] slot: Slot, parent_slot: Slot, @@ -1036,19 +989,15 @@ pub(crate) fn make_shreds_from_data( stats: &mut ProcessShredsStats, ) -> Result, Error> { let now = Instant::now(); - let chained = chained_merkle_root.is_some(); - - // only sign if last fec set in slot and is chained - let sign_last_fec_set = chained && is_last_in_slot; let proof_size = PROOF_ENTRIES_FOR_32_32_BATCH; // unsigned data_buffer size - let data_buffer_per_shred_size = ShredData::capacity(proof_size, chained, false)?; + let data_buffer_per_shred_size = ShredData::capacity(proof_size, false)?; let data_buffer_total_size = DATA_SHREDS_PER_FEC_BLOCK * data_buffer_per_shred_size; // signed data_buffer size - let data_buffer_per_shred_size_signed = if sign_last_fec_set { - ShredData::capacity(proof_size, chained, true)? + let data_buffer_per_shred_size_signed = if is_last_in_slot { + ShredData::capacity(proof_size, true)? } else { 0 }; @@ -1060,7 +1009,6 @@ pub(crate) fn make_shreds_from_data( signature: Signature::default(), shred_variant: ShredVariant::MerkleData { proof_size, - chained, resigned: false, }, slot, @@ -1073,7 +1021,6 @@ pub(crate) fn make_shreds_from_data( let mut common_header_code = ShredCommonHeader { shred_variant: ShredVariant::MerkleCode { proof_size, - chained, resigned: false, }, index: next_code_index, @@ -1094,7 +1041,7 @@ pub(crate) fn make_shreds_from_data( } }; - let (mut unsigned_data, signed_data) = if sign_last_fec_set { + let (mut unsigned_data, signed_data) = if is_last_in_slot { // Reserve at least one signed batch (may be empty) at the end. if data.len() > data_buffer_total_size_signed { // sign everything except the last batch @@ -1111,7 +1058,7 @@ pub(crate) fn make_shreds_from_data( stats.data_bytes += unsigned_data.len() + signed_data.len(); let unsigned_sets = unsigned_data.len().div_ceil(data_buffer_total_size); - let number_of_fec_sets = if sign_last_fec_set { + let number_of_fec_sets = if is_last_in_slot { unsigned_sets + 1 } else { unsigned_sets @@ -1147,11 +1094,10 @@ pub(crate) fn make_shreds_from_data( // 2.) Shreds is_empty, which only happens when we entered w/ zero data. // // In either case, we want to generate empty data shreds. - if !unsigned_data.is_empty() || (shreds.is_empty() && !sign_last_fec_set) { + if !unsigned_data.is_empty() || (shreds.is_empty() && !is_last_in_slot) { stats.padding_bytes += data_buffer_total_size - unsigned_data.len(); shred_leftover_data( proof_size, - chained, false, unsigned_data, data_buffer_per_shred_size, @@ -1161,11 +1107,10 @@ pub(crate) fn make_shreds_from_data( &mut shreds, ); } - if !signed_data.is_empty() || (shreds.is_empty() && sign_last_fec_set) { + if !signed_data.is_empty() || (shreds.is_empty() && is_last_in_slot) { stats.padding_bytes += data_buffer_total_size_signed - signed_data.len(); shred_leftover_data( proof_size, - chained, true, signed_data, data_buffer_per_shred_size_signed, @@ -1200,45 +1145,21 @@ pub(crate) fn make_shreds_from_data( let batches: Vec<&mut [Shred]> = shreds .chunk_by_mut(|a, b| a.fec_set_index() == b.fec_set_index()) .collect(); - if let Some(chained_merkle_root) = chained_merkle_root { - // We have to process erasure batches serially because the Merkle tree - // (and so the signature) cannot be computed without the Merkle root of - // the previous erasure batch. - batches - .into_iter() - .try_fold(chained_merkle_root, |chained_merkle_root, batch| { - finish_erasure_batch( - Some(thread_pool), - keypair, - batch, - Some(chained_merkle_root), - reed_solomon_cache, - ) - })?; - } else if batches.len() <= 1 { - for batch in batches { + + // We have to process erasure batches serially because the Merkle tree + // (and so the signature) cannot be computed without the Merkle root of + // the previous erasure batch. + batches + .into_iter() + .try_fold(chained_merkle_root, |chained_merkle_root, batch| { finish_erasure_batch( Some(thread_pool), keypair, batch, - None, // chained_merkle_root + chained_merkle_root, reed_solomon_cache, - )?; - } - } else { - thread_pool.install(|| { - batches.into_par_iter().try_for_each(|batch| { - finish_erasure_batch( - None, // thread_pool - keypair, - batch, - None, // chained_merkle_root - reed_solomon_cache, - ) - .map(|_| ()) - }) + ) })?; - } stats.gen_coding_elapsed += now.elapsed().as_micros() as u64; Ok(shreds) } @@ -1246,7 +1167,6 @@ pub(crate) fn make_shreds_from_data( #[allow(clippy::too_many_arguments)] fn shred_leftover_data( proof_size: u8, - chained: bool, resigned: bool, data: &[u8], data_buffer_per_shred_size: usize, @@ -1257,12 +1177,10 @@ fn shred_leftover_data( ) { common_header_data.shred_variant = ShredVariant::MerkleData { proof_size, - chained, resigned, }; common_header_code.shred_variant = ShredVariant::MerkleCode { proof_size, - chained, resigned, }; common_header_data.fec_set_index = common_header_data.index; @@ -1291,7 +1209,7 @@ fn finish_erasure_batch( keypair: &Keypair, shreds: &mut [Shred], // The Merkle root of the previous erasure batch if chained. - chained_merkle_root: Option, + chained_merkle_root: Hash, reed_solomon_cache: &ReedSolomonCache, ) -> Result { debug_assert_eq!(shreds.iter().map(Shred::fec_set_index).dedup().count(), 1); @@ -1338,11 +1256,10 @@ fn finish_erasure_batch( .collect::, _>>()?, )?; // Set the chained_merkle_root for each shred. - if let Some(chained_merkle_root) = chained_merkle_root { - for shred in shreds.iter_mut() { - shred.set_chained_merkle_root(&chained_merkle_root)?; - } + for shred in shreds.iter_mut() { + shred.set_chained_merkle_root(&chained_merkle_root)?; } + // Compute the Merkle tree for the erasure batch. let tree = match thread_pool { None => { @@ -1393,11 +1310,10 @@ mod test { }; // Total size of a data shred including headers and merkle proof. - fn shred_data_size_of_payload(proof_size: u8, chained: bool, resigned: bool) -> usize { - assert!(chained || !resigned); + fn shred_data_size_of_payload(proof_size: u8, resigned: bool) -> usize { ShredData::SIZE_OF_HEADERS - + ShredData::capacity(proof_size, chained, resigned).unwrap() - + if chained { SIZE_OF_MERKLE_ROOT } else { 0 } + + ShredData::capacity(proof_size, resigned).unwrap() + + SIZE_OF_MERKLE_ROOT + usize::from(proof_size) * SIZE_OF_MERKLE_PROOF_ENTRY + if resigned { SIZE_OF_SIGNATURE } else { 0 } } @@ -1406,56 +1322,49 @@ mod test { // All payload excluding merkle proof and the signature are erasure coded. // Therefore the data capacity is equal to erasure encoded shard size minus // size of erasure encoded header. - fn shred_data_capacity(proof_size: u8, chained: bool, resigned: bool) -> usize { + fn shred_data_capacity(proof_size: u8, resigned: bool) -> usize { const SIZE_OF_ERASURE_ENCODED_HEADER: usize = ShredData::SIZE_OF_HEADERS - SIZE_OF_SIGNATURE; - ShredCode::capacity(proof_size, chained, resigned).unwrap() - SIZE_OF_ERASURE_ENCODED_HEADER + ShredCode::capacity(proof_size, resigned).unwrap() - SIZE_OF_ERASURE_ENCODED_HEADER } - fn shred_data_size_of_erasure_encoded_slice( - proof_size: u8, - chained: bool, - resigned: bool, - ) -> usize { + fn shred_data_size_of_erasure_encoded_slice(proof_size: u8, resigned: bool) -> usize { ShredData::SIZE_OF_PAYLOAD - SIZE_OF_SIGNATURE - - if chained { SIZE_OF_MERKLE_ROOT } else { 0 } + - SIZE_OF_MERKLE_ROOT - usize::from(proof_size) * SIZE_OF_MERKLE_PROOF_ENTRY - if resigned { SIZE_OF_SIGNATURE } else { 0 } } - #[test_case(false, false)] - #[test_case(true, false)] - #[test_case(true, true)] - fn test_shred_data_size_of_payload(chained: bool, resigned: bool) { + #[test_case(false)] + #[test_case(true)] + fn test_shred_data_size_of_payload(resigned: bool) { for proof_size in 0..0x15 { assert_eq!( ShredData::SIZE_OF_PAYLOAD, - shred_data_size_of_payload(proof_size, chained, resigned) + shred_data_size_of_payload(proof_size, resigned) ); } } - #[test_case(false, false)] - #[test_case(true, false)] - #[test_case(true, true)] - fn test_shred_data_capacity(chained: bool, resigned: bool) { + #[test_case(false)] + #[test_case(true)] + fn test_shred_data_capacity(resigned: bool) { for proof_size in 0..0x15 { assert_eq!( - ShredData::capacity(proof_size, chained, resigned).unwrap(), - shred_data_capacity(proof_size, chained, resigned) + ShredData::capacity(proof_size, resigned).unwrap(), + shred_data_capacity(proof_size, resigned) ); } } - #[test_case(false, false)] - #[test_case(true, false)] - #[test_case(true, true)] - fn test_shred_code_capacity(chained: bool, resigned: bool) { + #[test_case(false)] + #[test_case(true)] + fn test_shred_code_capacity(resigned: bool) { for proof_size in 0..0x15 { assert_eq!( - ShredCode::capacity(proof_size, chained, resigned).unwrap(), - shred_data_size_of_erasure_encoded_slice(proof_size, chained, resigned), + ShredCode::capacity(proof_size, resigned).unwrap(), + shred_data_size_of_erasure_encoded_slice(proof_size, resigned), ); } } @@ -1485,35 +1394,27 @@ mod test { } } - #[test_case(19, false, false)] - #[test_case(19, true, false)] - #[test_case(19, true, true)] - #[test_case(31, false, false)] - #[test_case(31, true, false)] - #[test_case(31, true, true)] - #[test_case(32, false, false)] - #[test_case(32, true, false)] - #[test_case(32, true, true)] - #[test_case(33, false, false)] - #[test_case(33, true, false)] - #[test_case(33, true, true)] - #[test_case(37, false, false)] - #[test_case(37, true, false)] - #[test_case(37, true, true)] - #[test_case(64, false, false)] - #[test_case(64, true, false)] - #[test_case(64, true, true)] - #[test_case(73, false, false)] - #[test_case(73, true, false)] - #[test_case(73, true, true)] - fn test_recover_merkle_shreds(num_shreds: usize, chained: bool, resigned: bool) { + #[test_case(19, false)] + #[test_case(19, true)] + #[test_case(31, false)] + #[test_case(31, true)] + #[test_case(32, false)] + #[test_case(32, true)] + #[test_case(33, false)] + #[test_case(33, true)] + #[test_case(37, false)] + #[test_case(37, true)] + #[test_case(64, false)] + #[test_case(64, true)] + #[test_case(73, false)] + #[test_case(73, true)] + fn test_recover_merkle_shreds(num_shreds: usize, resigned: bool) { let mut rng = rand::thread_rng(); let reed_solomon_cache = ReedSolomonCache::default(); for num_data_shreds in 1..num_shreds { let num_coding_shreds = num_shreds - num_data_shreds; run_recover_merkle_shreds( &mut rng, - chained, resigned, num_data_shreds, num_coding_shreds, @@ -1524,7 +1425,6 @@ mod test { fn run_recover_merkle_shreds( rng: &mut R, - chained: bool, resigned: bool, num_data_shreds: usize, num_coding_shreds: usize, @@ -1533,12 +1433,11 @@ mod test { let keypair = Keypair::new(); let num_shreds = num_data_shreds + num_coding_shreds; let proof_size = get_proof_size(num_shreds); - let capacity = ShredData::capacity(proof_size, chained, resigned).unwrap(); + let capacity = ShredData::capacity(proof_size, resigned).unwrap(); let common_header = ShredCommonHeader { signature: Signature::default(), shred_variant: ShredVariant::MerkleData { proof_size, - chained, resigned, }, slot: 145_865_705, @@ -1595,7 +1494,6 @@ mod test { let common_header = ShredCommonHeader { shred_variant: ShredVariant::MerkleCode { proof_size, - chained, resigned, }, index: common_header.index + i as u32 + 7, @@ -1687,77 +1585,48 @@ mod test { #[test_matrix( [0, 15600, 31200, 46800], - [true, false], [true, false] )] - fn test_make_shreds_from_data(data_size: usize, chained: bool, is_last_in_slot: bool) { + fn test_make_shreds_from_data(data_size: usize, is_last_in_slot: bool) { let mut rng = rand::thread_rng(); let data_size = data_size.saturating_sub(16); let reed_solomon_cache = ReedSolomonCache::default(); for data_size in data_size..data_size + 32 { - run_make_shreds_from_data( - &mut rng, - data_size, - chained, - is_last_in_slot, - &reed_solomon_cache, - ); + run_make_shreds_from_data(&mut rng, data_size, is_last_in_slot, &reed_solomon_cache); } } - #[test_matrix( - [true, false], - [true, false] - )] - fn test_make_shreds_from_data_rand(chained: bool, is_last_in_slot: bool) { + #[test_case(true)] + #[test_case(false)] + fn test_make_shreds_from_data_rand(is_last_in_slot: bool) { let mut rng = rand::thread_rng(); let reed_solomon_cache = ReedSolomonCache::default(); for _ in 0..32 { let data_size = rng.gen_range(0..31200 * 7); - run_make_shreds_from_data( - &mut rng, - data_size, - chained, - is_last_in_slot, - &reed_solomon_cache, - ); + run_make_shreds_from_data(&mut rng, data_size, is_last_in_slot, &reed_solomon_cache); } } #[ignore] - #[test_matrix( - [true, false], - [true, false] - )] - fn test_make_shreds_from_data_paranoid(chained: bool, is_last_in_slot: bool) { + #[test_case(true)] + #[test_case(false)] + fn test_make_shreds_from_data_paranoid(is_last_in_slot: bool) { let mut rng = rand::thread_rng(); let reed_solomon_cache = ReedSolomonCache::default(); for data_size in 0..=PACKET_DATA_SIZE * 4 * 64 { - run_make_shreds_from_data( - &mut rng, - data_size, - chained, - is_last_in_slot, - &reed_solomon_cache, - ); + run_make_shreds_from_data(&mut rng, data_size, is_last_in_slot, &reed_solomon_cache); } } fn run_make_shreds_from_data( rng: &mut R, data_size: usize, - chained: bool, is_last_in_slot: bool, reed_solomon_cache: &ReedSolomonCache, ) { let thread_pool = ThreadPoolBuilder::new().num_threads(2).build().unwrap(); let keypair = Keypair::new(); - let chained_merkle_root = chained.then(|| Hash::new_from_array(rng.gen())); - - // only sign last batch if it is chained and is the last in slot - // let resigned = chained && is_last_in_slot; - let sign_last_fec_set = chained && is_last_in_slot; - + let chained_merkle_root = Hash::new_from_array(rng.gen()); let slot = 149_745_689; let parent_slot = slot - rng.gen_range(1..65536); let shred_version = rng.gen(); @@ -1812,12 +1681,8 @@ mod test { let shred_type = ShredType::from(shred_variant); let key = ShredId::new(slot, index, shred_type); let merkle_root = shred.merkle_root().unwrap(); - let chained_merkle_root = if chained { - Some(shred.chained_merkle_root().unwrap()) - } else { - assert_matches!(shred.chained_merkle_root(), Err(Error::InvalidShredVariant)); - None - }; + let chained_merkle_root = shred.chained_merkle_root().unwrap(); + assert!(signature.verify(pubkey.as_ref(), merkle_root.as_ref())); // Verify shred::layout api. let shred = shred.payload(); @@ -1834,7 +1699,7 @@ mod test { assert_eq!(shred::layout::get_merkle_root(shred), Some(merkle_root)); assert_eq!( shred::layout::get_chained_merkle_root(shred), - chained_merkle_root + Some(chained_merkle_root) ); let data = shred::layout::get_signed_data(shred).unwrap(); assert_eq!(data, merkle_root); @@ -1845,7 +1710,7 @@ mod test { let mut num_coding_shreds = 0; for (index, shred) in shreds.iter().enumerate() { let common_header = shred.common_header(); - let resigned = sign_last_fec_set && index >= shreds.len() - 64; + let resigned = is_last_in_slot && index >= shreds.len() - 64; assert_eq!(common_header.slot, slot); assert_eq!(common_header.version, shred_version); @@ -1857,7 +1722,6 @@ mod test { common_header.shred_variant, ShredVariant::MerkleCode { proof_size, - chained, resigned } ); @@ -1875,7 +1739,6 @@ mod test { common_header.shred_variant, ShredVariant::MerkleData { proof_size, - chained, resigned } ); @@ -1911,25 +1774,22 @@ mod test { } assert!(num_coding_shreds >= num_data_shreds); // Verify chained Merkle roots. - if let Some(chained_merkle_root) = chained_merkle_root { - let chained_merkle_roots: HashMap = - std::iter::once((0, chained_merkle_root)) - .chain( - shreds - .iter() - .sorted_unstable_by_key(|shred| shred.fec_set_index()) - .dedup_by(|shred, other| shred.fec_set_index() == other.fec_set_index()) - .map(|shred| (shred.fec_set_index(), shred.merkle_root().unwrap())), - ) - .tuple_windows() - .map(|((_, merkle_root), (fec_set_index, _))| (fec_set_index, merkle_root)) - .collect(); - for shred in &shreds { - assert_eq!( - shred.chained_merkle_root().unwrap(), - chained_merkle_roots[&shred.fec_set_index()] - ); - } + let chained_merkle_roots: HashMap = std::iter::once((0, chained_merkle_root)) + .chain( + shreds + .iter() + .sorted_unstable_by_key(|shred| shred.fec_set_index()) + .dedup_by(|shred, other| shred.fec_set_index() == other.fec_set_index()) + .map(|shred| (shred.fec_set_index(), shred.merkle_root().unwrap())), + ) + .tuple_windows() + .map(|((_, merkle_root), (fec_set_index, _))| (fec_set_index, merkle_root)) + .collect(); + for shred in &shreds { + assert_eq!( + shred.chained_merkle_root().unwrap(), + chained_merkle_roots[&shred.fec_set_index()] + ); } // Assert that only the last shred is LAST_SHRED_IN_SLOT. assert_eq!( diff --git a/ledger/src/shred/shred_data.rs b/ledger/src/shred/shred_data.rs index fae7990a779b3d..ae3e4ccb4b7892 100644 --- a/ledger/src/shred/shred_data.rs +++ b/ledger/src/shred/shred_data.rs @@ -85,20 +85,8 @@ impl ShredData { // Maximum size of ledger data that can be embedded in a data-shred. // merkle_proof_size is the number of merkle proof entries. // None indicates a legacy data-shred. - pub fn capacity( - merkle_variant: Option<( - u8, // proof_size - bool, // chained - bool, // resigned - )>, - ) -> Result { - match merkle_variant { - None => Err(Error::InvalidShredVariant), - Some((proof_size, chained, resigned)) => { - debug_assert!(chained || !resigned); - merkle::ShredData::capacity(proof_size, chained, resigned) - } - } + pub fn capacity(proof_size: u8, resigned: bool) -> Result { + merkle::ShredData::capacity(proof_size, resigned) } pub(super) fn retransmitter_signature(&self) -> Result { diff --git a/ledger/src/shred/wire.rs b/ledger/src/shred/wire.rs index 0e74997f36a0e6..f4c6b21b62e91c 100644 --- a/ledger/src/shred/wire.rs +++ b/ledger/src/shred/wire.rs @@ -160,15 +160,8 @@ pub(crate) fn get_data(shred: &[u8]) -> Result<&[u8], Error> { ShredVariant::MerkleCode { .. } => Err(Error::InvalidShredType), ShredVariant::MerkleData { proof_size, - chained, resigned, - } => shred::merkle::ShredData::get_data( - shred, - proof_size, - chained, - resigned, - get_data_size(shred)?, - ), + } => shred::merkle::ShredData::get_data(shred, proof_size, resigned, get_data_size(shred)?), } } @@ -213,14 +206,12 @@ pub(crate) fn get_signed_data(shred: &[u8]) -> Option { let data = match get_shred_variant(shred).ok()? { ShredVariant::MerkleCode { proof_size, - chained, resigned, - } => shred::merkle::ShredCode::get_merkle_root(shred, proof_size, chained, resigned)?, + } => shred::merkle::ShredCode::get_merkle_root(shred, proof_size, resigned)?, ShredVariant::MerkleData { proof_size, - chained, resigned, - } => shred::merkle::ShredData::get_merkle_root(shred, proof_size, chained, resigned)?, + } => shred::merkle::ShredData::get_merkle_root(shred, proof_size, resigned)?, }; Some(data) } @@ -239,14 +230,12 @@ pub fn get_merkle_root(shred: &[u8]) -> Option { match get_shred_variant(shred).ok()? { ShredVariant::MerkleCode { proof_size, - chained, resigned, - } => shred::merkle::ShredCode::get_merkle_root(shred, proof_size, chained, resigned), + } => shred::merkle::ShredCode::get_merkle_root(shred, proof_size, resigned), ShredVariant::MerkleData { proof_size, - chained, resigned, - } => shred::merkle::ShredData::get_merkle_root(shred, proof_size, chained, resigned), + } => shred::merkle::ShredData::get_merkle_root(shred, proof_size, resigned), } } @@ -254,18 +243,12 @@ pub(crate) fn get_chained_merkle_root(shred: &[u8]) -> Option { let offset = match get_shred_variant(shred).ok()? { ShredVariant::MerkleCode { proof_size, - chained, resigned, - } => { - shred::merkle::ShredCode::get_chained_merkle_root_offset(proof_size, chained, resigned) - } + } => shred::merkle::ShredCode::get_chained_merkle_root_offset(proof_size, resigned), ShredVariant::MerkleData { proof_size, - chained, resigned, - } => { - shred::merkle::ShredData::get_chained_merkle_root_offset(proof_size, chained, resigned) - } + } => shred::merkle::ShredData::get_chained_merkle_root_offset(proof_size, resigned), } .ok()?; let merkle_root = shred.get(offset..offset + SIZE_OF_MERKLE_ROOT)?; @@ -278,18 +261,12 @@ fn get_retransmitter_signature_offset(shred: &[u8]) -> Result { match get_shred_variant(shred)? { ShredVariant::MerkleCode { proof_size, - chained, resigned, - } => shred::merkle::ShredCode::get_retransmitter_signature_offset( - proof_size, chained, resigned, - ), + } => shred::merkle::ShredCode::get_retransmitter_signature_offset(proof_size, resigned), ShredVariant::MerkleData { proof_size, - chained, resigned, - } => shred::merkle::ShredData::get_retransmitter_signature_offset( - proof_size, chained, resigned, - ), + } => shred::merkle::ShredData::get_retransmitter_signature_offset(proof_size, resigned), } } @@ -305,12 +282,10 @@ pub fn is_retransmitter_signed_variant(shred: &[u8]) -> Result { match get_shred_variant(shred)? { ShredVariant::MerkleCode { proof_size: _, - chained: _, resigned, } => Ok(resigned), ShredVariant::MerkleData { proof_size: _, - chained: _, resigned, } => Ok(resigned), } @@ -360,24 +335,18 @@ pub fn resign_shred(shred: &mut [u8], keypair: &Keypair) -> Result<(), Error> { let (offset, merkle_root) = match get_shred_variant(shred)? { ShredVariant::MerkleCode { proof_size, - chained, resigned, } => ( - shred::merkle::ShredCode::get_retransmitter_signature_offset( - proof_size, chained, resigned, - )?, - shred::merkle::ShredCode::get_merkle_root(shred, proof_size, chained, resigned) + shred::merkle::ShredCode::get_retransmitter_signature_offset(proof_size, resigned)?, + shred::merkle::ShredCode::get_merkle_root(shred, proof_size, resigned) .ok_or(Error::InvalidMerkleRoot)?, ), ShredVariant::MerkleData { proof_size, - chained, resigned, } => ( - shred::merkle::ShredData::get_retransmitter_signature_offset( - proof_size, chained, resigned, - )?, - shred::merkle::ShredData::get_merkle_root(shred, proof_size, chained, resigned) + shred::merkle::ShredData::get_retransmitter_signature_offset(proof_size, resigned)?, + shred::merkle::ShredData::get_merkle_root(shred, proof_size, resigned) .ok_or(Error::InvalidMerkleRoot)?, ), }; diff --git a/ledger/src/shredder.rs b/ledger/src/shredder.rs index 9b3a3e55c355c6..d9890d6747d933 100644 --- a/ledger/src/shredder.rs +++ b/ledger/src/shredder.rs @@ -108,7 +108,7 @@ impl Shredder { let shreds = shred::merkle::make_shreds_from_data( thread_pool, keypair, - Some(chained_merkle_root), + chained_merkle_root, data, self.slot, self.parent_slot, @@ -188,7 +188,8 @@ impl Shredder { // For backward compatibility. This is needed when the data shred // payload is None, so that deserializing to Vec results in // an empty vector. - let data_buffer_size = ShredData::capacity(/*merkle_proof_size:*/ None).unwrap(); + let data_buffer_size = + ShredData::capacity(/*proof_size:*/ 0, /*resigned:*/ false).unwrap(); Ok(vec![0u8; data_buffer_size]) } else { Ok(data) @@ -494,7 +495,7 @@ mod tests { let shredder = Shredder::new(slot, slot - 5, 0, 0).unwrap(); // Create enough entries to make > 1 shred let data_buffer_size = - ShredData::capacity(/*merkle_proof_size:*/ Some((6, true, false))).unwrap(); + ShredData::capacity(/*proof_size:*/ 6, /*resigned:*/ false).unwrap(); let num_entries = max_ticks_per_n_shreds(1, Some(data_buffer_size)) + 1; let entries: Vec<_> = (0..num_entries) .map(|_| { diff --git a/ledger/tests/shred.rs b/ledger/tests/shred.rs index a4c2b36cb0380e..7931c6e66a7f58 100644 --- a/ledger/tests/shred.rs +++ b/ledger/tests/shred.rs @@ -200,7 +200,7 @@ fn setup_different_sized_fec_blocks( let keypair1 = Keypair::new(); let tx0 = system_transaction::transfer(&keypair0, &keypair1.pubkey(), 1, Hash::default()); let entry = Entry::new(&Hash::default(), 1, vec![tx0]); - let merkle_capacity = ShredData::capacity(Some((6, true, true))).unwrap(); + let merkle_capacity = ShredData::capacity(/*proof_size:*/ 6, /*resigned:*/ true).unwrap(); let chained_merkle_root = Hash::default(); assert!(DATA_SHREDS_PER_FEC_BLOCK > 2);