From 4c2631b8bc08ef878fc3040673802dd04593c6d8 Mon Sep 17 00:00:00 2001 From: behzad nouri Date: Sat, 11 Dec 2021 14:47:20 +0000 Subject: [PATCH 1/2] uses Option for SlotMeta.last_index (#21775) SlotMeta.last_index may be unknown and current code is using u64::MAX to indicate that: https://github.com/solana-labs/solana/blob/6c108c8fc/ledger/src/blockstore_meta.rs#L169-L174 This lacks type-safety and can introduce bugs if not always checked for Several instances of slot_meta.last_index + 1 are also subject to overflow. This commit updates the type to Option. Backward compatibility is maintained by customizing serde serialize/deserialize implementations. (cherry picked from commit e08139f949b6ea43d7cda0122b04da45169eea27) # Conflicts: # core/src/repair_generic_traversal.rs # ledger/src/blockstore.rs # ledger/src/blockstore_meta.rs --- core/src/repair_generic_traversal.rs | 343 +++++++++++++++++++++++++++ ledger/src/blockstore.rs | 266 +++++++++++++++++++-- ledger/src/blockstore_meta.rs | 48 +++- 3 files changed, 623 insertions(+), 34 deletions(-) create mode 100644 core/src/repair_generic_traversal.rs diff --git a/core/src/repair_generic_traversal.rs b/core/src/repair_generic_traversal.rs new file mode 100644 index 00000000000000..b5d78667828f3a --- /dev/null +++ b/core/src/repair_generic_traversal.rs @@ -0,0 +1,343 @@ +use { + crate::{ + heaviest_subtree_fork_choice::HeaviestSubtreeForkChoice, repair_service::RepairService, + serve_repair::ShredRepairType, tree_diff::TreeDiff, + }, + solana_ledger::{blockstore::Blockstore, blockstore_meta::SlotMeta}, + solana_sdk::{clock::Slot, hash::Hash}, + std::collections::{HashMap, HashSet}, +}; + +struct GenericTraversal<'a> { + tree: &'a HeaviestSubtreeForkChoice, + pending: Vec, +} + +impl<'a> GenericTraversal<'a> { + pub fn new(tree: &'a HeaviestSubtreeForkChoice) -> Self { + Self { + tree, + pending: vec![tree.root().0], + } + } +} + +impl<'a> Iterator for GenericTraversal<'a> { + type Item = Slot; + fn next(&mut self) -> Option { + let next = self.pending.pop(); + if let Some(slot) = next { + let children: Vec<_> = self + .tree + .children(&(slot, Hash::default())) + .unwrap() + .iter() + .map(|(child_slot, _)| *child_slot) + .collect(); + self.pending.extend(children); + } + next + } +} + +pub fn get_unknown_last_index( + tree: &HeaviestSubtreeForkChoice, + blockstore: &Blockstore, + slot_meta_cache: &mut HashMap>, + processed_slots: &mut HashSet, + limit: usize, +) -> Vec { + let iter = GenericTraversal::new(tree); + let mut unknown_last = Vec::new(); + for slot in iter { + if processed_slots.contains(&slot) { + continue; + } + let slot_meta = slot_meta_cache + .entry(slot) + .or_insert_with(|| blockstore.meta(slot).unwrap()); + if let Some(slot_meta) = slot_meta { + if slot_meta.last_index.is_none() { + let shred_index = blockstore.get_index(slot).unwrap(); + let num_processed_shreds = if let Some(shred_index) = shred_index { + shred_index.data().num_shreds() as u64 + } else { + slot_meta.consumed + }; + unknown_last.push((slot, slot_meta.received, num_processed_shreds)); + processed_slots.insert(slot); + } + } + } + // prioritize slots with more received shreds + unknown_last.sort_by(|(_, _, count1), (_, _, count2)| count2.cmp(count1)); + unknown_last + .iter() + .take(limit) + .map(|(slot, received, _)| ShredRepairType::HighestShred(*slot, *received)) + .collect() +} + +fn get_unrepaired_path( + start_slot: Slot, + blockstore: &Blockstore, + slot_meta_cache: &mut HashMap>, + visited: &mut HashSet, +) -> Vec { + let mut path = Vec::new(); + let mut slot = start_slot; + while !visited.contains(&slot) { + visited.insert(slot); + let slot_meta = slot_meta_cache + .entry(slot) + .or_insert_with(|| blockstore.meta(slot).unwrap()); + if let Some(slot_meta) = slot_meta { + if slot_meta.is_full() { + break; + } + path.push(slot); + slot = slot_meta.parent_slot; + } + } + path.reverse(); + path +} + +pub fn get_closest_completion( + tree: &HeaviestSubtreeForkChoice, + blockstore: &Blockstore, + slot_meta_cache: &mut HashMap>, + processed_slots: &mut HashSet, + limit: usize, +) -> Vec { + let mut v: Vec<(Slot, u64)> = Vec::default(); + let iter = GenericTraversal::new(tree); + for slot in iter { + if processed_slots.contains(&slot) { + continue; + } + let slot_meta = slot_meta_cache + .entry(slot) + .or_insert_with(|| blockstore.meta(slot).unwrap()); + if let Some(slot_meta) = slot_meta { + if slot_meta.is_full() { + continue; + } + if let Some(last_index) = slot_meta.last_index { + let shred_index = blockstore.get_index(slot).unwrap(); + let dist = if let Some(shred_index) = shred_index { + let shred_count = shred_index.data().num_shreds() as u64; + if last_index.saturating_add(1) < shred_count { + datapoint_error!( + "repair_generic_traversal_error", + ( + "error", + format!( + "last_index + 1 < shred_count. last_index={} shred_count={}", + last_index, shred_count, + ), + String + ), + ); + } + last_index.saturating_add(1).saturating_sub(shred_count) + } else { + if last_index < slot_meta.consumed { + datapoint_error!( + "repair_generic_traversal_error", + ( + "error", + format!( + "last_index < slot_meta.consumed. last_index={} slot_meta.consumed={}", + last_index, + slot_meta.consumed, + ), + String + ), + ); + } + last_index.saturating_sub(slot_meta.consumed) + }; + v.push((slot, dist)); + processed_slots.insert(slot); + } + } + } + v.sort_by(|(_, d1), (_, d2)| d1.cmp(d2)); + + let mut visited = HashSet::new(); + let mut repairs = Vec::new(); + for (slot, _) in v { + if repairs.len() >= limit { + break; + } + // attempt to repair heaviest slots starting with their parents + let path = get_unrepaired_path(slot, blockstore, slot_meta_cache, &mut visited); + for slot in path { + if repairs.len() >= limit { + break; + } + let slot_meta = slot_meta_cache.get(&slot).unwrap().as_ref().unwrap(); + let new_repairs = RepairService::generate_repairs_for_slot( + blockstore, + slot, + slot_meta, + limit - repairs.len(), + ); + repairs.extend(new_repairs); + } + } + + repairs +} + +#[cfg(test)] +pub mod test { + use { + super::*, + solana_ledger::{ + blockstore::{Blockstore, MAX_TURBINE_PROPAGATION_IN_MS}, + get_tmp_ledger_path, + }, + solana_sdk::hash::Hash, + std::{thread::sleep, time::Duration}, + trees::{tr, Tree, TreeWalk}, + }; + + #[test] + fn test_get_unknown_last_index() { + let (blockstore, heaviest_subtree_fork_choice) = setup_forks(); + let last_shred = blockstore.meta(0).unwrap().unwrap().received; + let mut slot_meta_cache = HashMap::default(); + let mut processed_slots = HashSet::default(); + let repairs = get_unknown_last_index( + &heaviest_subtree_fork_choice, + &blockstore, + &mut slot_meta_cache, + &mut processed_slots, + 10, + ); + assert_eq!( + repairs, + [0, 1, 3, 5, 2, 4] + .iter() + .map(|slot| ShredRepairType::HighestShred(*slot, last_shred)) + .collect::>() + ); + } + + #[test] + fn test_get_closest_completion() { + let (blockstore, heaviest_subtree_fork_choice) = setup_forks(); + let mut slot_meta_cache = HashMap::default(); + let mut processed_slots = HashSet::default(); + let repairs = get_closest_completion( + &heaviest_subtree_fork_choice, + &blockstore, + &mut slot_meta_cache, + &mut processed_slots, + 10, + ); + assert_eq!(repairs, []); + + let forks = tr(0) / (tr(1) / (tr(2) / (tr(4))) / (tr(3) / (tr(5)))); + let ledger_path = get_tmp_ledger_path!(); + let blockstore = Blockstore::open(&ledger_path).unwrap(); + add_tree_with_missing_shreds( + &blockstore, + forks.clone(), + false, + true, + 100, + Hash::default(), + ); + let heaviest_subtree_fork_choice = HeaviestSubtreeForkChoice::new_from_tree(forks); + sleep(Duration::from_millis(MAX_TURBINE_PROPAGATION_IN_MS)); + let mut slot_meta_cache = HashMap::default(); + let mut processed_slots = HashSet::default(); + let repairs = get_closest_completion( + &heaviest_subtree_fork_choice, + &blockstore, + &mut slot_meta_cache, + &mut processed_slots, + 2, + ); + assert_eq!( + repairs, + [ShredRepairType::Shred(0, 3), ShredRepairType::Shred(1, 3)] + ); + } + + fn add_tree_with_missing_shreds( + blockstore: &Blockstore, + forks: Tree, + is_orphan: bool, + is_slot_complete: bool, + num_ticks: u64, + starting_hash: Hash, + ) { + let mut walk = TreeWalk::from(forks); + let mut blockhashes = HashMap::new(); + while let Some(visit) = walk.get() { + let slot = *visit.node().data(); + if blockstore.meta(slot).unwrap().is_some() + && blockstore.orphan(slot).unwrap().is_none() + { + // If slot exists in blockstore and is not an orphan, then skip it + walk.forward(); + continue; + } + let parent = walk.get_parent().map(|n| *n.data()); + if parent.is_some() || !is_orphan { + let parent_hash = parent + // parent won't exist for first node in a tree where + // `is_orphan == true` + .and_then(|parent| blockhashes.get(&parent)) + .unwrap_or(&starting_hash); + let entries = solana_entry::entry::create_ticks( + num_ticks * (std::cmp::max(1, slot - parent.unwrap_or(slot))), + 0, + *parent_hash, + ); + blockhashes.insert(slot, entries.last().unwrap().hash); + + let mut shreds = solana_ledger::blockstore::entries_to_test_shreds( + entries.clone(), + slot, + parent.unwrap_or(slot), + is_slot_complete, + 0, + ); + + // remove next to last shred + let shred = shreds.pop().unwrap(); + shreds.pop().unwrap(); + shreds.push(shred); + + blockstore.insert_shreds(shreds, None, false).unwrap(); + } + walk.forward(); + } + } + + fn setup_forks() -> (Blockstore, HeaviestSubtreeForkChoice) { + /* + Build fork structure: + slot 0 + | + slot 1 + / \ + slot 2 | + | slot 3 + slot 4 | + slot 5 + */ + + let forks = tr(0) / (tr(1) / (tr(2) / (tr(4))) / (tr(3) / (tr(5)))); + let ledger_path = get_tmp_ledger_path!(); + let blockstore = Blockstore::open(&ledger_path).unwrap(); + blockstore.add_tree(forks.clone(), false, false, 2, Hash::default()); + + (blockstore, HeaviestSubtreeForkChoice::new_from_tree(forks)) + } +} diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index ca5f86ef93e1f5..98e7d0c21e36c4 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -1345,14 +1345,14 @@ impl Blockstore { // Check that we do not receive shred_index >= than the last_index // for the slot let last_index = slot_meta.last_index; - if shred_index >= last_index { + if last_index.map(|ix| shred_index >= ix).unwrap_or_default() { let leader_pubkey = leader_schedule .and_then(|leader_schedule| leader_schedule.slot_leader_at(slot, None)); let ending_shred: Cow> = self.get_data_shred_from_just_inserted_or_db( just_inserted_data_shreds, slot, - last_index, + last_index.unwrap(), ); if self @@ -1371,7 +1371,7 @@ impl Blockstore { ( "error", format!( - "Leader {:?}, slot {}: received index {} >= slot.last_index {}, shred_source: {:?}", + "Leader {:?}, slot {}: received index {} >= slot.last_index {:?}, shred_source: {:?}", leader_pubkey, slot, shred_index, last_index, shred_source ), String @@ -1516,7 +1516,14 @@ impl Blockstore { i64 ), ("slot", slot_meta.slot, i64), - ("last_index", slot_meta.last_index, i64), + ( + "last_index", + slot_meta + .last_index + .and_then(|ix| i64::try_from(ix).ok()) + .unwrap_or(-1), + i64 + ), ("num_repaired", num_repaired, i64), ("num_recovered", num_recovered, i64), ); @@ -1548,7 +1555,8 @@ impl Blockstore { .collect() } - pub fn get_data_shreds( + #[cfg(test)] + fn get_data_shreds( &self, slot: Slot, from_index: u64, @@ -3157,20 +3165,11 @@ fn update_slot_meta( slot_meta.first_shred_timestamp = timestamp() - slot_time_elapsed; } slot_meta.consumed = new_consumed; - slot_meta.last_index = { - // If the last index in the slot hasn't been set before, then - // set it to this shred index - if slot_meta.last_index == std::u64::MAX { - if is_last_in_slot { - u64::from(index) - } else { - std::u64::MAX - } - } else { - slot_meta.last_index - } - }; - + // If the last index in the slot hasn't been set before, then + // set it to this shred index + if is_last_in_slot && slot_meta.last_index.is_none() { + slot_meta.last_index = Some(u64::from(index)); + } update_completed_data_indexes( is_last_in_slot || is_last_in_data, index, @@ -3944,6 +3943,7 @@ pub mod tests { //let mut shreds_per_slot = 0 as u64; let mut shreds_per_slot = vec![]; +<<<<<<< HEAD for i in 0..num_slots { let mut new_ticks = create_ticks(ticks_per_slot, 0, Hash::default()); let num_shreds = ledger @@ -3961,6 +3961,48 @@ pub mod tests { .unwrap() as u64; shreds_per_slot.push(num_shreds); ticks.append(&mut new_ticks); +======= + let ticks_per_slot = 10; + let num_slots = 10; + let mut ticks = vec![]; + //let mut shreds_per_slot = 0 as u64; + let mut shreds_per_slot = vec![]; + + for i in 0..num_slots { + let mut new_ticks = create_ticks(ticks_per_slot, 0, Hash::default()); + let num_shreds = blockstore + .write_entries( + i, + 0, + 0, + ticks_per_slot, + Some(i.saturating_sub(1)), + true, + &Arc::new(Keypair::new()), + new_ticks.clone(), + 0, + ) + .unwrap() as u64; + shreds_per_slot.push(num_shreds); + ticks.append(&mut new_ticks); + } + + for i in 0..num_slots { + let meta = blockstore.meta(i).unwrap().unwrap(); + let num_shreds = shreds_per_slot[i as usize]; + assert_eq!(meta.consumed, num_shreds); + assert_eq!(meta.received, num_shreds); + assert_eq!(meta.last_index, Some(num_shreds - 1)); + if i == num_slots - 1 { + assert!(meta.next_slots.is_empty()); + } else { + assert_eq!(meta.next_slots, vec![i + 1]); + } + if i == 0 { + assert_eq!(meta.parent_slot, 0); + } else { + assert_eq!(meta.parent_slot, i - 1); +>>>>>>> e08139f94 (uses Option for SlotMeta.last_index (#21775)) } for i in 0..num_slots { @@ -4201,7 +4243,7 @@ pub mod tests { assert_eq!(meta.consumed, num_shreds); assert_eq!(meta.received, num_shreds); assert_eq!(meta.parent_slot, 0); - assert_eq!(meta.last_index, num_shreds - 1); + assert_eq!(meta.last_index, Some(num_shreds - 1)); assert!(meta.next_slots.is_empty()); assert!(meta.is_connected); @@ -4230,7 +4272,7 @@ pub mod tests { .meta(0) .unwrap() .expect("Expected metadata object to exist"); - assert_eq!(meta.last_index, num_shreds - 1); + assert_eq!(meta.last_index, Some(num_shreds - 1)); if i != 0 { assert_eq!(result.len(), 0); assert!(meta.consumed == 0 && meta.received == num_shreds as u64); @@ -4438,10 +4480,36 @@ pub mod tests { let meta = blockstore.meta(slot).unwrap().unwrap(); assert_eq!(meta.received, num_shreds); +<<<<<<< HEAD assert_eq!(meta.consumed, num_shreds); assert_eq!(meta.parent_slot, parent_slot); assert_eq!(meta.last_index, num_shreds - 1); } +======= + } else { + trace!("got here"); + assert_eq!(meta.received, num_shreds - 1); + } + assert_eq!(meta.consumed, 0); + if num_shreds % 2 == 0 { + assert_eq!(meta.last_index, Some(num_shreds - 1)); + } else { + assert_eq!(meta.last_index, None); + } + + blockstore.insert_shreds(even_shreds, None, false).unwrap(); + + assert_eq!( + blockstore.get_slot_entries(slot, 0).unwrap(), + original_entries, + ); + + let meta = blockstore.meta(slot).unwrap().unwrap(); + assert_eq!(meta.received, num_shreds); + assert_eq!(meta.consumed, num_shreds); + assert_eq!(meta.parent_slot, parent_slot); + assert_eq!(meta.last_index, Some(num_shreds - 1)); +>>>>>>> e08139f94 (uses Option for SlotMeta.last_index (#21775)) } Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); @@ -4697,6 +4765,7 @@ pub mod tests { assert_eq!(s1.parent_slot, 0); assert_eq!(s1.last_index, shreds_per_slot as u64 - 1); +<<<<<<< HEAD // 2) Write to the second slot let shreds2 = shreds .drain(shreds_per_slot..2 * shreds_per_slot) @@ -4734,6 +4803,56 @@ pub mod tests { assert_eq!(s.last_index, shreds_per_slot as u64 - 1); assert!(s.is_connected); } +======= + // 1) Write to the first slot + let shreds1 = shreds + .drain(shreds_per_slot..2 * shreds_per_slot) + .collect_vec(); + blockstore.insert_shreds(shreds1, None, false).unwrap(); + let s1 = blockstore.meta(1).unwrap().unwrap(); + assert!(s1.next_slots.is_empty()); + // Slot 1 is not trunk because slot 0 hasn't been inserted yet + assert!(!s1.is_connected); + assert_eq!(s1.parent_slot, 0); + assert_eq!(s1.last_index, Some(shreds_per_slot as u64 - 1)); + + // 2) Write to the second slot + let shreds2 = shreds + .drain(shreds_per_slot..2 * shreds_per_slot) + .collect_vec(); + blockstore.insert_shreds(shreds2, None, false).unwrap(); + let s2 = blockstore.meta(2).unwrap().unwrap(); + assert!(s2.next_slots.is_empty()); + // Slot 2 is not trunk because slot 0 hasn't been inserted yet + assert!(!s2.is_connected); + assert_eq!(s2.parent_slot, 1); + assert_eq!(s2.last_index, Some(shreds_per_slot as u64 - 1)); + + // Check the first slot again, it should chain to the second slot, + // but still isn't part of the trunk + let s1 = blockstore.meta(1).unwrap().unwrap(); + assert_eq!(s1.next_slots, vec![2]); + assert!(!s1.is_connected); + assert_eq!(s1.parent_slot, 0); + assert_eq!(s1.last_index, Some(shreds_per_slot as u64 - 1)); + + // 3) Write to the zeroth slot, check that every slot + // is now part of the trunk + blockstore.insert_shreds(shreds, None, false).unwrap(); + for i in 0..3 { + let s = blockstore.meta(i).unwrap().unwrap(); + // The last slot will not chain to any other slots + if i != 2 { + assert_eq!(s.next_slots, vec![i + 1]); + } + if i == 0 { + assert_eq!(s.parent_slot, 0); + } else { + assert_eq!(s.parent_slot, i - 1); + } + assert_eq!(s.last_index, Some(shreds_per_slot as u64 - 1)); + assert!(s.is_connected); +>>>>>>> e08139f94 (uses Option for SlotMeta.last_index (#21775)) } Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } @@ -4819,7 +4938,33 @@ pub mod tests { } } +<<<<<<< HEAD Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); +======= + // Write the shreds for the other half of the slots that we didn't insert earlier + blockstore + .insert_shreds(missing_slots, None, false) + .unwrap(); + + for i in 0..num_slots { + // Check that all the slots chain correctly once the missing slots + // have been filled + let s = blockstore.meta(i as u64).unwrap().unwrap(); + if i != num_slots - 1 { + assert_eq!(s.next_slots, vec![i as u64 + 1]); + } else { + assert!(s.next_slots.is_empty()); + } + + if i == 0 { + assert_eq!(s.parent_slot, 0); + } else { + assert_eq!(s.parent_slot, i - 1); + } + assert_eq!(s.last_index, Some(shreds_per_slot as u64 - 1)); + assert!(s.is_connected); + } +>>>>>>> e08139f94 (uses Option for SlotMeta.last_index (#21775)) } #[test] @@ -4876,12 +5021,16 @@ pub mod tests { } } +<<<<<<< HEAD // Iteratively finish every 3rd slot, and check that all slots up to and including // slot_index + 3 become part of the trunk for slot_index in 0..num_slots { if slot_index % 3 == 0 { let shred = missing_shreds.remove(0); blockstore.insert_shreds(vec![shred], None, false).unwrap(); +======= + assert_eq!(s.last_index, Some(shreds_per_slot as u64 - 1)); +>>>>>>> e08139f94 (uses Option for SlotMeta.last_index (#21775)) for i in 0..num_slots { let s = blockstore.meta(i as u64).unwrap().unwrap(); @@ -4904,6 +5053,22 @@ pub mod tests { assert_eq!(s.last_index, shreds_per_slot as u64 - 1); } +<<<<<<< HEAD +======= + if i <= slot_index as u64 + 3 { + assert!(s.is_connected); + } else { + assert!(!s.is_connected); + } + + if i == 0 { + assert_eq!(s.parent_slot, 0); + } else { + assert_eq!(s.parent_slot, i - 1); + } + + assert_eq!(s.last_index, Some(shreds_per_slot as u64 - 1)); +>>>>>>> e08139f94 (uses Option for SlotMeta.last_index (#21775)) } } } @@ -5095,6 +5260,7 @@ pub mod tests { vec![0] ); +<<<<<<< HEAD // Write some slot that also chains to existing slots and orphan, // nothing should change let (shred4, _) = make_slot_entries(4, 0, 1); @@ -5154,6 +5320,14 @@ pub mod tests { // Write shreds to the database if should_bulk_write { blockstore.insert_shreds(shreds, None, false).unwrap(); +======= + let meta = blockstore.meta(i).unwrap().unwrap(); + assert_eq!(meta.received, 1); + assert_eq!(meta.last_index, Some(0)); + if i != 0 { + assert_eq!(meta.parent_slot, i - 1); + assert_eq!(meta.consumed, 1); +>>>>>>> e08139f94 (uses Option for SlotMeta.last_index (#21775)) } else { for _ in 0..num_shreds { let shred = shreds.remove(0); @@ -5496,6 +5670,7 @@ pub mod tests { } else { panic!("Shred in unexpected format") } +<<<<<<< HEAD assert!(!blockstore.should_insert_data_shred( &shred7, &slot_meta, @@ -5504,6 +5679,29 @@ pub mod tests { None, ShredSource::Repaired, )); +======= + }; + assert!(!blockstore.should_insert_data_shred( + &shred7, + &slot_meta, + &HashMap::new(), + &last_root, + None, + ShredSource::Repaired, + )); + assert!(blockstore.has_duplicate_shreds_in_slot(0)); + + // Insert all pending shreds + let mut shred8 = shreds[8].clone(); + blockstore.insert_shreds(shreds, None, false).unwrap(); + let slot_meta = blockstore.meta(0).unwrap().unwrap(); + + // Trying to insert a shred with index > the "is_last" shred should fail + if shred8.is_data() { + shred8.set_slot(slot_meta.last_index.unwrap() + 1); + } else { + panic!("Shred in unexpected format") +>>>>>>> e08139f94 (uses Option for SlotMeta.last_index (#21775)) } Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } @@ -5771,7 +5969,7 @@ pub mod tests { assert_eq!(slot_meta.consumed, num_shreds); assert_eq!(slot_meta.received, num_shreds); - assert_eq!(slot_meta.last_index, num_shreds - 1); + assert_eq!(slot_meta.last_index, Some(num_shreds - 1)); assert!(slot_meta.is_full()); let (shreds, _) = make_slot_entries(0, 0, 22); @@ -5780,7 +5978,7 @@ pub mod tests { assert_eq!(slot_meta.consumed, num_shreds); assert_eq!(slot_meta.received, num_shreds); - assert_eq!(slot_meta.last_index, num_shreds - 1); + assert_eq!(slot_meta.last_index, Some(num_shreds - 1)); assert!(slot_meta.is_full()); assert!(blockstore.has_duplicate_shreds_in_slot(0)); @@ -8559,11 +8757,33 @@ pub mod tests { assert_eq!(meta.consumed, num_shreds); assert_eq!(meta.received, num_shreds); assert_eq!(meta.parent_slot, 0); +<<<<<<< HEAD assert_eq!(meta.last_index, num_shreds - 1); assert!(blockstore.is_full(0)); assert!(!blockstore.is_dead(0)); } Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); +======= + assert_eq!(meta.last_index, Some(last_index)); + assert!(!blockstore.is_full(0)); + } + + let duplicate_shreds = entries_to_test_shreds(original_entries.clone(), 0, 0, true, 0); + let num_shreds = duplicate_shreds.len() as u64; + blockstore + .insert_shreds(duplicate_shreds, None, false) + .unwrap(); + + assert_eq!(blockstore.get_slot_entries(0, 0).unwrap(), original_entries); + + let meta = blockstore.meta(0).unwrap().unwrap(); + assert_eq!(meta.consumed, num_shreds); + assert_eq!(meta.received, num_shreds); + assert_eq!(meta.parent_slot, 0); + assert_eq!(meta.last_index, Some(num_shreds - 1)); + assert!(blockstore.is_full(0)); + assert!(!blockstore.is_dead(0)); +>>>>>>> e08139f94 (uses Option for SlotMeta.last_index (#21775)) } #[test] diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index aa79fc097f5b22..e6d6a365bda205 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -3,7 +3,7 @@ use { erasure::ErasureConfig, shred::{Shred, ShredType}, }, - serde::{Deserialize, Serialize}, + serde::{Deserialize, Deserializer, Serialize, Serializer}, solana_sdk::{clock::Slot, hash::Hash}, std::{ collections::BTreeSet, @@ -27,8 +27,10 @@ pub struct SlotMeta { // The timestamp of the first time a shred was added for this slot pub first_shred_timestamp: u64, // The index of the shred that is flagged as the last shred for this slot. - pub last_index: u64, + #[serde(with = "serde_compat")] + pub last_index: Option, // The slot height of the block this one derives from. + // TODO use Option instead. pub parent_slot: Slot, // The list of slots, each of which contains a block that derives // from this one. @@ -40,6 +42,27 @@ pub struct SlotMeta { pub completed_data_indexes: BTreeSet, } +// Serde implementation of serialize and deserialize for Option +// where None is represented as u64::MAX; for backward compatibility. +mod serde_compat { + use super::*; + + pub(super) fn serialize(val: &Option, serializer: S) -> Result + where + S: Serializer, + { + val.unwrap_or(u64::MAX).serialize(serializer) + } + + pub(super) fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let val = u64::deserialize(deserializer)?; + Ok((val != u64::MAX).then(|| val)) + } +} + #[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] /// Index recording presence/absence of shreds pub struct Index { @@ -168,30 +191,34 @@ impl ShredIndex { impl SlotMeta { pub fn is_full(&self) -> bool { - // last_index is std::u64::MAX when it has no information about how + // last_index is None when it has no information about how // many shreds will fill this slot. // Note: A full slot with zero shreds is not possible. - if self.last_index == std::u64::MAX { - return false; - } - // Should never happen - if self.consumed > self.last_index + 1 { + if self + .last_index + .map(|ix| self.consumed > ix + 1) + .unwrap_or_default() + { datapoint_error!( "blockstore_error", ( "error", format!( - "Observed a slot meta with consumed: {} > meta.last_index + 1: {}", + "Observed a slot meta with consumed: {} > meta.last_index + 1: {:?}", self.consumed, - self.last_index + 1 + self.last_index.map(|ix| ix + 1), ), String ) ); } +<<<<<<< HEAD self.consumed == self.last_index + 1 +======= + Some(self.consumed) == self.last_index.map(|ix| ix + 1) +>>>>>>> e08139f94 (uses Option for SlotMeta.last_index (#21775)) } pub fn is_parent_set(&self) -> bool { @@ -209,7 +236,6 @@ impl SlotMeta { slot, parent_slot, is_connected: slot == 0, - last_index: std::u64::MAX, ..SlotMeta::default() } } From 0bb9a3969c1945e45e9bbdd7c2076a95f580d13c Mon Sep 17 00:00:00 2001 From: behzad nouri Date: Thu, 3 Feb 2022 13:37:52 -0500 Subject: [PATCH 2/2] removes mergify merge conflicts --- core/src/repair_generic_traversal.rs | 343 --------------------------- ledger/src/blockstore.rs | 251 ++------------------ ledger/src/blockstore_meta.rs | 4 - 3 files changed, 15 insertions(+), 583 deletions(-) delete mode 100644 core/src/repair_generic_traversal.rs diff --git a/core/src/repair_generic_traversal.rs b/core/src/repair_generic_traversal.rs deleted file mode 100644 index b5d78667828f3a..00000000000000 --- a/core/src/repair_generic_traversal.rs +++ /dev/null @@ -1,343 +0,0 @@ -use { - crate::{ - heaviest_subtree_fork_choice::HeaviestSubtreeForkChoice, repair_service::RepairService, - serve_repair::ShredRepairType, tree_diff::TreeDiff, - }, - solana_ledger::{blockstore::Blockstore, blockstore_meta::SlotMeta}, - solana_sdk::{clock::Slot, hash::Hash}, - std::collections::{HashMap, HashSet}, -}; - -struct GenericTraversal<'a> { - tree: &'a HeaviestSubtreeForkChoice, - pending: Vec, -} - -impl<'a> GenericTraversal<'a> { - pub fn new(tree: &'a HeaviestSubtreeForkChoice) -> Self { - Self { - tree, - pending: vec![tree.root().0], - } - } -} - -impl<'a> Iterator for GenericTraversal<'a> { - type Item = Slot; - fn next(&mut self) -> Option { - let next = self.pending.pop(); - if let Some(slot) = next { - let children: Vec<_> = self - .tree - .children(&(slot, Hash::default())) - .unwrap() - .iter() - .map(|(child_slot, _)| *child_slot) - .collect(); - self.pending.extend(children); - } - next - } -} - -pub fn get_unknown_last_index( - tree: &HeaviestSubtreeForkChoice, - blockstore: &Blockstore, - slot_meta_cache: &mut HashMap>, - processed_slots: &mut HashSet, - limit: usize, -) -> Vec { - let iter = GenericTraversal::new(tree); - let mut unknown_last = Vec::new(); - for slot in iter { - if processed_slots.contains(&slot) { - continue; - } - let slot_meta = slot_meta_cache - .entry(slot) - .or_insert_with(|| blockstore.meta(slot).unwrap()); - if let Some(slot_meta) = slot_meta { - if slot_meta.last_index.is_none() { - let shred_index = blockstore.get_index(slot).unwrap(); - let num_processed_shreds = if let Some(shred_index) = shred_index { - shred_index.data().num_shreds() as u64 - } else { - slot_meta.consumed - }; - unknown_last.push((slot, slot_meta.received, num_processed_shreds)); - processed_slots.insert(slot); - } - } - } - // prioritize slots with more received shreds - unknown_last.sort_by(|(_, _, count1), (_, _, count2)| count2.cmp(count1)); - unknown_last - .iter() - .take(limit) - .map(|(slot, received, _)| ShredRepairType::HighestShred(*slot, *received)) - .collect() -} - -fn get_unrepaired_path( - start_slot: Slot, - blockstore: &Blockstore, - slot_meta_cache: &mut HashMap>, - visited: &mut HashSet, -) -> Vec { - let mut path = Vec::new(); - let mut slot = start_slot; - while !visited.contains(&slot) { - visited.insert(slot); - let slot_meta = slot_meta_cache - .entry(slot) - .or_insert_with(|| blockstore.meta(slot).unwrap()); - if let Some(slot_meta) = slot_meta { - if slot_meta.is_full() { - break; - } - path.push(slot); - slot = slot_meta.parent_slot; - } - } - path.reverse(); - path -} - -pub fn get_closest_completion( - tree: &HeaviestSubtreeForkChoice, - blockstore: &Blockstore, - slot_meta_cache: &mut HashMap>, - processed_slots: &mut HashSet, - limit: usize, -) -> Vec { - let mut v: Vec<(Slot, u64)> = Vec::default(); - let iter = GenericTraversal::new(tree); - for slot in iter { - if processed_slots.contains(&slot) { - continue; - } - let slot_meta = slot_meta_cache - .entry(slot) - .or_insert_with(|| blockstore.meta(slot).unwrap()); - if let Some(slot_meta) = slot_meta { - if slot_meta.is_full() { - continue; - } - if let Some(last_index) = slot_meta.last_index { - let shred_index = blockstore.get_index(slot).unwrap(); - let dist = if let Some(shred_index) = shred_index { - let shred_count = shred_index.data().num_shreds() as u64; - if last_index.saturating_add(1) < shred_count { - datapoint_error!( - "repair_generic_traversal_error", - ( - "error", - format!( - "last_index + 1 < shred_count. last_index={} shred_count={}", - last_index, shred_count, - ), - String - ), - ); - } - last_index.saturating_add(1).saturating_sub(shred_count) - } else { - if last_index < slot_meta.consumed { - datapoint_error!( - "repair_generic_traversal_error", - ( - "error", - format!( - "last_index < slot_meta.consumed. last_index={} slot_meta.consumed={}", - last_index, - slot_meta.consumed, - ), - String - ), - ); - } - last_index.saturating_sub(slot_meta.consumed) - }; - v.push((slot, dist)); - processed_slots.insert(slot); - } - } - } - v.sort_by(|(_, d1), (_, d2)| d1.cmp(d2)); - - let mut visited = HashSet::new(); - let mut repairs = Vec::new(); - for (slot, _) in v { - if repairs.len() >= limit { - break; - } - // attempt to repair heaviest slots starting with their parents - let path = get_unrepaired_path(slot, blockstore, slot_meta_cache, &mut visited); - for slot in path { - if repairs.len() >= limit { - break; - } - let slot_meta = slot_meta_cache.get(&slot).unwrap().as_ref().unwrap(); - let new_repairs = RepairService::generate_repairs_for_slot( - blockstore, - slot, - slot_meta, - limit - repairs.len(), - ); - repairs.extend(new_repairs); - } - } - - repairs -} - -#[cfg(test)] -pub mod test { - use { - super::*, - solana_ledger::{ - blockstore::{Blockstore, MAX_TURBINE_PROPAGATION_IN_MS}, - get_tmp_ledger_path, - }, - solana_sdk::hash::Hash, - std::{thread::sleep, time::Duration}, - trees::{tr, Tree, TreeWalk}, - }; - - #[test] - fn test_get_unknown_last_index() { - let (blockstore, heaviest_subtree_fork_choice) = setup_forks(); - let last_shred = blockstore.meta(0).unwrap().unwrap().received; - let mut slot_meta_cache = HashMap::default(); - let mut processed_slots = HashSet::default(); - let repairs = get_unknown_last_index( - &heaviest_subtree_fork_choice, - &blockstore, - &mut slot_meta_cache, - &mut processed_slots, - 10, - ); - assert_eq!( - repairs, - [0, 1, 3, 5, 2, 4] - .iter() - .map(|slot| ShredRepairType::HighestShred(*slot, last_shred)) - .collect::>() - ); - } - - #[test] - fn test_get_closest_completion() { - let (blockstore, heaviest_subtree_fork_choice) = setup_forks(); - let mut slot_meta_cache = HashMap::default(); - let mut processed_slots = HashSet::default(); - let repairs = get_closest_completion( - &heaviest_subtree_fork_choice, - &blockstore, - &mut slot_meta_cache, - &mut processed_slots, - 10, - ); - assert_eq!(repairs, []); - - let forks = tr(0) / (tr(1) / (tr(2) / (tr(4))) / (tr(3) / (tr(5)))); - let ledger_path = get_tmp_ledger_path!(); - let blockstore = Blockstore::open(&ledger_path).unwrap(); - add_tree_with_missing_shreds( - &blockstore, - forks.clone(), - false, - true, - 100, - Hash::default(), - ); - let heaviest_subtree_fork_choice = HeaviestSubtreeForkChoice::new_from_tree(forks); - sleep(Duration::from_millis(MAX_TURBINE_PROPAGATION_IN_MS)); - let mut slot_meta_cache = HashMap::default(); - let mut processed_slots = HashSet::default(); - let repairs = get_closest_completion( - &heaviest_subtree_fork_choice, - &blockstore, - &mut slot_meta_cache, - &mut processed_slots, - 2, - ); - assert_eq!( - repairs, - [ShredRepairType::Shred(0, 3), ShredRepairType::Shred(1, 3)] - ); - } - - fn add_tree_with_missing_shreds( - blockstore: &Blockstore, - forks: Tree, - is_orphan: bool, - is_slot_complete: bool, - num_ticks: u64, - starting_hash: Hash, - ) { - let mut walk = TreeWalk::from(forks); - let mut blockhashes = HashMap::new(); - while let Some(visit) = walk.get() { - let slot = *visit.node().data(); - if blockstore.meta(slot).unwrap().is_some() - && blockstore.orphan(slot).unwrap().is_none() - { - // If slot exists in blockstore and is not an orphan, then skip it - walk.forward(); - continue; - } - let parent = walk.get_parent().map(|n| *n.data()); - if parent.is_some() || !is_orphan { - let parent_hash = parent - // parent won't exist for first node in a tree where - // `is_orphan == true` - .and_then(|parent| blockhashes.get(&parent)) - .unwrap_or(&starting_hash); - let entries = solana_entry::entry::create_ticks( - num_ticks * (std::cmp::max(1, slot - parent.unwrap_or(slot))), - 0, - *parent_hash, - ); - blockhashes.insert(slot, entries.last().unwrap().hash); - - let mut shreds = solana_ledger::blockstore::entries_to_test_shreds( - entries.clone(), - slot, - parent.unwrap_or(slot), - is_slot_complete, - 0, - ); - - // remove next to last shred - let shred = shreds.pop().unwrap(); - shreds.pop().unwrap(); - shreds.push(shred); - - blockstore.insert_shreds(shreds, None, false).unwrap(); - } - walk.forward(); - } - } - - fn setup_forks() -> (Blockstore, HeaviestSubtreeForkChoice) { - /* - Build fork structure: - slot 0 - | - slot 1 - / \ - slot 2 | - | slot 3 - slot 4 | - slot 5 - */ - - let forks = tr(0) / (tr(1) / (tr(2) / (tr(4))) / (tr(3) / (tr(5)))); - let ledger_path = get_tmp_ledger_path!(); - let blockstore = Blockstore::open(&ledger_path).unwrap(); - blockstore.add_tree(forks.clone(), false, false, 2, Hash::default()); - - (blockstore, HeaviestSubtreeForkChoice::new_from_tree(forks)) - } -} diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 98e7d0c21e36c4..331ec124721b24 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -46,7 +46,7 @@ use { cell::RefCell, cmp, collections::{hash_map::Entry as HashMapEntry, BTreeMap, BTreeSet, HashMap, HashSet}, - convert::TryInto, + convert::{TryFrom, TryInto}, fs, io::{Error as IoError, ErrorKind}, path::{Path, PathBuf}, @@ -3943,7 +3943,6 @@ pub mod tests { //let mut shreds_per_slot = 0 as u64; let mut shreds_per_slot = vec![]; -<<<<<<< HEAD for i in 0..num_slots { let mut new_ticks = create_ticks(ticks_per_slot, 0, Hash::default()); let num_shreds = ledger @@ -3961,48 +3960,6 @@ pub mod tests { .unwrap() as u64; shreds_per_slot.push(num_shreds); ticks.append(&mut new_ticks); -======= - let ticks_per_slot = 10; - let num_slots = 10; - let mut ticks = vec![]; - //let mut shreds_per_slot = 0 as u64; - let mut shreds_per_slot = vec![]; - - for i in 0..num_slots { - let mut new_ticks = create_ticks(ticks_per_slot, 0, Hash::default()); - let num_shreds = blockstore - .write_entries( - i, - 0, - 0, - ticks_per_slot, - Some(i.saturating_sub(1)), - true, - &Arc::new(Keypair::new()), - new_ticks.clone(), - 0, - ) - .unwrap() as u64; - shreds_per_slot.push(num_shreds); - ticks.append(&mut new_ticks); - } - - for i in 0..num_slots { - let meta = blockstore.meta(i).unwrap().unwrap(); - let num_shreds = shreds_per_slot[i as usize]; - assert_eq!(meta.consumed, num_shreds); - assert_eq!(meta.received, num_shreds); - assert_eq!(meta.last_index, Some(num_shreds - 1)); - if i == num_slots - 1 { - assert!(meta.next_slots.is_empty()); - } else { - assert_eq!(meta.next_slots, vec![i + 1]); - } - if i == 0 { - assert_eq!(meta.parent_slot, 0); - } else { - assert_eq!(meta.parent_slot, i - 1); ->>>>>>> e08139f94 (uses Option for SlotMeta.last_index (#21775)) } for i in 0..num_slots { @@ -4010,7 +3967,7 @@ pub mod tests { let num_shreds = shreds_per_slot[i as usize]; assert_eq!(meta.consumed, num_shreds); assert_eq!(meta.received, num_shreds); - assert_eq!(meta.last_index, num_shreds - 1); + assert_eq!(meta.last_index, Some(num_shreds - 1)); if i == num_slots - 1 { assert!(meta.next_slots.is_empty()); } else { @@ -4466,9 +4423,9 @@ pub mod tests { } assert_eq!(meta.consumed, 0); if num_shreds % 2 == 0 { - assert_eq!(meta.last_index, num_shreds - 1); + assert_eq!(meta.last_index, Some(num_shreds - 1)); } else { - assert_eq!(meta.last_index, std::u64::MAX); + assert_eq!(meta.last_index, None); } blockstore.insert_shreds(even_shreds, None, false).unwrap(); @@ -4480,36 +4437,10 @@ pub mod tests { let meta = blockstore.meta(slot).unwrap().unwrap(); assert_eq!(meta.received, num_shreds); -<<<<<<< HEAD assert_eq!(meta.consumed, num_shreds); assert_eq!(meta.parent_slot, parent_slot); - assert_eq!(meta.last_index, num_shreds - 1); - } -======= - } else { - trace!("got here"); - assert_eq!(meta.received, num_shreds - 1); - } - assert_eq!(meta.consumed, 0); - if num_shreds % 2 == 0 { assert_eq!(meta.last_index, Some(num_shreds - 1)); - } else { - assert_eq!(meta.last_index, None); } - - blockstore.insert_shreds(even_shreds, None, false).unwrap(); - - assert_eq!( - blockstore.get_slot_entries(slot, 0).unwrap(), - original_entries, - ); - - let meta = blockstore.meta(slot).unwrap().unwrap(); - assert_eq!(meta.received, num_shreds); - assert_eq!(meta.consumed, num_shreds); - assert_eq!(meta.parent_slot, parent_slot); - assert_eq!(meta.last_index, Some(num_shreds - 1)); ->>>>>>> e08139f94 (uses Option for SlotMeta.last_index (#21775)) } Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); @@ -4763,9 +4694,8 @@ pub mod tests { // Slot 1 is not trunk because slot 0 hasn't been inserted yet assert!(!s1.is_connected); assert_eq!(s1.parent_slot, 0); - assert_eq!(s1.last_index, shreds_per_slot as u64 - 1); + assert_eq!(s1.last_index, Some(shreds_per_slot as u64 - 1)); -<<<<<<< HEAD // 2) Write to the second slot let shreds2 = shreds .drain(shreds_per_slot..2 * shreds_per_slot) @@ -4776,7 +4706,7 @@ pub mod tests { // Slot 2 is not trunk because slot 0 hasn't been inserted yet assert!(!s2.is_connected); assert_eq!(s2.parent_slot, 1); - assert_eq!(s2.last_index, shreds_per_slot as u64 - 1); + assert_eq!(s2.last_index, Some(shreds_per_slot as u64 - 1)); // Check the first slot again, it should chain to the second slot, // but still isn't part of the trunk @@ -4784,7 +4714,7 @@ pub mod tests { assert_eq!(s1.next_slots, vec![2]); assert!(!s1.is_connected); assert_eq!(s1.parent_slot, 0); - assert_eq!(s1.last_index, shreds_per_slot as u64 - 1); + assert_eq!(s1.last_index, Some(shreds_per_slot as u64 - 1)); // 3) Write to the zeroth slot, check that every slot // is now part of the trunk @@ -4800,59 +4730,9 @@ pub mod tests { } else { assert_eq!(s.parent_slot, i - 1); } - assert_eq!(s.last_index, shreds_per_slot as u64 - 1); + assert_eq!(s.last_index, Some(shreds_per_slot as u64 - 1)); assert!(s.is_connected); } -======= - // 1) Write to the first slot - let shreds1 = shreds - .drain(shreds_per_slot..2 * shreds_per_slot) - .collect_vec(); - blockstore.insert_shreds(shreds1, None, false).unwrap(); - let s1 = blockstore.meta(1).unwrap().unwrap(); - assert!(s1.next_slots.is_empty()); - // Slot 1 is not trunk because slot 0 hasn't been inserted yet - assert!(!s1.is_connected); - assert_eq!(s1.parent_slot, 0); - assert_eq!(s1.last_index, Some(shreds_per_slot as u64 - 1)); - - // 2) Write to the second slot - let shreds2 = shreds - .drain(shreds_per_slot..2 * shreds_per_slot) - .collect_vec(); - blockstore.insert_shreds(shreds2, None, false).unwrap(); - let s2 = blockstore.meta(2).unwrap().unwrap(); - assert!(s2.next_slots.is_empty()); - // Slot 2 is not trunk because slot 0 hasn't been inserted yet - assert!(!s2.is_connected); - assert_eq!(s2.parent_slot, 1); - assert_eq!(s2.last_index, Some(shreds_per_slot as u64 - 1)); - - // Check the first slot again, it should chain to the second slot, - // but still isn't part of the trunk - let s1 = blockstore.meta(1).unwrap().unwrap(); - assert_eq!(s1.next_slots, vec![2]); - assert!(!s1.is_connected); - assert_eq!(s1.parent_slot, 0); - assert_eq!(s1.last_index, Some(shreds_per_slot as u64 - 1)); - - // 3) Write to the zeroth slot, check that every slot - // is now part of the trunk - blockstore.insert_shreds(shreds, None, false).unwrap(); - for i in 0..3 { - let s = blockstore.meta(i).unwrap().unwrap(); - // The last slot will not chain to any other slots - if i != 2 { - assert_eq!(s.next_slots, vec![i + 1]); - } - if i == 0 { - assert_eq!(s.parent_slot, 0); - } else { - assert_eq!(s.parent_slot, i - 1); - } - assert_eq!(s.last_index, Some(shreds_per_slot as u64 - 1)); - assert!(s.is_connected); ->>>>>>> e08139f94 (uses Option for SlotMeta.last_index (#21775)) } Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } @@ -4933,38 +4813,12 @@ pub mod tests { } else { assert_eq!(s.parent_slot, i - 1); } - assert_eq!(s.last_index, shreds_per_slot as u64 - 1); + assert_eq!(s.last_index, Some(shreds_per_slot as u64 - 1)); assert!(s.is_connected); } } -<<<<<<< HEAD Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); -======= - // Write the shreds for the other half of the slots that we didn't insert earlier - blockstore - .insert_shreds(missing_slots, None, false) - .unwrap(); - - for i in 0..num_slots { - // Check that all the slots chain correctly once the missing slots - // have been filled - let s = blockstore.meta(i as u64).unwrap().unwrap(); - if i != num_slots - 1 { - assert_eq!(s.next_slots, vec![i as u64 + 1]); - } else { - assert!(s.next_slots.is_empty()); - } - - if i == 0 { - assert_eq!(s.parent_slot, 0); - } else { - assert_eq!(s.parent_slot, i - 1); - } - assert_eq!(s.last_index, Some(shreds_per_slot as u64 - 1)); - assert!(s.is_connected); - } ->>>>>>> e08139f94 (uses Option for SlotMeta.last_index (#21775)) } #[test] @@ -5011,7 +4865,7 @@ pub mod tests { assert_eq!(s.parent_slot, i - 1); } - assert_eq!(s.last_index, shreds_per_slot as u64 - 1); + assert_eq!(s.last_index, Some(shreds_per_slot as u64 - 1)); // Other than slot 0, no slots should be part of the trunk if i != 0 { @@ -5021,16 +4875,12 @@ pub mod tests { } } -<<<<<<< HEAD // Iteratively finish every 3rd slot, and check that all slots up to and including // slot_index + 3 become part of the trunk for slot_index in 0..num_slots { if slot_index % 3 == 0 { let shred = missing_shreds.remove(0); blockstore.insert_shreds(vec![shred], None, false).unwrap(); -======= - assert_eq!(s.last_index, Some(shreds_per_slot as u64 - 1)); ->>>>>>> e08139f94 (uses Option for SlotMeta.last_index (#21775)) for i in 0..num_slots { let s = blockstore.meta(i as u64).unwrap().unwrap(); @@ -5051,24 +4901,8 @@ pub mod tests { assert_eq!(s.parent_slot, i - 1); } - assert_eq!(s.last_index, shreds_per_slot as u64 - 1); - } -<<<<<<< HEAD -======= - if i <= slot_index as u64 + 3 { - assert!(s.is_connected); - } else { - assert!(!s.is_connected); + assert_eq!(s.last_index, Some(shreds_per_slot as u64 - 1)); } - - if i == 0 { - assert_eq!(s.parent_slot, 0); - } else { - assert_eq!(s.parent_slot, i - 1); - } - - assert_eq!(s.last_index, Some(shreds_per_slot as u64 - 1)); ->>>>>>> e08139f94 (uses Option for SlotMeta.last_index (#21775)) } } } @@ -5260,7 +5094,6 @@ pub mod tests { vec![0] ); -<<<<<<< HEAD // Write some slot that also chains to existing slots and orphan, // nothing should change let (shred4, _) = make_slot_entries(4, 0, 1); @@ -5320,14 +5153,6 @@ pub mod tests { // Write shreds to the database if should_bulk_write { blockstore.insert_shreds(shreds, None, false).unwrap(); -======= - let meta = blockstore.meta(i).unwrap().unwrap(); - assert_eq!(meta.received, 1); - assert_eq!(meta.last_index, Some(0)); - if i != 0 { - assert_eq!(meta.parent_slot, i - 1); - assert_eq!(meta.consumed, 1); ->>>>>>> e08139f94 (uses Option for SlotMeta.last_index (#21775)) } else { for _ in 0..num_shreds { let shred = shreds.remove(0); @@ -5343,7 +5168,7 @@ pub mod tests { let meta = blockstore.meta(i).unwrap().unwrap(); assert_eq!(meta.received, 1); - assert_eq!(meta.last_index, 0); + assert_eq!(meta.last_index, Some(0)); if i != 0 { assert_eq!(meta.parent_slot, i - 1); assert_eq!(meta.consumed, 1); @@ -5666,11 +5491,10 @@ pub mod tests { // Trying to insert a shred with index > the "is_last" shred should fail if shred8.is_data() { - shred8.set_slot(slot_meta.last_index + 1); + shred8.set_slot(slot_meta.last_index.unwrap() + 1); } else { panic!("Shred in unexpected format") } -<<<<<<< HEAD assert!(!blockstore.should_insert_data_shred( &shred7, &slot_meta, @@ -5679,29 +5503,6 @@ pub mod tests { None, ShredSource::Repaired, )); -======= - }; - assert!(!blockstore.should_insert_data_shred( - &shred7, - &slot_meta, - &HashMap::new(), - &last_root, - None, - ShredSource::Repaired, - )); - assert!(blockstore.has_duplicate_shreds_in_slot(0)); - - // Insert all pending shreds - let mut shred8 = shreds[8].clone(); - blockstore.insert_shreds(shreds, None, false).unwrap(); - let slot_meta = blockstore.meta(0).unwrap().unwrap(); - - // Trying to insert a shred with index > the "is_last" shred should fail - if shred8.is_data() { - shred8.set_slot(slot_meta.last_index.unwrap() + 1); - } else { - panic!("Shred in unexpected format") ->>>>>>> e08139f94 (uses Option for SlotMeta.last_index (#21775)) } Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } @@ -8741,7 +8542,7 @@ pub mod tests { assert_eq!(meta.consumed, 0); assert_eq!(meta.received, last_index + 1); assert_eq!(meta.parent_slot, 0); - assert_eq!(meta.last_index, last_index); + assert_eq!(meta.last_index, Some(last_index)); assert!(!blockstore.is_full(0)); } @@ -8757,33 +8558,11 @@ pub mod tests { assert_eq!(meta.consumed, num_shreds); assert_eq!(meta.received, num_shreds); assert_eq!(meta.parent_slot, 0); -<<<<<<< HEAD - assert_eq!(meta.last_index, num_shreds - 1); + assert_eq!(meta.last_index, Some(num_shreds - 1)); assert!(blockstore.is_full(0)); assert!(!blockstore.is_dead(0)); } Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); -======= - assert_eq!(meta.last_index, Some(last_index)); - assert!(!blockstore.is_full(0)); - } - - let duplicate_shreds = entries_to_test_shreds(original_entries.clone(), 0, 0, true, 0); - let num_shreds = duplicate_shreds.len() as u64; - blockstore - .insert_shreds(duplicate_shreds, None, false) - .unwrap(); - - assert_eq!(blockstore.get_slot_entries(0, 0).unwrap(), original_entries); - - let meta = blockstore.meta(0).unwrap().unwrap(); - assert_eq!(meta.consumed, num_shreds); - assert_eq!(meta.received, num_shreds); - assert_eq!(meta.parent_slot, 0); - assert_eq!(meta.last_index, Some(num_shreds - 1)); - assert!(blockstore.is_full(0)); - assert!(!blockstore.is_dead(0)); ->>>>>>> e08139f94 (uses Option for SlotMeta.last_index (#21775)) } #[test] diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index e6d6a365bda205..be5864595be9df 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -214,11 +214,7 @@ impl SlotMeta { ); } -<<<<<<< HEAD - self.consumed == self.last_index + 1 -======= Some(self.consumed) == self.last_index.map(|ix| ix + 1) ->>>>>>> e08139f94 (uses Option for SlotMeta.last_index (#21775)) } pub fn is_parent_set(&self) -> bool {