From ab65e12fe3d2f4d4dd5c5e3fc8fd8acb480b11ce Mon Sep 17 00:00:00 2001 From: Steven Czabaniuk Date: Fri, 2 Feb 2024 22:35:54 -0400 Subject: [PATCH 01/12] ledger-tool: Add command to create shreds from bigtable data --- ledger-tool/src/bigtable.rs | 106 +++++++++++++++++++++++++++++++++++- 1 file changed, 104 insertions(+), 2 deletions(-) diff --git a/ledger-tool/src/bigtable.rs b/ledger-tool/src/bigtable.rs index 17f7ba598fa473..a597de7509cf68 100644 --- a/ledger-tool/src/bigtable.rs +++ b/ledger-tool/src/bigtable.rs @@ -22,11 +22,14 @@ use { display::println_transaction, CliBlock, CliTransaction, CliTransactionConfirmation, OutputFormat, }, + solana_entry::entry::Entry, solana_ledger::{ - bigtable_upload::ConfirmedBlockUploadConfig, blockstore::Blockstore, + bigtable_upload::ConfirmedBlockUploadConfig, + blockstore::Blockstore, blockstore_options::AccessType, + shred::{ProcessShredsStats, ReedSolomonCache, Shredder}, }, - solana_sdk::{clock::Slot, pubkey::Pubkey, signature::Signature}, + solana_sdk::{clock::Slot, pubkey::Pubkey, signature::Signature, signer::keypair::Keypair}, solana_storage_bigtable::CredentialType, solana_transaction_status::{ConfirmedBlock, UiTransactionEncoding, VersionedConfirmedBlock}, std::{ @@ -164,6 +167,63 @@ async fn entries( Ok(()) } +async fn shreds( + blockstore: Arc, + starting_slot: Slot, + ending_slot: Slot, + config: solana_storage_bigtable::LedgerStorageConfig, +) -> Result<(), Box> { + let bigtable = solana_storage_bigtable::LedgerStorage::new_with_config(config) + .await + .map_err(|err| format!("Failed to connect to storage: {err:?}"))?; + + // Make the range inclusive of both starting and ending slot + let limit = (ending_slot - starting_slot + 1) as usize; + let mut slots = bigtable.get_confirmed_blocks(starting_slot, limit).await?; + slots.retain(|&slot| slot <= ending_slot); + + let keypair = Keypair::from_bytes(&[0; 64]).unwrap(); + // TODO: parse this / allow command line flag ? + let shred_version = 0; + + for slot in slots.iter() { + let block = bigtable.get_confirmed_block(*slot).await?; + let entry_summaries = bigtable.get_entries(*slot).await?; + let entries: Vec<_> = entry_summaries + .map(|entry_summary| { + let num_hashes = entry_summary.num_hashes; + let hash = entry_summary.hash; + let transactions = block.transactions[entry_summary.starting_transaction_index + ..entry_summary.starting_transaction_index + + entry_summary.num_transactions as usize] + .iter() + .map(|tx_with_meta| tx_with_meta.get_transaction()) + .collect(); + Entry { + num_hashes, + hash, + transactions, + } + }) + .collect(); + + let shredder = Shredder::new(*slot, block.parent_slot, 0, shred_version)?; + let (data_shreds, _coding_shreds) = shredder.entries_to_shreds( + &keypair, + &entries, + true, // last_in_slot + None, // chained_merkle_root + 0, // next_shred_index + 0, // next_code_index + false, // merkle_variant + &ReedSolomonCache::default(), + &mut ProcessShredsStats::default(), + ); + blockstore.insert_shreds(data_shreds, None, false)?; + } + Ok(()) +} + async fn blocks( starting_slot: Slot, limit: usize, @@ -848,6 +908,31 @@ impl BigTableSubCommand for App<'_, '_> { .required(true), ), ) + .subcommand( + SubCommand::with_name("shreds") + .about( + "Get confirmed blocks, shred them and insert the shreds into the \ + local Blockstore", + ) + .arg( + Arg::with_name("starting_slot") + .long("starting-slot") + .validator(is_slot) + .value_name("SLOT") + .takes_value(true) + .required(true) + .help("Reconstruct starting from this slot (inclusive)"), + ) + .arg( + Arg::with_name("ending_slot") + .long("ending-slot") + .validator(is_slot) + .value_name("SLOT") + .takes_value(true) + .required(true) + .help("Reconstruct ending with this slot (inclusive)"), + ), + ) .subcommand( SubCommand::with_name("confirm") .about("Confirm transaction by signature") @@ -1142,6 +1227,23 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) { }; runtime.block_on(entries(slot, output_format, config)) } + ("shreds", Some(arg_matches)) => { + let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot); + let ending_slot = value_t_or_exit!(arg_matches, "ending_slot", Slot); + let blockstore = Arc::new(crate::open_blockstore( + &canonicalize_ledger_path(ledger_path), + arg_matches, + AccessType::Primary, + )); + + let config = solana_storage_bigtable::LedgerStorageConfig { + read_only: true, + instance_name, + app_profile_id, + ..solana_storage_bigtable::LedgerStorageConfig::default() + }; + runtime.block_on(shreds(blockstore, starting_slot, ending_slot, config)) + } ("blocks", Some(arg_matches)) => { let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot); let limit = value_t_or_exit!(arg_matches, "limit", usize); From 322e474087d44d3a1c08b4440e9a32bb2d1f3401 Mon Sep 17 00:00:00 2001 From: Steven Czabaniuk Date: Mon, 5 Feb 2024 15:14:07 -0600 Subject: [PATCH 02/12] Add fallback incase entries data does not exist --- ledger-tool/src/bigtable.rs | 132 +++++++++++++++++++++++++++++------- 1 file changed, 109 insertions(+), 23 deletions(-) diff --git a/ledger-tool/src/bigtable.rs b/ledger-tool/src/bigtable.rs index a597de7509cf68..e243b44bbf8b8b 100644 --- a/ledger-tool/src/bigtable.rs +++ b/ledger-tool/src/bigtable.rs @@ -12,7 +12,7 @@ use { }, crossbeam_channel::unbounded, futures::stream::FuturesUnordered, - log::{debug, error, info}, + log::{debug, error, info, warn}, serde_json::json, solana_clap_utils::{ input_parsers::pubkey_of, @@ -22,14 +22,16 @@ use { display::println_transaction, CliBlock, CliTransaction, CliTransactionConfirmation, OutputFormat, }, - solana_entry::entry::Entry, + solana_entry::entry::{create_ticks, Entry}, solana_ledger::{ bigtable_upload::ConfirmedBlockUploadConfig, blockstore::Blockstore, blockstore_options::AccessType, shred::{ProcessShredsStats, ReedSolomonCache, Shredder}, }, - solana_sdk::{clock::Slot, pubkey::Pubkey, signature::Signature, signer::keypair::Keypair}, + solana_sdk::{ + clock::Slot, hash::Hash, pubkey::Pubkey, signature::Signature, signer::keypair::Keypair, + }, solana_storage_bigtable::CredentialType, solana_transaction_status::{ConfirmedBlock, UiTransactionEncoding, VersionedConfirmedBlock}, std::{ @@ -171,6 +173,7 @@ async fn shreds( blockstore: Arc, starting_slot: Slot, ending_slot: Slot, + allow_dummy_poh: bool, config: solana_storage_bigtable::LedgerStorageConfig, ) -> Result<(), Box> { let bigtable = solana_storage_bigtable::LedgerStorage::new_with_config(config) @@ -182,30 +185,107 @@ async fn shreds( let mut slots = bigtable.get_confirmed_blocks(starting_slot, limit).await?; slots.retain(|&slot| slot <= ending_slot); - let keypair = Keypair::from_bytes(&[0; 64]).unwrap(); - // TODO: parse this / allow command line flag ? + let keypair = Keypair::from_bytes(&[0; 64])?; + // TODO: parse this from CLI ? let shred_version = 0; + // TODO: parse from CLI OR extract from genesis + let num_ticks_per_slot = 64; + // TODO: parse from CLI OR extract from Bank; tick rate changed recently + let num_hashes_per_tick = 12500; for slot in slots.iter() { let block = bigtable.get_confirmed_block(*slot).await?; - let entry_summaries = bigtable.get_entries(*slot).await?; - let entries: Vec<_> = entry_summaries - .map(|entry_summary| { - let num_hashes = entry_summary.num_hashes; - let hash = entry_summary.hash; - let transactions = block.transactions[entry_summary.starting_transaction_index - ..entry_summary.starting_transaction_index - + entry_summary.num_transactions as usize] - .iter() - .map(|tx_with_meta| tx_with_meta.get_transaction()) - .collect(); - Entry { - num_hashes, - hash, - transactions, + let entry_summaries = match bigtable.get_entries(*slot).await { + Ok(summaries) => Some(summaries), + Err(err) => { + let err_msg = format!("Failed to get PoH entry data for {slot}: {err}"); + + if allow_dummy_poh { + warn!("{err_msg}. Will create dummy PoH data instead."); + } else { + return Err(format!( + "{err_msg}. Try passing --allow-dummy-poh to allow \ + creation of shreds with dummy PoH data" + ))?; } - }) - .collect(); + None + } + }; + + let entries = match entry_summaries { + Some(entry_summaries) => entry_summaries + .map(|entry_summary| { + let num_hashes = entry_summary.num_hashes; + let hash = entry_summary.hash; + let transactions = block.transactions[entry_summary.starting_transaction_index + ..entry_summary.starting_transaction_index + + entry_summary.num_transactions as usize] + .iter() + .map(|tx_with_meta| tx_with_meta.get_transaction()) + .collect(); + Entry { + num_hashes, + hash, + transactions, + } + }) + .collect(), + None => { + let num_total_ticks = ((slot - block.parent_slot) * num_ticks_per_slot) as usize; + let num_total_entries = num_total_ticks + block.transactions.len(); + let mut entries = Vec::with_capacity(num_total_entries); + + // Create virtual tick entries for any skipped slots + // + // These ticks are necessary so that the tick height is + // advanced to the proper value when this block is processed. + // + // Additionally, a blockhash will still be inserted into the + // recent blockhashes sysvar for skipped slots. So, these + // virtual ticks will have the proper PoH + let num_skipped_slots = slot - block.parent_slot - 1; + if num_skipped_slots > 0 { + let num_virtual_ticks = num_skipped_slots * num_ticks_per_slot; + let parent_blockhash = Hash::from_str(&block.previous_blockhash)?; + let virtual_ticks_entries = + create_ticks(num_virtual_ticks, num_hashes_per_tick, parent_blockhash); + entries.extend(virtual_ticks_entries.into_iter()); + } + + // Create transaction entries + // + // Keep it simple and just do one transaction per Entry + let transaction_entries = block.transactions.iter().map(|tx_with_meta| Entry { + num_hashes: 0, + hash: Hash::default(), + transactions: vec![tx_with_meta.get_transaction()], + }); + entries.extend(transaction_entries.into_iter()); + + // Create the tick entries for this slot + // + // We do not know the intermediate hashes, so just use default + // hash for all ticks. The exception is the final tick; the + // final tick determines the blockhash so set it the known + // blockhash from the bigtable block + let blockhash = Hash::from_str(&block.blockhash)?; + let tick_entries = (0..num_ticks_per_slot).map(|idx| { + let hash = if idx == num_ticks_per_slot - 1 { + blockhash + } else { + Hash::default() + }; + Entry { + num_hashes: 0, + hash, + transactions: vec![], + } + }); + entries.extend(tick_entries.into_iter()); + + entries + } + }; let shredder = Shredder::new(*slot, block.parent_slot, 0, shred_version)?; let (data_shreds, _coding_shreds) = shredder.entries_to_shreds( @@ -1242,7 +1322,13 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) { app_profile_id, ..solana_storage_bigtable::LedgerStorageConfig::default() }; - runtime.block_on(shreds(blockstore, starting_slot, ending_slot, config)) + runtime.block_on(shreds( + blockstore, + starting_slot, + ending_slot, + false, + config, + )) } ("blocks", Some(arg_matches)) => { let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot); From b484fb8c4621d8bd246922ffa7cf92712d6ec4be Mon Sep 17 00:00:00 2001 From: Steven Czabaniuk Date: Mon, 5 Feb 2024 15:41:49 -0600 Subject: [PATCH 03/12] More thought out terminology for args and help --- ledger-tool/src/bigtable.rs | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/ledger-tool/src/bigtable.rs b/ledger-tool/src/bigtable.rs index e243b44bbf8b8b..a7c2926cce8f66 100644 --- a/ledger-tool/src/bigtable.rs +++ b/ledger-tool/src/bigtable.rs @@ -173,7 +173,7 @@ async fn shreds( blockstore: Arc, starting_slot: Slot, ending_slot: Slot, - allow_dummy_poh: bool, + allow_mock_poh: bool, config: solana_storage_bigtable::LedgerStorageConfig, ) -> Result<(), Box> { let bigtable = solana_storage_bigtable::LedgerStorage::new_with_config(config) @@ -198,14 +198,14 @@ async fn shreds( let entry_summaries = match bigtable.get_entries(*slot).await { Ok(summaries) => Some(summaries), Err(err) => { - let err_msg = format!("Failed to get PoH entry data for {slot}: {err}"); + let err_msg = format!("Failed to get PoH entries for {slot}: {err}"); - if allow_dummy_poh { - warn!("{err_msg}. Will create dummy PoH data instead."); + if allow_mock_poh { + warn!("{err_msg}. Will create mock PoH entries instead."); } else { return Err(format!( - "{err_msg}. Try passing --allow-dummy-poh to allow \ - creation of shreds with dummy PoH data" + "{err_msg}. Try passing --allow-mock-poh to allow \ + creation of shreds with mocked PoH entries" ))?; } None @@ -991,8 +991,9 @@ impl BigTableSubCommand for App<'_, '_> { .subcommand( SubCommand::with_name("shreds") .about( - "Get confirmed blocks, shred them and insert the shreds into the \ - local Blockstore", + "Get confirmed blocks from BigTable, reassemble the transactions \ + and entries, shred the block and then insert the shredded blocks into \ + the local Blockstore", ) .arg( Arg::with_name("starting_slot") @@ -1001,7 +1002,7 @@ impl BigTableSubCommand for App<'_, '_> { .value_name("SLOT") .takes_value(true) .required(true) - .help("Reconstruct starting from this slot (inclusive)"), + .help("Start shred creation at this slot (inclusive)"), ) .arg( Arg::with_name("ending_slot") @@ -1010,7 +1011,18 @@ impl BigTableSubCommand for App<'_, '_> { .value_name("SLOT") .takes_value(true) .required(true) - .help("Reconstruct ending with this slot (inclusive)"), + .help("Stop shred creation at this slot (inclusive)"), + ) + .arg( + Arg::with_name("allow_mock_poh") + .long("allow-mock-poh") + .takes_value(false) + .help( + "For slots where PoH entries are unavailable, allow the \ + generation of mock PoH entries. The mock PoH entries enable \ + the shredded block(s) to be replayable if PoH verification is \ + disabled.", + ), ), ) .subcommand( @@ -1310,6 +1322,7 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) { ("shreds", Some(arg_matches)) => { let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot); let ending_slot = value_t_or_exit!(arg_matches, "ending_slot", Slot); + let allow_mock_poh = arg_matches.is_present("allow_mock_poh"); let blockstore = Arc::new(crate::open_blockstore( &canonicalize_ledger_path(ledger_path), arg_matches, @@ -1326,7 +1339,7 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) { blockstore, starting_slot, ending_slot, - false, + allow_mock_poh, config, )) } From b8d24b348a708c6a213c9b649c358b9d706b088d Mon Sep 17 00:00:00 2001 From: steviez Date: Thu, 6 Jun 2024 17:56:40 -0500 Subject: [PATCH 04/12] Replace hard-coded values with value extracted from a Bank The command will now start by unpacking a snapshot and extracting several values needed for shredding. This is better than hard-coding as some of the values, such as shred_version and hashes_per_tick, have changed since genesis --- ledger-tool/src/bigtable.rs | 64 ++++++++++++++++++++++++++++++------- 1 file changed, 53 insertions(+), 11 deletions(-) diff --git a/ledger-tool/src/bigtable.rs b/ledger-tool/src/bigtable.rs index a7c2926cce8f66..8a9ab0c87642cc 100644 --- a/ledger-tool/src/bigtable.rs +++ b/ledger-tool/src/bigtable.rs @@ -2,10 +2,12 @@ use { crate::{ ledger_path::canonicalize_ledger_path, + load_and_process_ledger_or_exit, open_genesis_config_by, output::{ encode_confirmed_block, CliBlockWithEntries, CliEntries, EncodedConfirmedBlockWithEntries, }, + parse_process_options, }, clap::{ value_t, value_t_or_exit, values_t_or_exit, App, AppSettings, Arg, ArgMatches, SubCommand, @@ -30,14 +32,15 @@ use { shred::{ProcessShredsStats, ReedSolomonCache, Shredder}, }, solana_sdk::{ - clock::Slot, hash::Hash, pubkey::Pubkey, signature::Signature, signer::keypair::Keypair, + clock::Slot, hash::Hash, pubkey::Pubkey, shred_version::compute_shred_version, + signature::Signature, signer::keypair::Keypair, }, solana_storage_bigtable::CredentialType, solana_transaction_status::{ConfirmedBlock, UiTransactionEncoding, VersionedConfirmedBlock}, std::{ cmp::min, collections::HashSet, - path::Path, + path::{Path, PathBuf}, process::exit, result::Result, str::FromStr, @@ -169,11 +172,18 @@ async fn entries( Ok(()) } +struct ShredConfig { + shred_version: u16, + num_hashes_per_tick: u64, + num_ticks_per_slot: u64, + allow_mock_poh: bool, +} + async fn shreds( blockstore: Arc, starting_slot: Slot, ending_slot: Slot, - allow_mock_poh: bool, + shred_config: ShredConfig, config: solana_storage_bigtable::LedgerStorageConfig, ) -> Result<(), Box> { let bigtable = solana_storage_bigtable::LedgerStorage::new_with_config(config) @@ -186,12 +196,12 @@ async fn shreds( slots.retain(|&slot| slot <= ending_slot); let keypair = Keypair::from_bytes(&[0; 64])?; - // TODO: parse this from CLI ? - let shred_version = 0; - // TODO: parse from CLI OR extract from genesis - let num_ticks_per_slot = 64; - // TODO: parse from CLI OR extract from Bank; tick rate changed recently - let num_hashes_per_tick = 12500; + let ShredConfig { + shred_version, + num_hashes_per_tick, + num_ticks_per_slot, + allow_mock_poh, + } = shred_config; for slot in slots.iter() { let block = bigtable.get_confirmed_block(*slot).await?; @@ -1240,6 +1250,13 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) { let verbose = matches.is_present("verbose"); let output_format = OutputFormat::from_matches(matches, "output_format", verbose); + let snapshot_archive_path = value_t!(matches, "snapshots", String) + .ok() + .map(PathBuf::from); + let incremental_snapshot_archive_path = + value_t!(matches, "incremental_snapshot_archive_path", String) + .ok() + .map(PathBuf::from); let (subcommand, sub_matches) = matches.subcommand(); let instance_name = get_global_subcommand_arg( @@ -1323,11 +1340,35 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) { let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot); let ending_slot = value_t_or_exit!(arg_matches, "ending_slot", Slot); let allow_mock_poh = arg_matches.is_present("allow_mock_poh"); + + let ledger_path = canonicalize_ledger_path(ledger_path); + let process_options = parse_process_options(&ledger_path, arg_matches); + let genesis_config = open_genesis_config_by(&ledger_path, arg_matches); let blockstore = Arc::new(crate::open_blockstore( - &canonicalize_ledger_path(ledger_path), + &ledger_path, arg_matches, AccessType::Primary, )); + let (bank_forks, _) = load_and_process_ledger_or_exit( + arg_matches, + &genesis_config, + blockstore.clone(), + process_options, + snapshot_archive_path, + incremental_snapshot_archive_path, + ); + + let bank = bank_forks.read().unwrap().working_bank(); + let shred_version = + compute_shred_version(&genesis_config.hash(), Some(&bank.hard_forks())); + let num_hashes_per_tick = bank.hashes_per_tick().unwrap_or(0); + let num_ticks_per_slot = bank.ticks_per_slot(); + let shred_config = ShredConfig { + shred_version, + num_hashes_per_tick, + num_ticks_per_slot, + allow_mock_poh, + }; let config = solana_storage_bigtable::LedgerStorageConfig { read_only: true, @@ -1335,11 +1376,12 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) { app_profile_id, ..solana_storage_bigtable::LedgerStorageConfig::default() }; + runtime.block_on(shreds( blockstore, starting_slot, ending_slot, - allow_mock_poh, + shred_config, config, )) } From 0d128362645e55496a9d6bd94ed950a6022afd5d Mon Sep 17 00:00:00 2001 From: steviez Date: Thu, 6 Jun 2024 18:19:10 -0500 Subject: [PATCH 05/12] Add a sanity check to ensure snapshot is in correct epoch --- ledger-tool/src/bigtable.rs | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/ledger-tool/src/bigtable.rs b/ledger-tool/src/bigtable.rs index 8a9ab0c87642cc..c207f2bf1355c4 100644 --- a/ledger-tool/src/bigtable.rs +++ b/ledger-tool/src/bigtable.rs @@ -1359,6 +1359,32 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) { ); let bank = bank_forks.read().unwrap().working_bank(); + // If mock PoH is allowed, ensure that the requested slots are in + // the same epoch as the working bank. This will ensure the values + // extracted from the Bank are accurate for the slot range + if allow_mock_poh { + let working_bank_epoch = bank.epoch(); + let epoch_schedule = bank.epoch_schedule(); + let starting_epoch = epoch_schedule.get_epoch(starting_slot); + let ending_epoch = epoch_schedule.get_epoch(ending_slot); + if starting_epoch != ending_epoch { + eprintln!( + "The specified --starting-slot and --ending-slot must be in the\ + same epoch. --starting-slot {starting_slot} is in epoch {starting_epoch},\ + but --ending-slot {ending_slot} is in epoch {ending_epoch}." + ); + exit(1); + } + if starting_epoch != working_bank_epoch { + eprintln!( + "The range of slots between --starting-slot and --ending-slot are in a \ + different epoch than the working bank. The specified range is in epoch \ + {starting_epoch}, but the working bank is in {working_bank_epoch}." + ); + exit(1); + } + } + let shred_version = compute_shred_version(&genesis_config.hash(), Some(&bank.hard_forks())); let num_hashes_per_tick = bank.hashes_per_tick().unwrap_or(0); From a5c0f355f25b59a9d60c431a3562b3c13a5b9a78 Mon Sep 17 00:00:00 2001 From: steviez Date: Mon, 10 Jun 2024 18:38:56 -0500 Subject: [PATCH 06/12] Add comment to explain the use of all 0's keypair --- ledger-tool/src/bigtable.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ledger-tool/src/bigtable.rs b/ledger-tool/src/bigtable.rs index c207f2bf1355c4..2b629e2b8c8e21 100644 --- a/ledger-tool/src/bigtable.rs +++ b/ledger-tool/src/bigtable.rs @@ -195,6 +195,14 @@ async fn shreds( let mut slots = bigtable.get_confirmed_blocks(starting_slot, limit).await?; slots.retain(|&slot| slot <= ending_slot); + // Create a "dummy" keypair to sign the shreds that will later be created. + // + // The validator shred ingestion path sigverifies shreds from the network + // using the known leader for any given slot. It is unlikely that a user of + // this tool will have access to these leader identity keypairs. However, + // shred sigverify occurs prior to Blockstore::insert_shreds(). Thus, the + // shreds being signed with the "dummy" keyapir can still be inserted and + // later read/replayed/etc let keypair = Keypair::from_bytes(&[0; 64])?; let ShredConfig { shred_version, From 83995acee52c5493142f1ae0627b8f5d641ddfac Mon Sep 17 00:00:00 2001 From: steviez Date: Tue, 11 Jun 2024 17:06:20 -0500 Subject: [PATCH 07/12] Use .get() instead of direct indexing to avoid possible panic We should never hit the panic, but if the entry summaries were recorded incorrectly or something, it could be possible and the given error is more descriptive than the generic panic out of bounds message --- ledger-tool/src/bigtable.rs | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/ledger-tool/src/bigtable.rs b/ledger-tool/src/bigtable.rs index 2b629e2b8c8e21..adac03a1c3865d 100644 --- a/ledger-tool/src/bigtable.rs +++ b/ledger-tool/src/bigtable.rs @@ -232,22 +232,36 @@ async fn shreds( let entries = match entry_summaries { Some(entry_summaries) => entry_summaries - .map(|entry_summary| { + .enumerate() + .map(|(i, entry_summary)| { let num_hashes = entry_summary.num_hashes; let hash = entry_summary.hash; - let transactions = block.transactions[entry_summary.starting_transaction_index - ..entry_summary.starting_transaction_index - + entry_summary.num_transactions as usize] + let starting_transaction_index = entry_summary.starting_transaction_index; + let num_transactions = entry_summary.num_transactions as usize; + + let Some(transactions) = block.transactions.get( + starting_transaction_index..starting_transaction_index + num_transactions, + ) else { + let num_block_transactions = block.transactions.len(); + return Err(format!( + "Entry summary {i} for slot {slot} with starting_transaction_index \ + {starting_transaction_index} and num_transactions {num_transactions} \ + is in conflict with the block has {num_block_transactions} \ + transactions" + )); + }; + let transactions = transactions .iter() .map(|tx_with_meta| tx_with_meta.get_transaction()) .collect(); - Entry { + + Ok(Entry { num_hashes, hash, transactions, - } + }) }) - .collect(), + .collect::, std::string::String>>()?, None => { let num_total_ticks = ((slot - block.parent_slot) * num_ticks_per_slot) as usize; let num_total_entries = num_total_ticks + block.transactions.len(); From 061dbd189765187c7bb004b341306212ca3ceb5f Mon Sep 17 00:00:00 2001 From: steviez Date: Tue, 11 Jun 2024 17:19:49 -0500 Subject: [PATCH 08/12] Ensure starting_slot <= ending_slot --- ledger-tool/src/bigtable.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ledger-tool/src/bigtable.rs b/ledger-tool/src/bigtable.rs index adac03a1c3865d..f5f7a5790725e8 100644 --- a/ledger-tool/src/bigtable.rs +++ b/ledger-tool/src/bigtable.rs @@ -191,7 +191,7 @@ async fn shreds( .map_err(|err| format!("Failed to connect to storage: {err:?}"))?; // Make the range inclusive of both starting and ending slot - let limit = (ending_slot - starting_slot + 1) as usize; + let limit = ending_slot.saturating_sub(starting_slot).saturating_add(1) as usize; let mut slots = bigtable.get_confirmed_blocks(starting_slot, limit).await?; slots.retain(|&slot| slot <= ending_slot); @@ -1361,6 +1361,13 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) { ("shreds", Some(arg_matches)) => { let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot); let ending_slot = value_t_or_exit!(arg_matches, "ending_slot", Slot); + if starting_slot > ending_slot { + eprintln!( + "The specified --starting-slot {starting_slot} must be less than or equal to \ + the specified --ending-slot {ending_slot}." + ); + exit(1); + } let allow_mock_poh = arg_matches.is_present("allow_mock_poh"); let ledger_path = canonicalize_ledger_path(ledger_path); From 65be0e0423f552a9c4a0ac50ec9d300643f7bca3 Mon Sep 17 00:00:00 2001 From: steviez Date: Mon, 17 Jun 2024 22:31:17 -0500 Subject: [PATCH 09/12] Update to use snapshot_args() and new load_and_process_ledger() --- ledger-tool/src/bigtable.rs | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/ledger-tool/src/bigtable.rs b/ledger-tool/src/bigtable.rs index f5f7a5790725e8..fc42ba3dc8b21d 100644 --- a/ledger-tool/src/bigtable.rs +++ b/ledger-tool/src/bigtable.rs @@ -1,6 +1,7 @@ //! The `bigtable` subcommand use { crate::{ + args::snapshot_args, ledger_path::canonicalize_ledger_path, load_and_process_ledger_or_exit, open_genesis_config_by, output::{ @@ -40,7 +41,7 @@ use { std::{ cmp::min, collections::HashSet, - path::{Path, PathBuf}, + path::Path, process::exit, result::Result, str::FromStr, @@ -1027,6 +1028,7 @@ impl BigTableSubCommand for App<'_, '_> { and entries, shred the block and then insert the shredded blocks into \ the local Blockstore", ) + .args(&snapshot_args()) .arg( Arg::with_name("starting_slot") .long("starting-slot") @@ -1272,13 +1274,6 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) { let verbose = matches.is_present("verbose"); let output_format = OutputFormat::from_matches(matches, "output_format", verbose); - let snapshot_archive_path = value_t!(matches, "snapshots", String) - .ok() - .map(PathBuf::from); - let incremental_snapshot_archive_path = - value_t!(matches, "incremental_snapshot_archive_path", String) - .ok() - .map(PathBuf::from); let (subcommand, sub_matches) = matches.subcommand(); let instance_name = get_global_subcommand_arg( @@ -1383,8 +1378,7 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) { &genesis_config, blockstore.clone(), process_options, - snapshot_archive_path, - incremental_snapshot_archive_path, + None, ); let bank = bank_forks.read().unwrap().working_bank(); From 58d630ae7c5c1aea38c8aa7e34b10764ca9fd2a2 Mon Sep 17 00:00:00 2001 From: steviez Date: Tue, 18 Jun 2024 13:39:33 -0500 Subject: [PATCH 10/12] Update to include genesis arg to avoid runtime error --- ledger-tool/src/bigtable.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ledger-tool/src/bigtable.rs b/ledger-tool/src/bigtable.rs index fc42ba3dc8b21d..fce2a02fc793cf 100644 --- a/ledger-tool/src/bigtable.rs +++ b/ledger-tool/src/bigtable.rs @@ -1,14 +1,14 @@ //! The `bigtable` subcommand use { crate::{ - args::snapshot_args, + args::{load_genesis_arg, snapshot_args}, ledger_path::canonicalize_ledger_path, load_and_process_ledger_or_exit, open_genesis_config_by, output::{ encode_confirmed_block, CliBlockWithEntries, CliEntries, EncodedConfirmedBlockWithEntries, }, - parse_process_options, + parse_process_options, LoadAndProcessLedgerOutput, }, clap::{ value_t, value_t_or_exit, values_t_or_exit, App, AppSettings, Arg, ArgMatches, SubCommand, @@ -1028,6 +1028,7 @@ impl BigTableSubCommand for App<'_, '_> { and entries, shred the block and then insert the shredded blocks into \ the local Blockstore", ) + .arg(load_genesis_arg()) .args(&snapshot_args()) .arg( Arg::with_name("starting_slot") @@ -1373,7 +1374,7 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) { arg_matches, AccessType::Primary, )); - let (bank_forks, _) = load_and_process_ledger_or_exit( + let LoadAndProcessLedgerOutput { bank_forks, .. } = load_and_process_ledger_or_exit( arg_matches, &genesis_config, blockstore.clone(), From 3db85ed0473fd5475a3df4413f59bc0e30e497ba Mon Sep 17 00:00:00 2001 From: steviez Date: Tue, 18 Jun 2024 15:25:05 -0500 Subject: [PATCH 11/12] Actually create a proper keypair Using a fixed seed should still allow for deterministic keypair generation --- ledger-tool/src/bigtable.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ledger-tool/src/bigtable.rs b/ledger-tool/src/bigtable.rs index fce2a02fc793cf..cc43c3907ceae2 100644 --- a/ledger-tool/src/bigtable.rs +++ b/ledger-tool/src/bigtable.rs @@ -34,7 +34,7 @@ use { }, solana_sdk::{ clock::Slot, hash::Hash, pubkey::Pubkey, shred_version::compute_shred_version, - signature::Signature, signer::keypair::Keypair, + signature::Signature, signer::keypair::keypair_from_seed, }, solana_storage_bigtable::CredentialType, solana_transaction_status::{ConfirmedBlock, UiTransactionEncoding, VersionedConfirmedBlock}, @@ -204,7 +204,7 @@ async fn shreds( // shred sigverify occurs prior to Blockstore::insert_shreds(). Thus, the // shreds being signed with the "dummy" keyapir can still be inserted and // later read/replayed/etc - let keypair = Keypair::from_bytes(&[0; 64])?; + let keypair = keypair_from_seed(&[0; 64])?; let ShredConfig { shred_version, num_hashes_per_tick, From 5f058dab0ab7c6613867436841e6ba92ac9b9bd6 Mon Sep 17 00:00:00 2001 From: steviez Date: Thu, 20 Jun 2024 14:12:08 -0500 Subject: [PATCH 12/12] grammar fix in error message --- ledger-tool/src/bigtable.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ledger-tool/src/bigtable.rs b/ledger-tool/src/bigtable.rs index cc43c3907ceae2..a79645e4282e08 100644 --- a/ledger-tool/src/bigtable.rs +++ b/ledger-tool/src/bigtable.rs @@ -247,7 +247,7 @@ async fn shreds( return Err(format!( "Entry summary {i} for slot {slot} with starting_transaction_index \ {starting_transaction_index} and num_transactions {num_transactions} \ - is in conflict with the block has {num_block_transactions} \ + is in conflict with the block, which has {num_block_transactions} \ transactions" )); };