diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bc056d70ab2..b762fc02f961 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,8 @@ - [#6061](https://github.com/ChainSafe/forest/pull/6061) Added `forest-cli state actor-cids` command for listing all actor CIDs in the state tree for the current tipset. +- [#5568](https://github.com/ChainSafe/forest/issues/5568) Added `--n-tipsets` flag to the `forest-tool index backfill` subcommand to specify the number of epochs to backfill. + ### Changed ### Removed diff --git a/docs/docs/users/reference/cli.md b/docs/docs/users/reference/cli.md index 0aaff8346936..62be619e7c85 100644 --- a/docs/docs/users/reference/cli.md +++ b/docs/docs/users/reference/cli.md @@ -2061,12 +2061,13 @@ Options: ``` Backfill index with Ethereum mappings, events, etc -Usage: forest-tool index backfill [OPTIONS] --to +Usage: forest-tool index backfill [OPTIONS] --to --n-tipsets Options: - -c, --config Optional TOML file containing forest daemon configuration - --chain Optional chain, will override the chain section of configuration file if used - --from The starting tipset epoch for back-filling (inclusive), defaults to chain head - --to The ending tipset epoch for back-filling (inclusive) - -h, --help Print help + -c, --config Optional TOML file containing forest daemon configuration + --chain Optional chain, will override the chain section of configuration file if used + --from The starting tipset epoch for back-filling (inclusive), defaults to chain head + --to The ending tipset epoch for back-filling (inclusive) + --n-tipsets The number of tipsets for back-filling (conflicts with --to flag) + -h, --help Print help ``` diff --git a/scripts/tests/api_compare/docker-compose.yml b/scripts/tests/api_compare/docker-compose.yml index 257f9a240481..3c9a67d6a115 100644 --- a/scripts/tests/api_compare/docker-compose.yml +++ b/scripts/tests/api_compare/docker-compose.yml @@ -42,7 +42,6 @@ services: - FIL_PROOFS_PARAMETER_CACHE=${FIL_PROOFS_PARAMETER_CACHE} - FULLNODE_API_INFO=/dns/forest/tcp/${FOREST_RPC_PORT}/http - FOREST_CHAIN_INDEXER_ENABLED=1 - - FOREST_ETH_MAPPINGS_RANGE=300 entrypoint: ["/bin/bash", "-c"] user: 0:0 command: @@ -60,7 +59,7 @@ services: SNAPSHOT_EPOCH="$(ls /data/*.car.zst | tail -n 1 | grep -Eo '[0-9]+' | tail -n 1)" # backfill the index db - forest-tool index backfill --from $$SNAPSHOT_EPOCH --to $(($$SNAPSHOT_EPOCH - 300)) --chain ${CHAIN} + forest-tool index backfill --from $$SNAPSHOT_EPOCH --n-tipsets 200 --chain ${CHAIN} forest --chain ${CHAIN} --encrypt-keystore false --no-gc \ --rpc-address 0.0.0.0:${FOREST_RPC_PORT} \ @@ -129,7 +128,6 @@ services: environment: - FIL_PROOFS_PARAMETER_CACHE=${FIL_PROOFS_PARAMETER_CACHE} - FULLNODE_API_INFO=/dns/api-serve/tcp/${FOREST_OFFLINE_RPC_PORT}/http - - FOREST_ETH_MAPPINGS_RANGE=300 entrypoint: ["/bin/bash", "-c"] command: - | @@ -191,7 +189,7 @@ services: FULLNODE_API_INFO="$(cat /var/lib/lotus/token):/dns/lotus/tcp/${LOTUS_RPC_PORT}/http" SNAPSHOT_EPOCH="$(ls /data/*.car.zst | tail -n 1 | grep -Eo '[0-9]+' | tail -n 1)" # backfill the index db first - lotus index validate-backfill --from $$SNAPSHOT_EPOCH --to $(($$SNAPSHOT_EPOCH - 300)) + lotus index validate-backfill --from $$SNAPSHOT_EPOCH --to $(($$SNAPSHOT_EPOCH - 200)) sleep 2 # `sethead` right after `sync wait` to ensure the head is not set in middle of a sync lotus chain sethead --epoch $(($$SNAPSHOT_EPOCH - 50)) diff --git a/scripts/tests/harness.sh b/scripts/tests/harness.sh index b5ca3e760a45..d3e5df6c06f7 100644 --- a/scripts/tests/harness.sh +++ b/scripts/tests/harness.sh @@ -4,7 +4,6 @@ # executed directly. export FOREST_CHAIN_INDEXER_ENABLED="1" -export FOREST_ETH_MAPPINGS_RANGE="300" export FOREST_PATH="forest" export FOREST_CLI_PATH="forest-cli" @@ -59,14 +58,17 @@ function backfill_db { snapshot_epoch=$(get_epoch_from_car_db) echo "Snapshot epoch: $snapshot_epoch" - # Default to 300 if no argument is provided - local backfill_epochs - backfill_epochs=${1:-300} + # Return an error if no argument is provided + if [[ -z "$1" ]]; then + echo "Error: No argument provided. Please provide the backfill epochs." + return 1 + fi - local to_epoch - to_epoch=$((snapshot_epoch - backfill_epochs)) + # Use the provided argument for backfill epochs + local backfill_epochs + backfill_epochs=$1 - $FOREST_TOOL_PATH index backfill --chain calibnet --from "$snapshot_epoch" --to "$to_epoch" + $FOREST_TOOL_PATH index backfill --chain calibnet --from "$snapshot_epoch" --n-tipsets "$backfill_epochs" } function forest_check_db_stats { diff --git a/src/daemon/db_util.rs b/src/daemon/db_util.rs index 224c01aa1122..c314c0adac65 100644 --- a/src/daemon/db_util.rs +++ b/src/daemon/db_util.rs @@ -321,6 +321,51 @@ async fn transcode_into_forest_car(from: &Path, to: &Path) -> anyhow::Result<()> Ok(()) } +async fn process_ts( + ts: &Tipset, + state_manager: &Arc>, + delegated_messages: &mut Vec<(crate::message::SignedMessage, u64)>, +) -> anyhow::Result<()> +where + DB: fvm_ipld_blockstore::Blockstore + Send + Sync + 'static, +{ + let epoch = ts.epoch(); + let tsk = ts.key().clone(); + + let state_output = state_manager + .compute_tipset_state(Arc::new(ts.clone()), NO_CALLBACK, VMTrace::NotTraced) + .await?; + for events_root in state_output.events_roots.iter().flatten() { + tracing::trace!("Indexing events root @{epoch}: {events_root}"); + + state_manager.chain_store().put_index(events_root, &tsk)?; + } + + delegated_messages.append( + &mut state_manager + .chain_store() + .headers_delegated_messages(ts.block_headers().iter())?, + ); + tracing::trace!("Indexing tipset @{}: {}", epoch, &tsk); + state_manager.chain_store().put_tipset_key(&tsk)?; + + Ok(()) +} + +pub enum RangeSpec { + To(ChainEpoch), + NumTipsets(usize), +} + +impl std::fmt::Display for RangeSpec { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + RangeSpec::To(epoch) => write!(f, "To epoch: {}", epoch), + RangeSpec::NumTipsets(n) => write!(f, "Tipsets: {}", n), + } + } +} + /// To support the Event RPC API, a new column has been added to parity-db to handle the mapping: /// - Events root [`Cid`] -> [`TipsetKey`]. /// @@ -332,47 +377,38 @@ async fn transcode_into_forest_car(from: &Path, to: &Path) -> anyhow::Result<()> pub async fn backfill_db( state_manager: &Arc>, head_ts: &Tipset, - to_epoch: ChainEpoch, + spec: RangeSpec, ) -> anyhow::Result<()> where DB: fvm_ipld_blockstore::Blockstore + Send + Sync + 'static, { - tracing::info!( - "Starting index backfill (from: {}, to: {})", - head_ts.epoch(), - to_epoch - ); + tracing::info!("Starting index backfill..."); let mut delegated_messages = vec![]; let mut num_backfills = 0; - for ts in head_ts - .clone() - .chain(&state_manager.chain_store().blockstore()) - .take_while(|ts| ts.epoch() >= to_epoch) - { - let epoch = ts.epoch(); - let tsk = ts.key().clone(); - let ts = Arc::new(ts); - - let state_output = state_manager - .compute_tipset_state(ts.clone(), NO_CALLBACK, VMTrace::NotTraced) - .await?; - for events_root in state_output.events_roots.iter().flatten() { - tracing::trace!("Indexing events root @{epoch}: {events_root}"); - state_manager.chain_store().put_index(events_root, &tsk)?; + match spec { + RangeSpec::To(to_epoch) => { + for ts in head_ts + .clone() + .chain(&state_manager.chain_store().blockstore()) + .take_while(|ts| ts.epoch() >= to_epoch) + { + process_ts(&ts, state_manager, &mut delegated_messages).await?; + num_backfills += 1; + } + } + RangeSpec::NumTipsets(n_tipsets) => { + for ts in head_ts + .clone() + .chain(&state_manager.chain_store().blockstore()) + .take(n_tipsets) + { + process_ts(&ts, state_manager, &mut delegated_messages).await?; + num_backfills += 1; + } } - - delegated_messages.append( - &mut state_manager - .chain_store() - .headers_delegated_messages(ts.block_headers().iter())?, - ); - tracing::trace!("Indexing tipset @{}: {}", epoch, &tsk); - state_manager.chain_store().put_tipset_key(&tsk)?; - - num_backfills += 1; } state_manager diff --git a/src/tool/offline_server/server.rs b/src/tool/offline_server/server.rs index dad9539313fb..8418f88b2300 100644 --- a/src/tool/offline_server/server.rs +++ b/src/tool/offline_server/server.rs @@ -7,6 +7,7 @@ use crate::chain_sync::SyncStatusReport; use crate::chain_sync::network_context::SyncNetworkContext; use crate::cli_shared::cli::EventsConfig; use crate::cli_shared::snapshot::TrustedVendor; +use crate::daemon::db_util::RangeSpec; use crate::daemon::db_util::backfill_db; use crate::db::{MemoryDB, car::ManyCar}; use crate::genesis::read_genesis_header; @@ -108,7 +109,7 @@ pub async fn start_offline_server( backfill_db( &state_manager, &head_ts, - head_ts.epoch() + 1 - index_backfill_epochs as i64, + RangeSpec::NumTipsets(index_backfill_epochs), ) .await?; } diff --git a/src/tool/subcommands/index_cmd.rs b/src/tool/subcommands/index_cmd.rs index c89c323acc78..22b1b65ef3ee 100644 --- a/src/tool/subcommands/index_cmd.rs +++ b/src/tool/subcommands/index_cmd.rs @@ -3,13 +3,14 @@ use std::{path::PathBuf, sync::Arc}; +use anyhow::bail; use clap::Subcommand; use crate::chain::ChainStore; use crate::chain::index::ResolveNullTipset; use crate::cli_shared::{chain_path, read_config}; -use crate::daemon::db_util::backfill_db; use crate::daemon::db_util::load_all_forest_cars; +use crate::daemon::db_util::{RangeSpec, backfill_db}; use crate::db::CAR_DB_DIR_NAME; use crate::db::car::ManyCar; use crate::db::db_engine::{db_root, open_db}; @@ -32,9 +33,12 @@ pub enum IndexCommands { /// The starting tipset epoch for back-filling (inclusive), defaults to chain head #[arg(long)] from: Option, - /// The ending tipset epoch for back-filling (inclusive) + /// Ending tipset epoch for back-filling (inclusive) #[arg(long)] - to: ChainEpoch, + to: Option, + /// Number of tipsets for back-filling + #[arg(long, conflicts_with = "to")] + n_tipsets: Option, }, } @@ -46,7 +50,17 @@ impl IndexCommands { chain, from, to, + n_tipsets, } => { + let spec = match (to, n_tipsets) { + (Some(x), None) => RangeSpec::To(*x), + (None, Some(x)) => RangeSpec::NumTipsets(*x), + (None, None) => { + bail!("You must provide either '--to' or '--n-tipsets'."); + } + _ => unreachable!(), // Clap ensures this case is handled + }; + let (_, config) = read_config(config.as_ref(), chain.clone())?; let chain_data_path = chain_path(&config); @@ -81,7 +95,7 @@ impl IndexCommands { println!("Database path: {}", db_root_dir.display()); println!("From epoch: {}", from.unwrap_or_else(|| head_ts.epoch())); - println!("To epoch: {to}"); + println!("{spec}"); println!("Head epoch: {}", head_ts.epoch()); let from_ts = if let Some(from) = from { @@ -94,7 +108,7 @@ impl IndexCommands { head_ts }; - backfill_db(&state_manager, &from_ts, *to).await?; + backfill_db(&state_manager, &from_ts, spec).await?; Ok(()) }