diff --git a/.github/workflows/forest.yml b/.github/workflows/forest.yml index b311bad430c9..5ffc2e5bf1f8 100644 --- a/.github/workflows/forest.yml +++ b/.github/workflows/forest.yml @@ -306,6 +306,28 @@ jobs: - name: Snapshot export check v2 run: ./scripts/tests/calibnet_export_f3_check.sh timeout-minutes: ${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }} + calibnet-export-diff-check: + needs: + - build-ubuntu + name: Diff snapshot export checks + runs-on: ubuntu-24.04 + steps: + - run: lscpu + - uses: actions/cache@v4 + with: + path: "${{ env.FIL_PROOFS_PARAMETER_CACHE }}" + key: proof-params-keys + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v5 + with: + name: "forest-${{ runner.os }}" + path: ~/.cargo/bin + - name: Set permissions + run: | + chmod +x ~/.cargo/bin/forest* + - name: Diff snapshot export check + run: ./scripts/tests/calibnet_export_diff_check.sh + timeout-minutes: ${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }} calibnet-no-discovery-checks: needs: - build-ubuntu diff --git a/CHANGELOG.md b/CHANGELOG.md index 194ce955b1d2..5677a6e6a00a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,8 @@ ### Added +- [#6074](https://github.com/ChainSafe/forest/issues/6074) Added `forest-cli snapshot export-diff` subcommand for exporting a diff snapshot. + - [#6061](https://github.com/ChainSafe/forest/pull/6061) Added `forest-cli state actor-cids` command for listing all actor CIDs in the state tree for the current tipset. ### Changed diff --git a/scripts/tests/calibnet_export_diff_check.sh b/scripts/tests/calibnet_export_diff_check.sh new file mode 100755 index 000000000000..add8dfc8fc14 --- /dev/null +++ b/scripts/tests/calibnet_export_diff_check.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# This script is checking the correctness of +# the diff snapshot export feature. +# It requires both the `forest` and `forest-cli` binaries to be in the PATH. + +set -euo pipefail + +source "$(dirname "$0")/harness.sh" + +forest_init "$@" + +db_path=$($FOREST_TOOL_PATH db stats --chain calibnet | grep "Database path:" | cut -d':' -f2- | xargs) +snapshot=$(find "$db_path/car_db"/*.car.zst | tail -n 1) +snapshot_epoch=$(forest_query_epoch "$snapshot") + +echo "Exporting diff snapshot @ $snapshot_epoch with forest-cli snapshot export-diff" +$FOREST_CLI_PATH snapshot export-diff --from "$snapshot_epoch" --to "$((snapshot_epoch - 900))" -d 900 -o diff1 + +$FOREST_CLI_PATH shutdown --force + +echo "Exporting diff snapshot @ $snapshot_epoch with forest-tool archive export" +$FOREST_TOOL_PATH archive export --epoch "$snapshot_epoch" --depth 900 --diff "$((snapshot_epoch - 900))" --diff-depth 900 -o diff2 "$snapshot" + +echo "Comparing diff snapshots" +diff diff1 diff2 diff --git a/src/cli/subcommands/snapshot_cmd.rs b/src/cli/subcommands/snapshot_cmd.rs index c157c38c578e..cbcda5b20835 100644 --- a/src/cli/subcommands/snapshot_cmd.rs +++ b/src/cli/subcommands/snapshot_cmd.rs @@ -6,6 +6,7 @@ use crate::chain_sync::SyncConfig; use crate::cli_shared::snapshot::{self, TrustedVendor}; use crate::db::car::forest::new_forest_car_temp_path_in; use crate::networks::calibnet; +use crate::rpc::chain::ForestChainExportDiffParams; use crate::rpc::{self, chain::ForestChainExportParams, prelude::*, types::ApiTipsetKey}; use anyhow::Context as _; use chrono::DateTime; @@ -40,6 +41,21 @@ pub enum SnapshotCommands { #[arg(long, value_enum, default_value_t = FilecoinSnapshotVersion::V1)] format: FilecoinSnapshotVersion, }, + /// Export a diff snapshot between `from` and `to` epochs to `` + ExportDiff { + /// `./forest_snapshot_diff_{chain}_{from}_{to}+{depth}.car.zst`. + #[arg(short, long, default_value = ".", verbatim_doc_comment)] + output_path: PathBuf, + /// Epoch to export from + #[arg(long)] + from: i64, + /// Epoch to diff against + #[arg(long)] + to: i64, + /// How many state-roots to include. Lower limit is 900 for `calibnet` and `mainnet`. + #[arg(short, long)] + depth: Option, + }, } impl SnapshotCommands { @@ -138,6 +154,77 @@ impl SnapshotCommands { println!("Export completed."); Ok(()) } + Self::ExportDiff { + output_path, + from, + to, + depth, + } => { + let raw_network_name = StateNetworkName::call(&client, ()).await?; + + // For historical reasons and backwards compatibility if snapshot services or their + // consumers relied on the `calibnet`, we use `calibnet` as the chain name. + let chain_name = if raw_network_name == calibnet::NETWORK_GENESIS_NAME { + calibnet::NETWORK_COMMON_NAME + } else { + raw_network_name.as_str() + }; + + let depth = depth.unwrap_or_else(|| from - to); + anyhow::ensure!(depth > 0, "depth must be positive"); + + let output_path = match output_path.is_dir() { + true => output_path.join(format!( + "forest_snapshot_diff_{chain_name}_{from}_{to}+{depth}.car.zst" + )), + false => output_path.clone(), + }; + + let output_dir = output_path.parent().context("invalid output path")?; + let temp_path = new_forest_car_temp_path_in(output_dir)?; + + let params = ForestChainExportDiffParams { + output_path: temp_path.to_path_buf(), + from, + to, + depth, + }; + + let pb = ProgressBar::new_spinner().with_style( + ProgressStyle::with_template( + "{spinner} {msg} {binary_total_bytes} written in {elapsed} ({binary_bytes_per_sec})", + ) + .expect("indicatif template must be valid"), + ).with_message(format!("Exporting {} ...", output_path.display())); + pb.enable_steady_tick(std::time::Duration::from_millis(80)); + let handle = tokio::spawn({ + let path: PathBuf = (&temp_path).into(); + let pb = pb.clone(); + let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(1)); + async move { + loop { + interval.tick().await; + if let Ok(meta) = std::fs::metadata(&path) { + pb.set_position(meta.len()); + } + } + } + }); + + // Manually construct RpcRequest because snapshot export could + // take a few hours on mainnet + client + .call(ForestChainExportDiff::request((params,))?.with_timeout(Duration::MAX)) + .await?; + + handle.abort(); + pb.finish(); + _ = handle.await; + + temp_path.persist(output_path)?; + println!("Export completed."); + Ok(()) + } } } } diff --git a/src/ipld/util.rs b/src/ipld/util.rs index 4b584489855e..b4ba68cc5edd 100644 --- a/src/ipld/util.rs +++ b/src/ipld/util.rs @@ -110,7 +110,7 @@ pin_project! { db: DB, dfs: VecDeque, // Depth-first work queue. seen: CidHashSet, - stateroot_limit: ChainEpoch, + stateroot_limit_exclusive: ChainEpoch, fail_on_dead_links: bool, } } @@ -140,20 +140,20 @@ impl ChainStream { /// /// * `db` - A database that implements [`Blockstore`] interface. /// * `tipset_iter` - An iterator of [`Tipset`], descending order `$child -> $parent`. -/// * `stateroot_limit` - An epoch that signifies how far back we need to inspect tipsets, +/// * `stateroot_limit` - An epoch that signifies how far back (exclusive) we need to inspect tipsets, /// in-depth. This has to be pre-calculated using this formula: `$cur_epoch - $depth`, where `$depth` /// is the number of `[`Tipset`]` that needs inspection. pub fn stream_chain, ITER: Iterator + Unpin>( db: DB, tipset_iter: ITER, - stateroot_limit: ChainEpoch, + stateroot_limit_exclusive: ChainEpoch, ) -> ChainStream { ChainStream { tipset_iter, db, dfs: VecDeque::new(), seen: CidHashSet::default(), - stateroot_limit, + stateroot_limit_exclusive, fail_on_dead_links: true, } } @@ -163,9 +163,9 @@ pub fn stream_chain, ITER: Iterator pub fn stream_graph, ITER: Iterator + Unpin>( db: DB, tipset_iter: ITER, - stateroot_limit: ChainEpoch, + stateroot_limit_exclusive: ChainEpoch, ) -> ChainStream { - stream_chain(db, tipset_iter, stateroot_limit).fail_on_dead_links(false) + stream_chain(db, tipset_iter, stateroot_limit_exclusive).fail_on_dead_links(false) } impl, ITER: Iterator + Unpin> Stream @@ -177,7 +177,7 @@ impl, ITER: Iterator + Unpin> Stream use Task::*; let fail_on_dead_links = self.fail_on_dead_links; - let stateroot_limit = self.stateroot_limit; + let stateroot_limit_exclusive = self.stateroot_limit_exclusive; let this = self.project(); loop { @@ -253,7 +253,7 @@ impl, ITER: Iterator + Unpin> Stream } // Process block messages. - if block.epoch > stateroot_limit { + if block.epoch > stateroot_limit_exclusive { this.dfs.push_back(Iterate( block.epoch, *block.cid(), @@ -266,7 +266,7 @@ impl, ITER: Iterator + Unpin> Stream // Visit the block if it's within required depth. And a special case for `0` // epoch to match Lotus' implementation. - if block.epoch == 0 || block.epoch > stateroot_limit { + if block.epoch == 0 || block.epoch > stateroot_limit_exclusive { // NOTE: In the original `walk_snapshot` implementation we walk the dag // immediately. Which is what we do here as well, but using a queue. this.dfs.push_back(Iterate( diff --git a/src/rpc/methods/chain.rs b/src/rpc/methods/chain.rs index 53a502dc4ced..4ecc22cf871c 100644 --- a/src/rpc/methods/chain.rs +++ b/src/rpc/methods/chain.rs @@ -52,6 +52,8 @@ use tokio::task::JoinHandle; const HEAD_CHANNEL_CAPACITY: usize = 10; +static CHAIN_EXPORT_LOCK: LazyLock> = LazyLock::new(|| Mutex::new(())); + /// Subscribes to head changes from the chain store and broadcasts new blocks. /// /// # Notes @@ -325,9 +327,7 @@ impl RpcMethod<1> for ForestChainExport { dry_run, } = params; - static LOCK: LazyLock> = LazyLock::new(|| Mutex::new(())); - - let _locked = LOCK.try_lock(); + let _locked = CHAIN_EXPORT_LOCK.try_lock(); if _locked.is_err() { return Err(anyhow::anyhow!("Another chain export job is still in progress").into()); } @@ -404,6 +404,62 @@ impl RpcMethod<1> for ForestChainExport { } } +pub enum ForestChainExportDiff {} +impl RpcMethod<1> for ForestChainExportDiff { + const NAME: &'static str = "Forest.ChainExportDiff"; + const PARAM_NAMES: [&'static str; 1] = ["params"]; + const API_PATHS: BitFlags = ApiPaths::all(); + const PERMISSION: Permission = Permission::Read; + + type Params = (ForestChainExportDiffParams,); + type Ok = (); + + async fn handle( + ctx: Ctx, + (params,): Self::Params, + ) -> Result { + let ForestChainExportDiffParams { + from, + to, + depth, + output_path, + } = params; + + let _locked = CHAIN_EXPORT_LOCK.try_lock(); + if _locked.is_err() { + return Err( + anyhow::anyhow!("Another chain export diff job is still in progress").into(), + ); + } + + let chain_finality = ctx.chain_config().policy.chain_finality; + if depth < chain_finality { + return Err( + anyhow::anyhow!(format!("depth must be greater than {chain_finality}")).into(), + ); + } + + let head = ctx.chain_store().heaviest_tipset(); + let start_ts = + ctx.chain_index() + .tipset_by_height(from, head, ResolveNullTipset::TakeOlder)?; + + crate::tool::subcommands::archive_cmd::do_export( + &ctx.store_owned(), + start_ts, + output_path, + None, + depth, + Some(to), + Some(chain_finality), + true, + ) + .await?; + + Ok(()) + } +} + pub enum ChainExport {} impl RpcMethod<1> for ChainExport { const NAME: &'static str = "Filecoin.ChainExport"; @@ -1000,6 +1056,15 @@ pub struct ForestChainExportParams { } lotus_json_with_self!(ForestChainExportParams); +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct ForestChainExportDiffParams { + pub from: ChainEpoch, + pub to: ChainEpoch, + pub depth: i64, + pub output_path: PathBuf, +} +lotus_json_with_self!(ForestChainExportDiffParams); + #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] pub struct ChainExportParams { pub epoch: ChainEpoch, diff --git a/src/rpc/mod.rs b/src/rpc/mod.rs index 55461f7fb3dd..de8c8d82d851 100644 --- a/src/rpc/mod.rs +++ b/src/rpc/mod.rs @@ -81,6 +81,7 @@ macro_rules! for_each_rpc_method { $callback!($crate::rpc::chain::ChainStatObj); $callback!($crate::rpc::chain::ChainTipSetWeight); $callback!($crate::rpc::chain::ForestChainExport); + $callback!($crate::rpc::chain::ForestChainExportDiff); // common vertical $callback!($crate::rpc::common::Session); diff --git a/src/tool/subcommands/api_cmd/test_snapshots_ignored.txt b/src/tool/subcommands/api_cmd/test_snapshots_ignored.txt index e174ae3c0d9a..3a679e3e0d03 100644 --- a/src/tool/subcommands/api_cmd/test_snapshots_ignored.txt +++ b/src/tool/subcommands/api_cmd/test_snapshots_ignored.txt @@ -78,6 +78,7 @@ Filecoin.WalletSign Filecoin.WalletSignMessage Filecoin.Web3ClientVersion Forest.ChainExport +Forest.ChainExportDiff Forest.ChainGetMinBaseFee Forest.NetInfo Forest.SnapshotGC diff --git a/src/tool/subcommands/archive_cmd.rs b/src/tool/subcommands/archive_cmd.rs index aa58156701ac..fb338c89c27e 100644 --- a/src/tool/subcommands/archive_cmd.rs +++ b/src/tool/subcommands/archive_cmd.rs @@ -37,7 +37,7 @@ use crate::daemon::bundle::load_actor_bundles; use crate::db::car::{AnyCar, ManyCar, forest::DEFAULT_FOREST_CAR_COMPRESSION_LEVEL}; use crate::f3::snapshot::F3SnapshotHeader; use crate::interpreter::VMTrace; -use crate::ipld::stream_graph; +use crate::ipld::{stream_chain, stream_graph}; use crate::networks::{ChainConfig, NetworkChain, butterflynet, calibnet, mainnet}; use crate::shim::address::CurrentNetwork; use crate::shim::clock::{ChainEpoch, EPOCH_DURATION_SECONDS, EPOCHS_IN_DAY}; @@ -263,7 +263,7 @@ impl ArchiveCommands { let heaviest_tipset = store.heaviest_tipset()?; do_export( &store.into(), - heaviest_tipset, + heaviest_tipset.into(), output_path, epoch, depth, @@ -510,9 +510,9 @@ fn build_output_path( } #[allow(clippy::too_many_arguments)] -async fn do_export( +pub async fn do_export( store: &Arc, - root: Tipset, + root: Arc, output_path: PathBuf, epoch_option: Option, depth: ChainEpochDelta, @@ -520,9 +520,9 @@ async fn do_export( diff_depth: Option, force: bool, ) -> anyhow::Result<()> { - let ts = Arc::new(root); + let ts = root; - let genesis = ts.genesis(&store)?; + let genesis = ts.genesis(store)?; let network = NetworkChain::from_genesis_or_devnet_placeholder(genesis.cid()); let epoch = epoch_option.unwrap_or(ts.epoch()); @@ -537,7 +537,7 @@ async fn do_export( info!("looking up a tipset by epoch: {}", epoch); - let index = ChainIndex::new(store); + let index = ChainIndex::new(store.clone()); let ts = index .tipset_by_height(epoch, ts, ResolveNullTipset::TakeOlder) @@ -549,7 +549,7 @@ async fn do_export( .context("diff epoch must be smaller than target epoch")?; let diff_ts: &Tipset = &diff_ts; let diff_limit = diff_depth.map(|depth| diff_ts.epoch() - depth).unwrap_or(0); - let mut stream = stream_graph( + let mut stream = stream_chain( store.clone(), diff_ts.clone().chain_owned(store.clone()), diff_limit, @@ -994,7 +994,7 @@ async fn export_lite_snapshot( let force = false; do_export( &store, - root, + root.into(), output_path.clone(), Some(epoch), depth, @@ -1027,7 +1027,7 @@ async fn export_diff_snapshot( let force = false; do_export( &store, - root, + root.into(), output_path.clone(), Some(epoch), depth, @@ -1169,7 +1169,7 @@ mod tests { let heaviest_tipset = store.heaviest_tipset().unwrap(); do_export( &store.into(), - heaviest_tipset, + heaviest_tipset.into(), output_path.path().into(), Some(0), 1, diff --git a/src/tool/subcommands/mod.rs b/src/tool/subcommands/mod.rs index 0a84ca0e15ea..ece1af08367a 100644 --- a/src/tool/subcommands/mod.rs +++ b/src/tool/subcommands/mod.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0, MIT mod api_cmd; -mod archive_cmd; +pub(crate) mod archive_cmd; mod backup_cmd; mod benchmark_cmd; mod car_cmd;