diff --git a/CHANGELOG.md b/CHANGELOG.md index 16d49a23bbbf..cbd9224e5b10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,8 @@ - [#6405](https://github.com/ChainSafe/forest/pull/6405) Enabled `Filecoin.EthGetLogs` for API v2. +- [#6421](https://github.com/ChainSafe/forest/pull/6421) Add an environment variable `FOREST_RPC_BACKFILL_FULL_TIPSET_FROM_NETWORK` to enable backfilling full tipsets from network in a few RPC methods. + ### Changed - [#6368](https://github.com/ChainSafe/forest/pull/6368): Migrated build and development tooling from Makefile to `mise`. Contributors should install `mise` and use `mise run` commands instead of `make` commands. diff --git a/docs/dictionary.txt b/docs/dictionary.txt index 5ef0e82e4db7..ed91172e69a4 100644 --- a/docs/dictionary.txt +++ b/docs/dictionary.txt @@ -1,6 +1,7 @@ 2k APIs backend +backfill backport benchmarking blockstore diff --git a/docs/docs/users/reference/env_variables.md b/docs/docs/users/reference/env_variables.md index 99017d6e73f2..c488020f2c48 100644 --- a/docs/docs/users/reference/env_variables.md +++ b/docs/docs/users/reference/env_variables.md @@ -53,6 +53,7 @@ process. | `FOREST_ZSTD_FRAME_CACHE_DEFAULT_MAX_SIZE` | positive integer | 268435456 | 536870912 | The default zstd frame cache max size in bytes | | `FOREST_JWT_DISABLE_EXP_VALIDATION` | 1 or true | empty | 1 | Whether or not to disable JWT expiration validation | | `FOREST_ETH_BLOCK_CACHE_SIZE` | positive integer | 500 | 1 | The size of Eth block cache | +| `FOREST_RPC_BACKFILL_FULL_TIPSET_FROM_NETWORK` | 1 or true | false | 1 | Whether or not to backfill full tipsets from the p2p network | ### `FOREST_F3_SIDECAR_FFI_BUILD_OPT_OUT` diff --git a/src/blocks/tipset.rs b/src/blocks/tipset.rs index 33fc9e462d18..e43202cdfe22 100644 --- a/src/blocks/tipset.rs +++ b/src/blocks/tipset.rs @@ -281,31 +281,6 @@ impl Tipset { Tipset::load(store, tsk)?.context("Required tipset missing from database") } - /// Constructs and returns a full tipset if messages from storage exists - pub fn fill_from_blockstore(&self, store: &impl Blockstore) -> Option { - // Find tipset messages. If any are missing, return `None`. - let blocks = self - .block_headers() - .iter() - .cloned() - .map(|header| { - let (bls_messages, secp_messages) = - crate::chain::store::block_messages(store, &header).ok()?; - Some(Block { - header, - bls_messages, - secp_messages, - }) - }) - .collect::>>()?; - - // the given tipset has already been verified, so this cannot fail - Some( - FullTipset::new(blocks) - .expect("block headers have already been verified so this check cannot fail"), - ) - } - /// Returns epoch of the tipset. pub fn epoch(&self) -> ChainEpoch { self.min_ticket_block().epoch diff --git a/src/chain_sync/mod.rs b/src/chain_sync/mod.rs index 790204c69b1e..998a5835c965 100644 --- a/src/chain_sync/mod.rs +++ b/src/chain_sync/mod.rs @@ -13,7 +13,7 @@ mod validation; pub use self::{ bad_block_cache::BadBlockCache, - chain_follower::{ChainFollower, load_full_tipset}, + chain_follower::{ChainFollower, get_full_tipset, load_full_tipset}, chain_muxer::SyncConfig, consensus::collect_errs, sync_status::{ForkSyncInfo, ForkSyncStage, NodeSyncStatus, SyncStatus, SyncStatusReport}, diff --git a/src/rpc/methods/chain.rs b/src/rpc/methods/chain.rs index dab1aa674158..5c6b049b40d0 100644 --- a/src/rpc/methods/chain.rs +++ b/src/rpc/methods/chain.rs @@ -10,6 +10,7 @@ use crate::blocks::RawBlockHeader; use crate::blocks::{Block, CachingBlockHeader, Tipset, TipsetKey}; use crate::chain::index::ResolveNullTipset; use crate::chain::{ChainStore, ExportOptions, FilecoinSnapshotVersion, HeadChange}; +use crate::chain_sync::{get_full_tipset, load_full_tipset}; use crate::cid_collections::CidHashSet; use crate::ipld::DfsIter; use crate::ipld::{CHAIN_EXPORT_STATUS, cancel_export, end_export, start_export}; @@ -30,6 +31,7 @@ use crate::shim::executor::Receipt; use crate::shim::message::Message; use crate::utils::db::CborStoreExt as _; use crate::utils::io::VoidAsyncWriter; +use crate::utils::misc::env::is_env_truthy; use anyhow::{Context as _, Result}; use cid::Cid; use fvm_ipld_blockstore::Blockstore; @@ -294,7 +296,7 @@ impl RpcMethod<1> for ChainGetParentMessages { type Ok = Vec; async fn handle( - ctx: Ctx, + ctx: Ctx, (block_cid,): Self::Params, ) -> Result { let store = ctx.store(); @@ -305,7 +307,7 @@ impl RpcMethod<1> for ChainGetParentMessages { Ok(vec![]) } else { let parent_tipset = Tipset::load_required(store, &block_header.parents)?; - load_api_messages_from_tipset(store, &parent_tipset) + load_api_messages_from_tipset(&ctx, parent_tipset.key()).await } } } @@ -368,13 +370,13 @@ impl RpcMethod<1> for ChainGetMessagesInTipset { type Ok = Vec; async fn handle( - ctx: Ctx, + ctx: Ctx, (ApiTipsetKey(tipset_key),): Self::Params, ) -> Result { let tipset = ctx .chain_store() .load_required_tipset_or_heaviest(&tipset_key)?; - load_api_messages_from_tipset(ctx.store(), &tipset) + load_api_messages_from_tipset(&ctx, tipset.key()).await } } @@ -1314,13 +1316,30 @@ pub(crate) fn chain_notify( receiver } -fn load_api_messages_from_tipset( - store: &impl Blockstore, - tipset: &Tipset, +async fn load_api_messages_from_tipset( + ctx: &crate::rpc::RPCState, + tipset_keys: &TipsetKey, ) -> Result, ServerError> { - let full_tipset = tipset - .fill_from_blockstore(store) - .context("Failed to load full tipset")?; + static SHOULD_BACKFILL: LazyLock = LazyLock::new(|| { + let enabled = is_env_truthy("FOREST_RPC_BACKFILL_FULL_TIPSET_FROM_NETWORK"); + if enabled { + tracing::warn!( + "Full tipset backfilling from network is enabled via FOREST_RPC_BACKFILL_FULL_TIPSET_FROM_NETWORK, excessive disk and bandwidth usage is expected." + ); + } + enabled + }); + let full_tipset = if *SHOULD_BACKFILL { + get_full_tipset( + &ctx.sync_network_context, + ctx.chain_store(), + None, + tipset_keys, + ) + .await? + } else { + load_full_tipset(ctx.chain_store(), tipset_keys)? + }; let blocks = full_tipset.into_blocks(); let mut messages = vec![]; let mut seen = CidHashSet::default();