feat(gc): keep message receipts and events

CHANGELOG.md

@@ -33,6 +33,8 @@
 
 ### Changed
 
+- [#6655](https://github.com/ChainSafe/forest/issues/6655): Updated garbage collector to keep message receipts and events.
+
 - [#6522](https://github.com/ChainSafe/forest/pull/6522): `Filecoin.EthTraceFilter` filter options `from_block` and `to_block` now default to `latest` tag when omitted for v1 and v2 API.
 
 ### Removed

src/chain/mod.rs

@@ -33,6 +33,8 @@ use tokio::io::{AsyncWrite, AsyncWriteExt, BufWriter};
 #[derive(Debug, Clone, Default)]
 pub struct ExportOptions {
     pub skip_checksum: bool,
+    pub include_receipts: bool,
+    pub include_events: bool,
     pub seen: CidHashSet,
 }
 
@@ -140,9 +142,15 @@ async fn export_to_forest_car<D: Digest>(
 ) -> anyhow::Result<Option<digest::Output<D>>> {
     let ExportOptions {
         skip_checksum,
+        include_receipts,
+        include_events,
         seen,
     } = options.unwrap_or_default();
 
+    if include_events && !include_receipts {
+        anyhow::bail!("message receipts must be included when events are included");
+    }
+
     let stateroot_lookup_limit = tipset.epoch() - lookup_depth;
 
     // Wrap writer in optional checksum calculator
@@ -161,6 +169,8 @@ async fn export_to_forest_car<D: Digest>(
             stateroot_lookup_limit,
         )
         .with_seen(seen)
+        .with_message_receipts(include_receipts)
+        .with_events(include_events)
         .track_progress(true),
     );
 

src/cli/subcommands/snapshot_cmd.rs

@@ -143,6 +143,8 @@ impl SnapshotCommands {
                     recent_roots: depth,
                     output_path: temp_path.to_path_buf(),
                     tipset_keys: tipset.key().clone().into(),
+                    include_receipts: false,
+                    include_events: false,
                     skip_checksum,
                     dry_run,
                 };

src/db/gc/snapshot.rs

@@ -256,7 +256,9 @@ where
             file,
             Some(ExportOptions {
                 skip_checksum: true,
-                ..Default::default()
+                include_receipts: true,
+                include_events: true,
+                seen: Default::default(),
             }),
         )
         .await?;

src/ipld/util.rs

@@ -5,6 +5,7 @@ use crate::blocks::Tipset;
 use crate::cid_collections::CidHashSet;
 use crate::ipld::Ipld;
 use crate::shim::clock::ChainEpoch;
+use crate::shim::executor::Receipt;
 use crate::utils::db::car_stream::CarBlock;
 use crate::utils::encoding::extract_cids;
 use crate::utils::multihash::prelude::*;
@@ -137,7 +138,9 @@ impl Iterator for DfsIter {
 
 enum IterateType {
     Message(Cid),
+    MessageReceipts(Cid),
     StateRoot(Cid),
+    EventsRoot(Cid),
 }
 
 enum Task {
@@ -155,6 +158,8 @@ pin_project! {
         seen: CidHashSet,
         stateroot_limit_exclusive: ChainEpoch,
         fail_on_dead_links: bool,
+        message_receipts: bool,
+        events: bool,
         track_progress: bool,
     }
 }
@@ -175,6 +180,19 @@ impl<DB, T> ChainStream<DB, T> {
         self
     }
 
+    /// Enable traversal of message receipt roots during chain export.
+    pub fn with_message_receipts(mut self, message_receipts: bool) -> Self {
+        self.message_receipts = message_receipts;
+        self
+    }
+
+    /// Enable traversal of events roots during chain export.
+    /// Requires message receipts to be enabled as well.
+    pub fn with_events(mut self, events: bool) -> Self {
+        self.events = events;
+        self
+    }
+
     #[allow(dead_code)]
     pub fn into_seen(self) -> CidHashSet {
         self.seen
@@ -204,6 +222,8 @@ pub fn stream_chain<DB: Blockstore, T: Borrow<Tipset>, ITER: Iterator<Item = T>
         seen: CidHashSet::default(),
         stateroot_limit_exclusive,
         fail_on_dead_links: true,
+        message_receipts: false,
+        events: false,
         track_progress: false,
     }
 }
@@ -276,6 +296,20 @@ impl<DB: Blockstore, T: Borrow<Tipset>, ITER: Iterator<Item = T> + Unpin> Stream
                                         IterateType::StateRoot(c) => {
                                             format!("state root {c}")
                                         }
+                                        IterateType::MessageReceipts(c) => {
+                                            // Forgive message receipts
+                                            tracing::trace!(
+                                                "[Iterate] missing key: {cid} from message receipts {c} in block {block_cid} at epoch {epoch}"
+                                            );
+                                            continue;
+                                        }
+                                        IterateType::EventsRoot(c) => {
+                                            // Forgive events
+                                            tracing::trace!(
+                                                "[Iterate] missing key: {cid} from events root {c} in block {block_cid} at epoch {epoch}"
+                                            );
+                                            continue;
+                                        }
                                     };
                                     return Poll::Ready(Some(Err(anyhow::anyhow!(
                                         "[Iterate] missing key: {cid} from {type_display} in block {block_cid} at epoch {epoch}"
@@ -318,6 +352,34 @@ impl<DB: Blockstore, T: Borrow<Tipset>, ITER: Iterator<Item = T> + Unpin> Stream
                                     .filter_map(ipld_to_cid)
                                     .collect(),
                             ));
+                            if *this.message_receipts {
+                                this.dfs.push_back(Iterate(
+                                    block.epoch,
+                                    *block.cid(),
+                                    IterateType::MessageReceipts(block.message_receipts),
+                                    DfsIter::from(block.message_receipts)
+                                        .filter_map(ipld_to_cid)
+                                        .collect(),
+                                ));
+                            }
+                            // ignore failure as receipts are not required by a lite snapshot
+                            if *this.events
+                                && let Ok(receipts) =
+                                    Receipt::get_receipts(this.db, block.message_receipts)
+                            {
+                                for receipt in receipts {
+                                    if let Some(events_root) = receipt.events_root() {
+                                        this.dfs.push_back(Iterate(
+                                            block.epoch,
+                                            *block.cid(),
+                                            IterateType::EventsRoot(events_root),
+                                            DfsIter::from(events_root)
+                                                .filter_map(ipld_to_cid)
+                                                .collect(),
+                                        ));
+                                    }
+                                }
+                            }
                         }
 
                         // Visit the block if it's within required depth. And a special case for `0`

src/rpc/methods/chain.rs

@@ -408,7 +408,7 @@ pub enum ForestChainExport {}
 impl RpcMethod<1> for ForestChainExport {
     const NAME: &'static str = "Forest.ChainExport";
     const PARAM_NAMES: [&'static str; 1] = ["params"];
-    const API_PATHS: BitFlags<ApiPaths> = ApiPaths::all();
+    const API_PATHS: BitFlags<ApiPaths> = ApiPaths::all_with_v2();
     const PERMISSION: Permission = Permission::Read;
 
     type Params = (ForestChainExportParams,);
@@ -425,6 +425,8 @@ impl RpcMethod<1> for ForestChainExport {
             recent_roots,
             output_path,
             tipset_keys: ApiTipsetKey(tsk),
+            include_receipts,
+            include_events,
             skip_checksum,
             dry_run,
         } = params;
@@ -448,6 +450,8 @@ impl RpcMethod<1> for ForestChainExport {
 
         let options = Some(ExportOptions {
             skip_checksum,
+            include_receipts,
+            include_events,
             seen: Default::default(),
         });
         let writer = if dry_run {
@@ -534,7 +538,7 @@ pub enum ForestChainExportStatus {}
 impl RpcMethod<0> for ForestChainExportStatus {
     const NAME: &'static str = "Forest.ChainExportStatus";
     const PARAM_NAMES: [&'static str; 0] = [];
-    const API_PATHS: BitFlags<ApiPaths> = ApiPaths::all();
+    const API_PATHS: BitFlags<ApiPaths> = ApiPaths::all_with_v2();
     const PERMISSION: Permission = Permission::Read;
 
     type Params = ();
@@ -575,7 +579,7 @@ pub enum ForestChainExportCancel {}
 impl RpcMethod<0> for ForestChainExportCancel {
     const NAME: &'static str = "Forest.ChainExportCancel";
     const PARAM_NAMES: [&'static str; 0] = [];
-    const API_PATHS: BitFlags<ApiPaths> = ApiPaths::all();
+    const API_PATHS: BitFlags<ApiPaths> = ApiPaths::all_with_v2();
     const PERMISSION: Permission = Permission::Read;
 
     type Params = ();
@@ -599,7 +603,7 @@ pub enum ForestChainExportDiff {}
 impl RpcMethod<1> for ForestChainExportDiff {
     const NAME: &'static str = "Forest.ChainExportDiff";
     const PARAM_NAMES: [&'static str; 1] = ["params"];
-    const API_PATHS: BitFlags<ApiPaths> = ApiPaths::all();
+    const API_PATHS: BitFlags<ApiPaths> = ApiPaths::all_with_v2();
     const PERMISSION: Permission = Permission::Read;
 
     type Params = (ForestChainExportDiffParams,);
@@ -682,6 +686,8 @@ impl RpcMethod<1> for ChainExport {
                 recent_roots,
                 output_path,
                 tipset_keys,
+                include_receipts: false,
+                include_events: false,
                 skip_checksum,
                 dry_run,
             },),
@@ -1468,6 +1474,10 @@ pub struct ForestChainExportParams {
     #[schemars(with = "LotusJson<ApiTipsetKey>")]
     #[serde(with = "crate::lotus_json")]
     pub tipset_keys: ApiTipsetKey,
+    #[serde(default)]
+    pub include_receipts: bool,
+    #[serde(default)]
+    pub include_events: bool,
     pub skip_checksum: bool,
     pub dry_run: bool,
 }