diff --git a/src/chain/store/chain_store.rs b/src/chain/store/chain_store.rs index 32e6e08ec932..f920c4f55e76 100644 --- a/src/chain/store/chain_store.rs +++ b/src/chain/store/chain_store.rs @@ -537,12 +537,11 @@ where { let amt = Amt::::load(root, db)?; - let mut cids = Vec::new(); - for i in 0..amt.count() { - if let Some(c) = amt.get(i)? { - cids.push(*c); - } - } + let mut cids = Vec::with_capacity(amt.count() as usize); + amt.for_each_cacheless(|_, c| { + cids.push(*c); + Ok(()) + })?; Ok(cids) } diff --git a/src/utils/encoding/cid_de_cbor.rs b/src/utils/encoding/cid_de_cbor.rs index 1daa49b7b1ef..95c2725bece2 100644 --- a/src/utils/encoding/cid_de_cbor.rs +++ b/src/utils/encoding/cid_de_cbor.rs @@ -63,7 +63,8 @@ impl<'de> DeserializeSeed<'de> for FilterCids<'_> { where V: de::MapAccess<'de>, { - self.0.reserve(visitor.size_hint().unwrap_or(0)); + let capacity = super::size_hint_cautious_cid(visitor.size_hint().unwrap_or(0)); + self.0.reserve(capacity); // This is where recursion happens, we unravel each [`Ipld`] till we reach all // the nodes. while visitor @@ -83,7 +84,8 @@ impl<'de> DeserializeSeed<'de> for FilterCids<'_> { where A: SeqAccess<'de>, { - self.0.reserve(seq.size_hint().unwrap_or(0)); + let capacity = super::size_hint_cautious_cid(seq.size_hint().unwrap_or(0)); + self.0.reserve(capacity); // This is where recursion happens, we unravel each [`Ipld`] till we reach all // the nodes. while seq.next_element_seed(FilterCids(self.0))?.is_some() { diff --git a/src/utils/encoding/mod.rs b/src/utils/encoding/mod.rs index 7dc7561164bf..0230c0ee0cd3 100644 --- a/src/utils/encoding/mod.rs +++ b/src/utils/encoding/mod.rs @@ -9,6 +9,14 @@ use serde::{Deserializer, Serializer, de, ser}; mod fallback_de_ipld_dagcbor; +/// Limit the the number of bytes that are used for pre-allocating `Vec`s. This follows what `serde` is +/// doing internally with `serde::private::size_hint::cautious()`. +/// The limit is set to 1 MiB, which is a reasonable upper bound for most use cases. +fn size_hint_cautious_cid(size_hint: usize) -> usize { + const MAX_PREALLOC_BYTES: usize = 1024 * 1024; + size_hint.min(MAX_PREALLOC_BYTES / std::mem::size_of::()) +} + /// This method will attempt to de-serialize given bytes using the regular /// `serde_ipld_dagcbor::from_slice`. Due to a historical issue in Lotus (see more in /// [FIP-0027](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0027.md), we must still @@ -258,4 +266,13 @@ mod tests { matches!(from_slice_with_fallback::(&corrupted).unwrap(), Ipld::Bytes(bytes) if bytes == [0x63, 0x74, 0x68, 0x75, 0x6c, 0xa0, 0xa1]) ) } + + #[test] + fn size_hint_cautious_test() { + assert_eq!(size_hint_cautious_cid(0), 0); + assert_eq!( + size_hint_cautious_cid(1024 * 1024), + 1024 * 1024 / std::mem::size_of::() + ); + } }