diff --git a/Cargo.lock b/Cargo.lock index 13e49335790b..d43227f7002d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -450,6 +450,36 @@ dependencies = [ "url", ] +[[package]] +name = "attribute-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0053e96dd3bec5b4879c23a138d6ef26f2cb936c9cdc96274ac2b9ed44b5bb54" +dependencies = [ + "attribute-derive-macro", + "derive-where", + "manyhow", + "proc-macro2", + "quote", + "syn 2.0.104", +] + +[[package]] +name = "attribute-derive-macro" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "463b53ad0fd5b460af4b1915fe045ff4d946d025fb6c4dc3337752eaa980f71b" +dependencies = [ + "collection_literals", + "interpolator", + "manyhow", + "proc-macro-utils", + "proc-macro2", + "quote", + "quote-use", + "syn 2.0.104", +] + [[package]] name = "autocfg" version = "1.5.0" @@ -1276,6 +1306,12 @@ dependencies = [ "thiserror 2.0.12", ] +[[package]] +name = "collection_literals" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b3f65b8fb8e88ba339f7d23a390fe1b0896217da05e2a66c584c9b29a91df8" + [[package]] name = "colorchoice" version = "1.0.4" @@ -1990,6 +2026,17 @@ dependencies = [ "syn 2.0.104", ] +[[package]] +name = "derive-where" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "510c292c8cf384b1a340b816a9a6cf2599eb8f566a44949024af88418000c50b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", +] + [[package]] name = "derive_arbitrary" version = "1.4.1" @@ -3092,6 +3139,8 @@ dependencies = [ "fvm_shared 2.11.1", "fvm_shared 3.13.1", "fvm_shared 4.7.2", + "get-size-derive2", + "get-size2", "gethostname", "git-version", "glob", @@ -3169,7 +3218,7 @@ dependencies = [ "serde", "serde_ipld_dagcbor", "serde_json", - "serde_tuple 1.1.1", + "serde_tuple 1.1.2", "serde_with", "serde_yaml", "serial_test", @@ -3836,6 +3885,26 @@ dependencies = [ "typenum", ] +[[package]] +name = "get-size-derive2" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "028f3cfad7c3e3b1d8d04ef0a1c03576f2d62800803fe1301a4cd262849f2dea" +dependencies = [ + "attribute-derive", + "quote", + "syn 2.0.104", +] + +[[package]] +name = "get-size2" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a09c2043819a3def7bfbb4927e7df96aab0da4cfd8824484b22d0c94e84458e" +dependencies = [ + "get-size-derive2", +] + [[package]] name = "gethostname" version = "1.0.2" @@ -4647,6 +4716,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "interpolator" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71dd52191aae121e8611f1e8dc3e324dd0dd1dee1e6dd91d10ee07a3cfb4d9d8" + [[package]] name = "io-uring" version = "0.7.8" @@ -5733,6 +5808,29 @@ dependencies = [ "libc", ] +[[package]] +name = "manyhow" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b33efb3ca6d3b07393750d4030418d594ab1139cee518f0dc88db70fec873587" +dependencies = [ + "manyhow-macros", + "proc-macro2", + "quote", + "syn 2.0.104", +] + +[[package]] +name = "manyhow-macros" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46fce34d199b78b6e6073abf984c9cf5fd3e9330145a93ee0738a7443e371495" +dependencies = [ + "proc-macro-utils", + "proc-macro2", + "quote", +] + [[package]] name = "matchers" version = "0.1.0" @@ -6927,6 +7025,17 @@ dependencies = [ "toml_edit 0.22.27", ] +[[package]] +name = "proc-macro-utils" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeaf08a13de400bc215877b5bdc088f241b12eb42f0a548d3390dc1c56bb7071" +dependencies = [ + "proc-macro2", + "quote", + "smallvec", +] + [[package]] name = "proc-macro2" version = "1.0.95" @@ -7142,6 +7251,28 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "quote-use" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9619db1197b497a36178cfc736dc96b271fe918875fbf1344c436a7e93d0321e" +dependencies = [ + "quote", + "quote-use-macros", +] + +[[package]] +name = "quote-use-macros" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82ebfb7faafadc06a7ab141a6f67bcfb24cb8beb158c6fe933f2f035afa99f35" +dependencies = [ + "proc-macro-utils", + "proc-macro2", + "quote", + "syn 2.0.104", +] + [[package]] name = "r-efi" version = "5.3.0" @@ -8100,12 +8231,12 @@ dependencies = [ [[package]] name = "serde_tuple" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be56ac39d7339f9b7fad7f252eb9f542e09f78ed6949c04d14c6f3d17bbbc90" +checksum = "52569c5296679bd28e2457f067f97d270077df67da0340647da5412c8eac8d9e" dependencies = [ "serde", - "serde_tuple_macros 1.1.1", + "serde_tuple_macros 1.1.2", ] [[package]] @@ -8121,9 +8252,9 @@ dependencies = [ [[package]] name = "serde_tuple_macros" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19543df7d1738995a8a68fea985c76428da623e10384c2878fd3f7272934bce0" +checksum = "2f46c707781471741d5f2670edb36476479b26e94cf43efe21ca3c220b97ef2e" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index e7c77be8a576..37c0211862da 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -93,6 +93,8 @@ fvm_shared2 = { package = "fvm_shared", version = "~2.11" } fvm_shared3 = { package = "fvm_shared", version = "~3.13", features = ["proofs"] } fvm_shared4 = { package = "fvm_shared", version = "~4.7", features = ["proofs"] } gethostname = "1" +get-size2 = { version = "0.5.2", features = ["derive"] } +get-size-derive2 = "*" git-version = "0.3" group = "0.13" hex = { version = "0.4", features = ["serde"] } @@ -183,7 +185,7 @@ semver = "1" serde = { version = "1", default-features = false, features = ["derive"] } serde_ipld_dagcbor = "0.6" serde_json = { version = "1", features = ["raw_value"] } -serde_tuple = "1" +serde_tuple = "1.1.2" serde_with = { version = "3", features = ["chrono_0_4"] } serde_yaml = "0.9" sha2 = { version = "0.10", default-features = false } diff --git a/src/beacon/beacon_entries.rs b/src/beacon/beacon_entries.rs index 06fac9ed0542..5b020a8c4af3 100644 --- a/src/beacon/beacon_entries.rs +++ b/src/beacon/beacon_entries.rs @@ -4,6 +4,7 @@ use crate::utils::encoding::serde_byte_array; use byteorder::{BigEndian, ByteOrder as _}; use digest::Digest as _; +use get_size2::GetSize; use serde_tuple::{self, Deserialize_tuple, Serialize_tuple}; /// The result from getting an entry from `Drand`. @@ -12,7 +13,17 @@ use serde_tuple::{self, Deserialize_tuple, Serialize_tuple}; /// This beacon entry is stored on chain in the block header. #[cfg_attr(test, derive(derive_quickcheck_arbitrary::Arbitrary))] #[derive( - Clone, Debug, Default, Eq, PartialEq, Hash, Ord, PartialOrd, Serialize_tuple, Deserialize_tuple, + Clone, + Debug, + Default, + Eq, + PartialEq, + Hash, + Ord, + PartialOrd, + Serialize_tuple, + Deserialize_tuple, + GetSize, )] pub struct BeaconEntry { round: u64, diff --git a/src/blocks/election_proof.rs b/src/blocks/election_proof.rs index 0d07c388fda4..3a07a797558f 100644 --- a/src/blocks/election_proof.rs +++ b/src/blocks/election_proof.rs @@ -130,13 +130,26 @@ impl Poiss { } } +use get_size2::GetSize; + /// Proofs generated by a miner which determines the reward they earn. /// This is generated from hashing a partial ticket and using the hash to /// generate a value. #[derive( - Clone, Debug, PartialEq, PartialOrd, Eq, Default, Ord, Serialize_tuple, Deserialize_tuple, Hash, + Clone, + Debug, + PartialEq, + PartialOrd, + Eq, + Default, + Ord, + Serialize_tuple, + Deserialize_tuple, + Hash, + GetSize, )] pub struct ElectionProof { + #[get_size(size = 200)] pub win_count: i64, pub vrfproof: VRFProof, } diff --git a/src/blocks/header.rs b/src/blocks/header.rs index dbbb19f3af6f..2c51e3f4052f 100644 --- a/src/blocks/header.rs +++ b/src/blocks/header.rs @@ -18,6 +18,7 @@ use crate::utils::{cid::CidCborExt as _, encoding::blake2b_256}; use cid::Cid; use fvm_ipld_blockstore::Blockstore; use fvm_ipld_encoding::CborStore as _; +use get_size2::GetSize; use num::BigInt; use serde::{Deserialize, Serialize}; use serde_tuple::{Deserialize_tuple, Serialize_tuple}; @@ -35,41 +36,85 @@ static FILECOIN_GENESIS_CID: std::sync::LazyLock = std::sync::LazyLock::new pub static GENESIS_BLOCK_PARENTS: std::sync::LazyLock = std::sync::LazyLock::new(|| nunny::vec![*FILECOIN_GENESIS_CID].into()); -#[derive(Deserialize_tuple, Serialize_tuple, Clone, Hash, Eq, PartialEq, Debug)] +#[derive(Deserialize_tuple, Serialize_tuple, Clone, Hash, Eq, PartialEq, Debug, GetSize)] pub struct RawBlockHeader { /// The address of the miner actor that mined this block + #[get_size(size = 20)] pub miner_address: Address, pub ticket: Option, pub election_proof: Option, /// The verifiable oracle randomness used to elect this block's author leader pub beacon_entries: Vec, + #[get_size(size_fn = post_proof_size_helper)] pub winning_post_proof: Vec, /// The set of parents this block was based on. /// Typically one, but can be several in the case where there were multiple /// winning ticket-holders for an epoch + #[get_size(size_fn = tipset_key_size_helper)] pub parents: TipsetKey, /// The aggregate chain weight of the parent set #[serde(with = "crate::shim::fvm_shared_latest::bigint::bigint_ser")] + #[get_size(size_fn = bigint_size_helper)] pub weight: BigInt, /// The period in which a new block is generated. /// There may be multiple rounds in an epoch. pub epoch: ChainEpoch, /// The CID of the parent state root after calculating parent tipset. + #[get_size(size_fn = cid_size_helper)] pub state_root: Cid, /// The CID of the root of an array of `MessageReceipts` + #[get_size(size_fn = cid_size_helper)] pub message_receipts: Cid, /// The CID of the Merkle links for `bls_messages` and `secp_messages` + #[get_size(size_fn = cid_size_helper)] pub messages: Cid, /// Aggregate signature of miner in block + #[get_size(size_fn = signature_size_helper)] pub bls_aggregate: Option, /// Block creation time, in seconds since the Unix epoch pub timestamp: u64, + #[get_size(size_fn = signature_size_helper)] pub signature: Option, pub fork_signal: u64, /// The base fee of the parent block + #[get_size(size_fn = tokenamount_size_helper)] pub parent_base_fee: TokenAmount, } +fn cid_size_helper(cid: &Cid) -> usize { + // Cid is a wrapper around a Vec, so we can use its size directly + cid.hash().digest().len() +} + +fn post_proof_size_helper(post_proof: &Vec) -> usize { + post_proof + .iter() + .map(|p| std::mem::size_of_val(p) + p.proof_bytes.len()) + .sum::() +} + +fn tipset_key_size_helper(tipset_key: &TipsetKey) -> usize { + // according to the smallcid docs, the median size of it is 40 bytes + tipset_key.len() * 40 +} + +fn bigint_size_helper(bigint: &BigInt) -> usize { + // BigInt is a wrapper around a Vec, so we can use its size directly + bigint.to_bytes_le().1.len() +} + +fn tokenamount_size_helper(token_amount: &TokenAmount) -> usize { + // TokenAmount is a wrapper around BigInt, so we can use its size directly + bigint_size_helper(token_amount.atto()) +} + +fn signature_size_helper(signature: &Option) -> usize { + // Signature is a wrapper around a Vec, so we can use its size directly + signature + .as_ref() + .map_or(0, |s| s.bytes().len() + std::mem::size_of::()) +} + #[cfg(test)] impl Default for RawBlockHeader { fn default() -> Self { diff --git a/src/blocks/ticket.rs b/src/blocks/ticket.rs index d221012ae79e..7fd2e4427c78 100644 --- a/src/blocks/ticket.rs +++ b/src/blocks/ticket.rs @@ -2,13 +2,24 @@ // SPDX-License-Identifier: Apache-2.0, MIT use crate::blocks::VRFProof; +use get_size2::GetSize; use serde_tuple::{self, Deserialize_tuple, Serialize_tuple}; /// A Ticket is a marker of a tick of the blockchain's clock. It is the source /// of randomness for proofs of storage and leader election. It is generated /// by the miner of a block using a `VRF` and a `VDF`. #[derive( - Clone, Debug, PartialEq, Eq, Default, Serialize_tuple, Deserialize_tuple, Hash, PartialOrd, Ord, + Clone, + Debug, + PartialEq, + Eq, + Default, + Serialize_tuple, + Deserialize_tuple, + Hash, + PartialOrd, + Ord, + GetSize, )] pub struct Ticket { /// A proof output by running a `VRF` on the `VDFResult` of the parent diff --git a/src/blocks/vrf_proof.rs b/src/blocks/vrf_proof.rs index b5ce17f92f40..f0ed128308c3 100644 --- a/src/blocks/vrf_proof.rs +++ b/src/blocks/vrf_proof.rs @@ -2,11 +2,14 @@ // SPDX-License-Identifier: Apache-2.0, MIT use crate::utils::encoding::{blake2b_256, serde_byte_array}; +use get_size2::GetSize; use serde::{Deserialize, Serialize}; /// The output from running a VRF proof. #[cfg_attr(test, derive(derive_quickcheck_arbitrary::Arbitrary))] -#[derive(Clone, Debug, PartialEq, Eq, Ord, PartialOrd, Default, Serialize, Deserialize, Hash)] +#[derive( + Clone, Debug, PartialEq, Eq, Ord, PartialOrd, Default, Serialize, Deserialize, Hash, GetSize, +)] pub struct VRFProof(#[serde(with = "serde_byte_array")] pub Vec); impl VRFProof { diff --git a/src/chain/store/index.rs b/src/chain/store/index.rs index cc33e174f717..eac18ed1326f 100644 --- a/src/chain/store/index.rs +++ b/src/chain/store/index.rs @@ -1,19 +1,23 @@ // Copyright 2019-2025 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT +use std::sync::atomic::AtomicU64; +use std::time::Instant; use std::{num::NonZeroUsize, sync::Arc}; use crate::beacon::{BeaconEntry, IGNORE_DRAND_VAR}; -use crate::blocks::{Tipset, TipsetKey}; +use crate::blocks::{RawBlockHeader, Tipset, TipsetKey}; use crate::chain::Error; use crate::metrics; use crate::shim::clock::ChainEpoch; use crate::utils::misc::env::is_env_truthy; use fvm_ipld_blockstore::Blockstore; +use get_size2::GetSize; use itertools::Itertools; use lru::LruCache; use nonzero_ext::nonzero; use parking_lot::Mutex; +use tracing::info; const DEFAULT_TIPSET_CACHE_SIZE: NonZeroUsize = nonzero!(131072_usize); @@ -27,6 +31,8 @@ pub struct ChainIndex { /// `Blockstore` pointer needed to load tipsets from cold storage. pub db: DB, + + counter: AtomicU64, } #[derive(Debug, Clone, Copy)] @@ -41,12 +47,54 @@ pub enum ResolveNullTipset { impl ChainIndex { pub fn new(db: DB) -> Self { let ts_cache = Mutex::new(LruCache::new(DEFAULT_TIPSET_CACHE_SIZE)); - Self { ts_cache, db } + Self { + ts_cache, + db, + counter: AtomicU64::new(0), + } } /// Loads a tipset from memory given the tipset keys and cache. Semantically /// identical to [`Tipset::load`] but the result is cached. pub fn load_tipset(&self, tsk: &TipsetKey) -> Result>, Error> { + self.counter + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + // every now and then, check the cache size + if self.counter.load(std::sync::atomic::Ordering::Relaxed) % 10000 == 0 { + let timer = Instant::now(); + let total = self + .ts_cache + .lock() + .iter() + .map(|(tsk, ts)| { + let tipset_size = tsk.len() * 40; + let ts_size = ts + .block_headers() + .iter() + .map(|bh| { + let raw = bh.clone().into_raw(); + raw.get_size() + std::mem::size_of::() + }) + .sum::(); + tipset_size + ts_size + }) + .sum::(); + info!( + "Tipset cache size: {} bytes ({} MB), {} entries in cache, took {} ms", + total, + total / (1024 * 1024), + self.ts_cache.lock().len(), + timer.elapsed().as_millis() + ); + + metrics::TIPSET_CACHE_SIZE + .get_or_create(&metrics::values::TIPSET) + .set(total as i64); + } + + metrics::TIPSET_CACHE_LEN + .get_or_create(&metrics::values::TIPSET) + .set(self.ts_cache.lock().len() as i64); if !is_env_truthy("FOREST_TIPSET_CACHE_DISABLED") { if let Some(ts) = self.ts_cache.lock().get(tsk) { metrics::LRU_CACHE_HIT diff --git a/src/chain_sync/bad_block_cache.rs b/src/chain_sync/bad_block_cache.rs index 6b6f2b278a67..52fa321819cf 100644 --- a/src/chain_sync/bad_block_cache.rs +++ b/src/chain_sync/bad_block_cache.rs @@ -4,18 +4,27 @@ use std::num::NonZeroUsize; use cid::Cid; +use get_size2::GetSize; use lru::LruCache; use nonzero_ext::nonzero; use parking_lot::Mutex; +use crate::metrics; + /// Thread-safe cache for tracking bad blocks. /// This cache is checked before validating a block, to ensure no duplicate /// work. -#[derive(Debug)] +#[derive(Debug, GetSize)] pub struct BadBlockCache { + #[get_size(size_fn = cache_helper)] cache: Mutex>, } +fn cache_helper(cache: &Mutex>) -> usize { + let cache = cache.lock(); + cache.iter().map(|(k, v)| k.get_size() + v.get_size()).sum() +} + impl Default for BadBlockCache { fn default() -> Self { Self::new(nonzero!(1usize << 15)) @@ -31,7 +40,16 @@ impl BadBlockCache { /// Puts a bad block `Cid` in the cache with a given reason. pub fn put(&self, c: Cid, reason: String) -> Option { - self.cache.lock().put(c, reason) + println!("Adding bad block to cache: {} with reason: {}", c, reason); + use get_size2::GetSize; + let v = self.cache.lock().put(c, reason); + crate::metrics::BAD_BLOCK_CACHE_LEN + .get_or_create(&metrics::values::BLOCK) + .set(self.cache.lock().len() as i64); + crate::metrics::BAD_BLOCK_CACHE_SIZE + .get_or_create(&metrics::values::BLOCK) + .set(self.get_size() as i64); + v } /// Returns `Some` with the reason if the block CID is in bad block cache. diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index 99921425f98f..ba83ec479e0f 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -11,6 +11,7 @@ use prometheus_client::{ metrics::{ counter::Counter, family::Family, + gauge::Gauge, histogram::{Histogram, exponential_buckets}, }, }; @@ -27,6 +28,46 @@ pub fn default_registry<'a>() -> RwLockWriteGuard<'a, prometheus_client::registr DEFAULT_REGISTRY.write() } +pub static BAD_BLOCK_CACHE_SIZE: LazyLock> = LazyLock::new(|| { + let metric = Family::default(); + DEFAULT_REGISTRY.write().register( + "bad_block_cache_size", + "Current size of the bad block cache", + metric.clone(), + ); + metric +}); + +pub static BAD_BLOCK_CACHE_LEN: LazyLock> = LazyLock::new(|| { + let metric = Family::default(); + DEFAULT_REGISTRY.write().register( + "bad_block_cache_len", + "Current len of the bad block cache", + metric.clone(), + ); + metric +}); + +pub static TIPSET_CACHE_SIZE: LazyLock> = LazyLock::new(|| { + let metric = Family::default(); + DEFAULT_REGISTRY.write().register( + "tipset_cache_size", + "Current size of the tipset cache", + metric.clone(), + ); + metric +}); + +pub static TIPSET_CACHE_LEN: LazyLock> = LazyLock::new(|| { + let metric = Family::default(); + DEFAULT_REGISTRY.write().register( + "tipset_cache_len", + "Current len of the tipset cache", + metric.clone(), + ); + metric +}); + pub static LRU_CACHE_HIT: LazyLock> = LazyLock::new(|| { let metric = Family::default(); DEFAULT_REGISTRY @@ -154,6 +195,7 @@ pub mod values { pub const TIPSET: KindLabel = KindLabel::new("tipset"); /// tipset cache in state manager pub const STATE_MANAGER_TIPSET: KindLabel = KindLabel::new("sm_tipset"); + pub const BLOCK: KindLabel = KindLabel::new("block"); } pub fn default_histogram() -> Histogram {