From ed05df0a5a0698bd8deefb2ba885b93d73cc7d56 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Mon, 15 Sep 2025 18:18:51 +0800 Subject: [PATCH 01/10] feat: add prometheus metrics `network_height` and `network_version` --- src/daemon/mod.rs | 25 +++++++-- src/metrics/mod.rs | 6 ++- src/networks/metrics.rs | 110 ++++++++++++++++++++++++++++++++-------- src/shim/version.rs | 6 +++ 4 files changed, 121 insertions(+), 26 deletions(-) diff --git a/src/daemon/mod.rs b/src/daemon/mod.rs index 2bbfaa35f6bf..a46409d21dcd 100644 --- a/src/daemon/mod.rs +++ b/src/daemon/mod.rs @@ -15,8 +15,10 @@ use crate::cli_shared::{ chain_path, cli::{CliOpts, Config}, }; -use crate::daemon::context::{AppContext, DbType}; -use crate::daemon::db_util::import_chain_as_forest_car; +use crate::daemon::{ + context::{AppContext, DbType}, + db_util::import_chain_as_forest_car, +}; use crate::db::gc::SnapshotGarbageCollector; use crate::db::ttl::EthMappingCollector; use crate::libp2p::{Libp2pService, PeerManager}; @@ -202,10 +204,22 @@ async fn maybe_start_metrics_service( ); let db_directory = crate::db::db_engine::db_root(&chain_path(config))?; let db = ctx.db.writer().clone(); + let chain_store = ctx.state_manager.chain_store().clone(); + let get_chain_head_height = move || chain_store.heaviest_tipset().epoch(); + let chain_store = ctx.state_manager.chain_store().clone(); + let get_chain_head_network_version = move || { + let epoch = chain_store.heaviest_tipset().epoch(); + chain_store.chain_config.network_version(epoch) + }; services.spawn(async { - crate::metrics::init_prometheus(prometheus_listener, db_directory, db) - .await - .context("Failed to initiate prometheus server") + crate::metrics::init_prometheus( + prometheus_listener, + db_directory, + db, + get_chain_head_network_version, + ) + .await + .context("Failed to initiate prometheus server") }); crate::metrics::default_registry().register_collector(Box::new( @@ -215,6 +229,7 @@ async fn maybe_start_metrics_service( .chain_store() .genesis_block_header() .timestamp, + get_chain_head_height, ), )); } diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index 99921425f98f..edf41b6fe00a 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -3,7 +3,7 @@ pub mod db; -use crate::db::DBStatistics; +use crate::{db::DBStatistics, shim::version::NetworkVersion}; use axum::{Router, http::StatusCode, response::IntoResponse, routing::get}; use parking_lot::{RwLock, RwLockWriteGuard}; use prometheus_client::{ @@ -69,6 +69,7 @@ pub async fn init_prometheus( prometheus_listener: TcpListener, db_directory: PathBuf, db: Arc, + get_chain_head_network_version: impl Fn() -> NetworkVersion + Send + Sync + 'static, ) -> anyhow::Result<()> where DB: DBStatistics + Send + Sync + 'static, @@ -86,6 +87,9 @@ where DEFAULT_REGISTRY .write() .register_collector(Box::new(crate::metrics::db::DBCollector::new(db_directory))); + DEFAULT_REGISTRY.write().register_collector(Box::new( + crate::networks::metrics::NetworkVersionCollector::new(get_chain_head_network_version), + )); // Create an configure HTTP server let app = Router::new() diff --git a/src/networks/metrics.rs b/src/networks/metrics.rs index 058d899ac83b..79d2f945be3f 100644 --- a/src/networks/metrics.rs +++ b/src/networks/metrics.rs @@ -1,47 +1,117 @@ // Copyright 2019-2025 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT -use prometheus_client::{collector::Collector, encoding::EncodeMetric, metrics::gauge::Gauge}; +use educe::Educe; +use prometheus_client::{ + collector::Collector, + encoding::{DescriptorEncoder, EncodeMetric}, + metrics::gauge::Gauge, +}; use super::calculate_expected_epoch; +use crate::shim::{clock::ChainEpoch, version::NetworkVersion}; -#[derive(Debug)] -pub struct NetworkHeightCollector { +#[derive(Educe)] +#[educe(Debug)] +pub struct NetworkHeightCollector +where + F: Fn() -> ChainEpoch, +{ block_delay_secs: u32, genesis_timestamp: u64, - network_height: Gauge, + #[educe(Debug(ignore))] + get_chain_head_height: F, } -impl NetworkHeightCollector { - pub fn new(block_delay_secs: u32, genesis_timestamp: u64) -> Self { +impl NetworkHeightCollector +where + F: Fn() -> ChainEpoch, +{ + pub fn new(block_delay_secs: u32, genesis_timestamp: u64, get_chain_head_height: F) -> Self { Self { block_delay_secs, genesis_timestamp, - network_height: Gauge::default(), + get_chain_head_height, } } } -impl Collector for NetworkHeightCollector { +impl Collector for NetworkHeightCollector +where + F: Fn() -> ChainEpoch + Send + Sync + 'static, +{ fn encode( &self, mut encoder: prometheus_client::encoding::DescriptorEncoder, ) -> Result<(), std::fmt::Error> { + { + let network_height: Gauge = Default::default(); + let epoch = (self.get_chain_head_height)(); + network_height.set(epoch); + let metric_encoder = encoder.encode_descriptor( + "network_height", + "The current network height", + None, + network_height.metric_type(), + )?; + network_height.encode(metric_encoder)?; + } + { + let expected_network_height: Gauge = Default::default(); + let expected_epoch = calculate_expected_epoch( + chrono::Utc::now().timestamp() as u64, + self.genesis_timestamp, + self.block_delay_secs, + ); + expected_network_height.set(expected_epoch); + let metric_encoder = encoder.encode_descriptor( + "expected_network_height", + "The expected network height based on the current time and the genesis block time", + None, + expected_network_height.metric_type(), + )?; + expected_network_height.encode(metric_encoder)?; + } + Ok(()) + } +} + +#[derive(Educe)] +#[educe(Debug)] +pub struct NetworkVersionCollector +where + F: Fn() -> NetworkVersion, +{ + #[educe(Debug(ignore))] + get_chain_head_network_version: F, +} + +impl NetworkVersionCollector +where + F: Fn() -> NetworkVersion, +{ + pub fn new(get_chain_head_network_version: F) -> Self { + Self { + get_chain_head_network_version, + } + } +} + +impl Collector for NetworkVersionCollector +where + F: Fn() -> NetworkVersion + Send + Sync + 'static, +{ + fn encode(&self, mut encoder: DescriptorEncoder) -> Result<(), std::fmt::Error> { + let network_version = (self.get_chain_head_network_version)(); + let nv_gauge: Gauge = Default::default(); + nv_gauge.set(u32::from(network_version) as _); let metric_encoder = encoder.encode_descriptor( - "expected_network_height", - "The expected network height based on the current time and the genesis block time", + "network_version", + "Network version of the current chain head", None, - self.network_height.metric_type(), + nv_gauge.metric_type(), )?; - - let expected_epoch = calculate_expected_epoch( - chrono::Utc::now().timestamp() as u64, - self.genesis_timestamp, - self.block_delay_secs, - ); - self.network_height.set(expected_epoch); - self.network_height.encode(metric_encoder)?; - + nv_gauge.encode(metric_encoder)?; Ok(()) } } diff --git a/src/shim/version.rs b/src/shim/version.rs index 8b327774b0d1..de0222ae531f 100644 --- a/src/shim/version.rs +++ b/src/shim/version.rs @@ -88,6 +88,12 @@ impl From for NetworkVersion { } } +impl From for u32 { + fn from(value: NetworkVersion) -> Self { + value.0.into() + } +} + impl From for NetworkVersion { fn from(value: NetworkVersion_v2) -> Self { NetworkVersion((value as u32).into()) From d1c8129722a0de993fd1d9826f01fd67504fb39c Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Mon, 15 Sep 2025 18:22:33 +0800 Subject: [PATCH 02/10] changelog --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b22a418ac41..a502d86e19fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,7 +30,10 @@ ### Added - [#6057](https://github.com/ChainSafe/forest/issues/6057) Added `--no-progress-timeout` to `forest-cli f3 ready` subcommand to exit when F3 is stuck for the given timeout. -- [#6000](https://github.com/ChainSafe/forest/pull/6000) Add support for the `Filecoin.StateDecodeParams` API methods to enable decoding actors method params. + +- [#6000](https://github.com/ChainSafe/forest/pull/6000) Added support for the `Filecoin.StateDecodeParams` API methods to enable decoding actors method params. + +- [#6079](https://github.com/ChainSafe/forest/pull/6079) Added prometheus metrics `network_height` and `network_version`. ### Changed From 7eda8990d5068376a10cc0ec9f485af79bccdd0f Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Mon, 15 Sep 2025 18:36:28 +0800 Subject: [PATCH 03/10] fix GC --- src/daemon/mod.rs | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/daemon/mod.rs b/src/daemon/mod.rs index a46409d21dcd..4106ccc0b9bc 100644 --- a/src/daemon/mod.rs +++ b/src/daemon/mod.rs @@ -204,12 +204,23 @@ async fn maybe_start_metrics_service( ); let db_directory = crate::db::db_engine::db_root(&chain_path(config))?; let db = ctx.db.writer().clone(); - let chain_store = ctx.state_manager.chain_store().clone(); - let get_chain_head_height = move || chain_store.heaviest_tipset().epoch(); - let chain_store = ctx.state_manager.chain_store().clone(); + // Use `Weak` to not dead lock GC. + let chain_store = Arc::downgrade(ctx.state_manager.chain_store()); + let get_chain_head_height = move || { + chain_store + .upgrade() + .map(|cs| cs.heaviest_tipset().epoch()) + .unwrap_or_default() + }; + // Use `Weak` to not dead lock GC. + let chain_store = Arc::downgrade(ctx.state_manager.chain_store()); let get_chain_head_network_version = move || { - let epoch = chain_store.heaviest_tipset().epoch(); - chain_store.chain_config.network_version(epoch) + if let Some(cs) = chain_store.upgrade() { + let epoch = cs.heaviest_tipset().epoch(); + cs.chain_config.network_version(epoch) + } else { + NetworkVersion::V0 + } }; services.spawn(async { crate::metrics::init_prometheus( From 3537f0d0a643aa12e2d5db4dfd0931aba15cd112 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Mon, 15 Sep 2025 18:38:41 +0800 Subject: [PATCH 04/10] update docs --- docs/docs/users/reference/metrics.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/docs/users/reference/metrics.md b/docs/docs/users/reference/metrics.md index 292e1f7cb114..0ccbe57d90c2 100644 --- a/docs/docs/users/reference/metrics.md +++ b/docs/docs/users/reference/metrics.md @@ -16,7 +16,9 @@ title: Metrics | `peer_failure_total` | Counter | Count | Total number of failed peer requests | | `full_peers` | Gauge | Count | Number of healthy peers recognized by the node | | `bad_peers` | Gauge | Count | Number of bad peers recognized by the node | +| `network_height` | Gauge | Count | The current network height | | `expected_network_height` | Gauge | Count | The expected network height based on the current time and the genesis block time | +| `network_version` | Gauge | Count | Network version of the current chain head | | `forest_db_size` | Gauge | Bytes | Size of Forest database in bytes | | `bitswap_message_count` | Counter | Count | Number of `bitswap` messages. Indexed by `type` | | `bitswap_container_capacities` | Gauge | Count | Capacity for each `bitswap` container. Indexed by `type` | From 4a5eb4ff1c2231068be504582870104f6682d90f Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Mon, 15 Sep 2025 19:22:37 +0800 Subject: [PATCH 05/10] network_version_revision --- docs/docs/users/reference/metrics.md | 1 + src/daemon/mod.rs | 36 ++++++++-------- src/metrics/mod.rs | 7 ++-- src/networks/metrics.rs | 61 +++++++++++++++++++--------- src/networks/mod.rs | 42 ++++++++++++++++--- 5 files changed, 100 insertions(+), 47 deletions(-) diff --git a/docs/docs/users/reference/metrics.md b/docs/docs/users/reference/metrics.md index 0ccbe57d90c2..ca21d3c40ea4 100644 --- a/docs/docs/users/reference/metrics.md +++ b/docs/docs/users/reference/metrics.md @@ -19,6 +19,7 @@ title: Metrics | `network_height` | Gauge | Count | The current network height | | `expected_network_height` | Gauge | Count | The expected network height based on the current time and the genesis block time | | `network_version` | Gauge | Count | Network version of the current chain head | +| `network_version_revision` | Gauge | Count | Network version revision of the current chain head | | `forest_db_size` | Gauge | Bytes | Size of Forest database in bytes | | `bitswap_message_count` | Counter | Count | Number of `bitswap` messages. Indexed by `type` | | `bitswap_container_capacities` | Gauge | Count | Capacity for each `bitswap` container. Indexed by `type` | diff --git a/src/daemon/mod.rs b/src/daemon/mod.rs index 4106ccc0b9bc..2c886ec54d79 100644 --- a/src/daemon/mod.rs +++ b/src/daemon/mod.rs @@ -204,33 +204,29 @@ async fn maybe_start_metrics_service( ); let db_directory = crate::db::db_engine::db_root(&chain_path(config))?; let db = ctx.db.writer().clone(); + // Use `Weak` to not dead lock GC. let chain_store = Arc::downgrade(ctx.state_manager.chain_store()); - let get_chain_head_height = move || { + let get_chain_head_height = Arc::new(move || { chain_store .upgrade() .map(|cs| cs.heaviest_tipset().epoch()) .unwrap_or_default() - }; - // Use `Weak` to not dead lock GC. - let chain_store = Arc::downgrade(ctx.state_manager.chain_store()); - let get_chain_head_network_version = move || { - if let Some(cs) = chain_store.upgrade() { - let epoch = cs.heaviest_tipset().epoch(); - cs.chain_config.network_version(epoch) - } else { - NetworkVersion::V0 + }); + services.spawn({ + let chain_config = ctx.chain_config().clone(); + let get_chain_head_height = get_chain_head_height.clone(); + async { + crate::metrics::init_prometheus( + prometheus_listener, + db_directory, + db, + chain_config, + get_chain_head_height, + ) + .await + .context("Failed to initiate prometheus server") } - }; - services.spawn(async { - crate::metrics::init_prometheus( - prometheus_listener, - db_directory, - db, - get_chain_head_network_version, - ) - .await - .context("Failed to initiate prometheus server") }); crate::metrics::default_registry().register_collector(Box::new( diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index edf41b6fe00a..7bd9869738e4 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -3,7 +3,7 @@ pub mod db; -use crate::{db::DBStatistics, shim::version::NetworkVersion}; +use crate::{db::DBStatistics, networks::ChainConfig, shim::clock::ChainEpoch}; use axum::{Router, http::StatusCode, response::IntoResponse, routing::get}; use parking_lot::{RwLock, RwLockWriteGuard}; use prometheus_client::{ @@ -69,7 +69,8 @@ pub async fn init_prometheus( prometheus_listener: TcpListener, db_directory: PathBuf, db: Arc, - get_chain_head_network_version: impl Fn() -> NetworkVersion + Send + Sync + 'static, + chain_config: Arc, + get_chain_head_height: Arc ChainEpoch + Send + Sync + 'static>, ) -> anyhow::Result<()> where DB: DBStatistics + Send + Sync + 'static, @@ -88,7 +89,7 @@ where .write() .register_collector(Box::new(crate::metrics::db::DBCollector::new(db_directory))); DEFAULT_REGISTRY.write().register_collector(Box::new( - crate::networks::metrics::NetworkVersionCollector::new(get_chain_head_network_version), + crate::networks::metrics::NetworkVersionCollector::new(chain_config, get_chain_head_height), )); // Create an configure HTTP server diff --git a/src/networks/metrics.rs b/src/networks/metrics.rs index 79d2f945be3f..8c996c9c5459 100644 --- a/src/networks/metrics.rs +++ b/src/networks/metrics.rs @@ -1,6 +1,8 @@ // Copyright 2019-2025 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT +use std::sync::Arc; + use educe::Educe; use prometheus_client::{ collector::Collector, @@ -9,7 +11,7 @@ use prometheus_client::{ }; use super::calculate_expected_epoch; -use crate::shim::{clock::ChainEpoch, version::NetworkVersion}; +use crate::{networks::ChainConfig, shim::clock::ChainEpoch}; #[derive(Educe)] #[educe(Debug)] @@ -20,14 +22,18 @@ where block_delay_secs: u32, genesis_timestamp: u64, #[educe(Debug(ignore))] - get_chain_head_height: F, + get_chain_head_height: Arc, } impl NetworkHeightCollector where F: Fn() -> ChainEpoch, { - pub fn new(block_delay_secs: u32, genesis_timestamp: u64, get_chain_head_height: F) -> Self { + pub fn new( + block_delay_secs: u32, + genesis_timestamp: u64, + get_chain_head_height: Arc, + ) -> Self { Self { block_delay_secs, genesis_timestamp, @@ -80,38 +86,55 @@ where #[educe(Debug)] pub struct NetworkVersionCollector where - F: Fn() -> NetworkVersion, + F: Fn() -> ChainEpoch, { + chain_config: Arc, #[educe(Debug(ignore))] - get_chain_head_network_version: F, + get_chain_head_height: Arc, } impl NetworkVersionCollector where - F: Fn() -> NetworkVersion, + F: Fn() -> ChainEpoch, { - pub fn new(get_chain_head_network_version: F) -> Self { + pub fn new(chain_config: Arc, get_chain_head_height: Arc) -> Self { Self { - get_chain_head_network_version, + chain_config, + get_chain_head_height, } } } impl Collector for NetworkVersionCollector where - F: Fn() -> NetworkVersion + Send + Sync + 'static, + F: Fn() -> ChainEpoch + Send + Sync + 'static, { fn encode(&self, mut encoder: DescriptorEncoder) -> Result<(), std::fmt::Error> { - let network_version = (self.get_chain_head_network_version)(); - let nv_gauge: Gauge = Default::default(); - nv_gauge.set(u32::from(network_version) as _); - let metric_encoder = encoder.encode_descriptor( - "network_version", - "Network version of the current chain head", - None, - nv_gauge.metric_type(), - )?; - nv_gauge.encode(metric_encoder)?; + let epoch = (self.get_chain_head_height)(); + { + let network_version = self.chain_config.network_version(epoch); + let nv_gauge: Gauge = Default::default(); + nv_gauge.set(u32::from(network_version) as _); + let metric_encoder = encoder.encode_descriptor( + "network_version", + "Network version of the current chain head", + None, + nv_gauge.metric_type(), + )?; + nv_gauge.encode(metric_encoder)?; + } + { + let network_version_revision = self.chain_config.network_version_revision(epoch); + let nv_gauge: Gauge = Default::default(); + nv_gauge.set(network_version_revision); + let metric_encoder = encoder.encode_descriptor( + "network_version_revision", + "Network version revision of the current chain head", + None, + nv_gauge.metric_type(), + )?; + nv_gauge.encode(metric_encoder)?; + } Ok(()) } } diff --git a/src/networks/mod.rs b/src/networks/mod.rs index d067c8157372..8648b34fcf11 100644 --- a/src/networks/mod.rs +++ b/src/networks/mod.rs @@ -10,7 +10,9 @@ use fil_actors_shared::v13::runtime::Policy; use itertools::Itertools; use libp2p::Multiaddr; use serde::{Deserialize, Serialize}; +use strum::IntoEnumIterator; use strum_macros::Display; +use strum_macros::EnumIter; use tracing::warn; use crate::beacon::{BeaconPoint, BeaconSchedule, DrandBeacon, DrandConfig}; @@ -134,7 +136,7 @@ impl NetworkChain { } /// Defines the meaningful heights of the protocol. -#[derive(Debug, Display, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)] +#[derive(Debug, Display, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash, EnumIter)] #[cfg_attr(test, derive(derive_quickcheck_arbitrary::Arbitrary))] pub enum Height { Breeze, @@ -411,19 +413,36 @@ impl ChainConfig { } } - /// Returns the network version at the given epoch. - /// If the epoch is before the first upgrade, the genesis network version is returned. - pub fn network_version(&self, epoch: ChainEpoch) -> NetworkVersion { + fn network_height(&self, epoch: ChainEpoch) -> Option { self.height_infos .iter() .sorted_by_key(|(_, info)| info.epoch) .rev() .find(|(_, info)| epoch > info.epoch) - .map(|(height, _)| NetworkVersion::from(*height)) + .map(|(height, _)| *height) + } + + /// Returns the network version at the given epoch. + /// If the epoch is before the first upgrade, the genesis network version is returned. + pub fn network_version(&self, epoch: ChainEpoch) -> NetworkVersion { + self.network_height(epoch) + .map(NetworkVersion::from) .unwrap_or(self.genesis_network_version()) .max(self.genesis_network) } + /// Returns the network version revision at the given epoch for distinguishing network upgrades + /// that do not bump the network version. + pub fn network_version_revision(&self, epoch: ChainEpoch) -> i64 { + if let Some(height) = self.network_height(epoch) { + let nv = NetworkVersion::from(height); + if let Some(rev0_height) = Height::iter().find(|h| NetworkVersion::from(*h) == nv) { + return (height as i64) - (rev0_height as i64); + } + } + 0 + } + pub fn get_beacon_schedule(&self, genesis_ts: u64) -> BeaconSchedule { let ds_iter = match self.network { NetworkChain::Mainnet => mainnet::DRAND_SCHEDULE.iter(), @@ -721,4 +740,17 @@ mod tests { ChainConfig::devnet(); ChainConfig::butterflynet(); } + + #[test] + fn network_version() { + let cfg = ChainConfig::calibnet(); + assert_eq!(cfg.network_version(1_013_134 - 1), NetworkVersion::V20); + assert_eq!(cfg.network_version(1_013_134), NetworkVersion::V20); + assert_eq!(cfg.network_version(1_013_134 + 1), NetworkVersion::V21); + assert_eq!(cfg.network_version_revision(1_013_134 + 1), 0); + assert_eq!(cfg.network_version(1_070_494), NetworkVersion::V21); + assert_eq!(cfg.network_version_revision(1_070_494), 0); + assert_eq!(cfg.network_version(1_070_494 + 1), NetworkVersion::V21); + assert_eq!(cfg.network_version_revision(1_070_494 + 1), 1); + } } From 3b5bd97035bebadb22afbde67f9912b6e358d768 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Mon, 15 Sep 2025 19:49:36 +0800 Subject: [PATCH 06/10] update docs --- docs/docs/users/reference/metrics.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/docs/docs/users/reference/metrics.md b/docs/docs/users/reference/metrics.md index ca21d3c40ea4..15dd36530822 100644 --- a/docs/docs/users/reference/metrics.md +++ b/docs/docs/users/reference/metrics.md @@ -282,6 +282,15 @@ head_epoch 2519530 ``` +
+ Example `network_height` output +``` +# HELP network_height The current network height +# TYPE network_height gauge +network_height 3020349 +``` +
+
Example `expected_network_height` output ``` @@ -291,6 +300,24 @@ expected_network_height 2519530 ```
+
+ Example `network_version` output +``` +# HELP network_version Network version of the current chain head +# TYPE network_version gauge +network_version 27 +``` +
+ +
+ Example `network_version_revision` output +``` +# HELP network_version_revision Network version revision of the current chain head +# TYPE network_version_revision gauge +network_version_revision 0 +``` +
+
Example `build_info` output ``` From 1bea7fe8bfb50283eeb5ccb7cb5a6c5c7bc90760 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Mon, 15 Sep 2025 19:56:42 +0800 Subject: [PATCH 07/10] reset prometheus registry after GC --- src/daemon/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/daemon/mod.rs b/src/daemon/mod.rs index 2c886ec54d79..46a94ec62aec 100644 --- a/src/daemon/mod.rs +++ b/src/daemon/mod.rs @@ -555,6 +555,8 @@ pub(super) async fn start_services( shutdown_send: mpsc::Sender<()>, on_app_context_and_db_initialized: impl Fn(&AppContext), ) -> anyhow::Result<()> { + // Cleanup default prometheus metrics registry + *crate::metrics::default_registry() = Default::default(); let mut services = JoinSet::new(); let network = config.chain(); let ctx = AppContext::init(opts, &config).await?; From 2be478f43378592d12ddcdf30740fd27cd09a6da Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Mon, 15 Sep 2025 21:28:07 +0800 Subject: [PATCH 08/10] add actor_version metrics --- CHANGELOG.md | 2 +- docs/docs/users/reference/metrics.md | 10 ++++++++ src/daemon/mod.rs | 35 +++++++++++++++++++------ src/metrics/mod.rs | 7 ++++- src/networks/metrics.rs | 38 ++++++++++++++++++++++------ src/networks/mod.rs | 4 +-- src/shim/state_tree.rs | 14 +++++++++- 7 files changed, 89 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a502d86e19fe..a5506eea5603 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,7 +33,7 @@ - [#6000](https://github.com/ChainSafe/forest/pull/6000) Added support for the `Filecoin.StateDecodeParams` API methods to enable decoding actors method params. -- [#6079](https://github.com/ChainSafe/forest/pull/6079) Added prometheus metrics `network_height` and `network_version`. +- [#6079](https://github.com/ChainSafe/forest/pull/6079) Added prometheus metrics `network_height`, `network_version`, `network_version_revision` and `actor_version`. ### Changed diff --git a/docs/docs/users/reference/metrics.md b/docs/docs/users/reference/metrics.md index 15dd36530822..d46ac120abaf 100644 --- a/docs/docs/users/reference/metrics.md +++ b/docs/docs/users/reference/metrics.md @@ -20,6 +20,7 @@ title: Metrics | `expected_network_height` | Gauge | Count | The expected network height based on the current time and the genesis block time | | `network_version` | Gauge | Count | Network version of the current chain head | | `network_version_revision` | Gauge | Count | Network version revision of the current chain head | +| `actor_version` | Gauge | Count | Actor version of the current chain head | | `forest_db_size` | Gauge | Bytes | Size of Forest database in bytes | | `bitswap_message_count` | Counter | Count | Number of `bitswap` messages. Indexed by `type` | | `bitswap_container_capacities` | Gauge | Count | Capacity for each `bitswap` container. Indexed by `type` | @@ -318,6 +319,15 @@ network_version_revision 0 ```
+
+ Example `actor_version` output +``` +# HELP actor_version Actor version of the current chain head +# TYPE actor_version gauge +actor_version 17 +``` +
+
Example `build_info` output ``` diff --git a/src/daemon/mod.rs b/src/daemon/mod.rs index 46a94ec62aec..ad5f1f0a41d6 100644 --- a/src/daemon/mod.rs +++ b/src/daemon/mod.rs @@ -28,6 +28,7 @@ use crate::rpc::RPCState; use crate::rpc::eth::filter::EthEventHandler; use crate::rpc::start_rpc; use crate::shim::clock::ChainEpoch; +use crate::shim::state_tree::StateTree; use crate::shim::version::NetworkVersion; use crate::utils; use crate::utils::{proofs_api::ensure_proof_params_downloaded, version::FOREST_VERSION_STRING}; @@ -205,13 +206,30 @@ async fn maybe_start_metrics_service( let db_directory = crate::db::db_engine::db_root(&chain_path(config))?; let db = ctx.db.writer().clone(); - // Use `Weak` to not dead lock GC. - let chain_store = Arc::downgrade(ctx.state_manager.chain_store()); - let get_chain_head_height = Arc::new(move || { - chain_store - .upgrade() - .map(|cs| cs.heaviest_tipset().epoch()) - .unwrap_or_default() + let get_chain_head_height = Arc::new({ + // Use `Weak` to not dead lock GC. + let chain_store = Arc::downgrade(ctx.state_manager.chain_store()); + move || { + chain_store + .upgrade() + .map(|cs| cs.heaviest_tipset().epoch()) + .unwrap_or_default() + } + }); + let get_chain_head_actor_version = Arc::new({ + // Use `Weak` to not dead lock GC. + let chain_store = Arc::downgrade(ctx.state_manager.chain_store()); + move || { + if let Some(cs) = chain_store.upgrade() + && let Ok(state) = + StateTree::new_from_root(cs.db.clone(), cs.heaviest_tipset().parent_state()) + && let Ok(bundle_meta) = state.get_actor_bundle_metadata() + && let Ok(actor_version) = bundle_meta.actor_major_version() + { + return actor_version; + } + 0 + } }); services.spawn({ let chain_config = ctx.chain_config().clone(); @@ -223,6 +241,7 @@ async fn maybe_start_metrics_service( db, chain_config, get_chain_head_height, + get_chain_head_actor_version, ) .await .context("Failed to initiate prometheus server") @@ -555,7 +574,7 @@ pub(super) async fn start_services( shutdown_send: mpsc::Sender<()>, on_app_context_and_db_initialized: impl Fn(&AppContext), ) -> anyhow::Result<()> { - // Cleanup default prometheus metrics registry + // Cleanup the default prometheus metrics registry *crate::metrics::default_registry() = Default::default(); let mut services = JoinSet::new(); let network = config.chain(); diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index 7bd9869738e4..8f325b92564a 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -71,6 +71,7 @@ pub async fn init_prometheus( db: Arc, chain_config: Arc, get_chain_head_height: Arc ChainEpoch + Send + Sync + 'static>, + get_chain_head_actor_version: Arc u64 + Send + Sync + 'static>, ) -> anyhow::Result<()> where DB: DBStatistics + Send + Sync + 'static, @@ -89,7 +90,11 @@ where .write() .register_collector(Box::new(crate::metrics::db::DBCollector::new(db_directory))); DEFAULT_REGISTRY.write().register_collector(Box::new( - crate::networks::metrics::NetworkVersionCollector::new(chain_config, get_chain_head_height), + crate::networks::metrics::NetworkVersionCollector::new( + chain_config, + get_chain_head_height, + get_chain_head_actor_version, + ), )); // Create an configure HTTP server diff --git a/src/networks/metrics.rs b/src/networks/metrics.rs index 8c996c9c5459..b3b71638a4bc 100644 --- a/src/networks/metrics.rs +++ b/src/networks/metrics.rs @@ -84,30 +84,40 @@ where #[derive(Educe)] #[educe(Debug)] -pub struct NetworkVersionCollector +pub struct NetworkVersionCollector where - F: Fn() -> ChainEpoch, + F1: Fn() -> ChainEpoch, + F2: Fn() -> u64, { chain_config: Arc, #[educe(Debug(ignore))] - get_chain_head_height: Arc, + get_chain_head_height: Arc, + #[educe(Debug(ignore))] + get_chain_head_actor_version: Arc, } -impl NetworkVersionCollector +impl NetworkVersionCollector where - F: Fn() -> ChainEpoch, + F1: Fn() -> ChainEpoch, + F2: Fn() -> u64, { - pub fn new(chain_config: Arc, get_chain_head_height: Arc) -> Self { + pub fn new( + chain_config: Arc, + get_chain_head_height: Arc, + get_chain_head_actor_version: Arc, + ) -> Self { Self { chain_config, get_chain_head_height, + get_chain_head_actor_version, } } } -impl Collector for NetworkVersionCollector +impl Collector for NetworkVersionCollector where - F: Fn() -> ChainEpoch + Send + Sync + 'static, + F1: Fn() -> ChainEpoch + Send + Sync + 'static, + F2: Fn() -> u64 + Send + Sync + 'static, { fn encode(&self, mut encoder: DescriptorEncoder) -> Result<(), std::fmt::Error> { let epoch = (self.get_chain_head_height)(); @@ -135,6 +145,18 @@ where )?; nv_gauge.encode(metric_encoder)?; } + { + let actor_version = (self.get_chain_head_actor_version)(); + let av_gauge: Gauge = Default::default(); + av_gauge.set(actor_version as _); + let metric_encoder = encoder.encode_descriptor( + "actor_version", + "Actor version of the current chain head", + None, + av_gauge.metric_type(), + )?; + av_gauge.encode(metric_encoder)?; + } Ok(()) } } diff --git a/src/networks/mod.rs b/src/networks/mod.rs index 8648b34fcf11..fc3723107de1 100644 --- a/src/networks/mod.rs +++ b/src/networks/mod.rs @@ -29,8 +29,8 @@ pub use network_name::{GenesisNetworkName, StateNetworkName}; mod actors_bundle; pub use actors_bundle::{ - ACTOR_BUNDLES, ACTOR_BUNDLES_METADATA, ActorBundleInfo, generate_actor_bundle, - get_actor_bundles_metadata, + ACTOR_BUNDLES, ACTOR_BUNDLES_METADATA, ActorBundleInfo, ActorBundleMetadata, + generate_actor_bundle, get_actor_bundles_metadata, }; mod drand; diff --git a/src/shim/state_tree.rs b/src/shim/state_tree.rs index a86b3fcb96c8..da5910cde9f7 100644 --- a/src/shim/state_tree.rs +++ b/src/shim/state_tree.rs @@ -5,7 +5,10 @@ use std::{ sync::Arc, }; -use crate::shim::actors::account; +use crate::{ + networks::{ACTOR_BUNDLES_METADATA, ActorBundleMetadata}, + shim::actors::account, +}; use anyhow::{Context as _, anyhow, bail}; use cid::Cid; use fvm_ipld_blockstore::Blockstore; @@ -194,6 +197,15 @@ where .with_context(|| format!("Actor not found: addr={addr}")) } + /// Get the actor bundle metadata + pub fn get_actor_bundle_metadata(&self) -> anyhow::Result<&ActorBundleMetadata> { + let system_actor_code = self.get_required_actor(&Address::SYSTEM_ACTOR)?.code; + ACTOR_BUNDLES_METADATA + .values() + .find(|v| v.manifest.get_system() == system_actor_code) + .with_context(|| format!("actor bundle not found for system actor {system_actor_code}")) + } + /// Get actor state from an address. Will be resolved to ID address. pub fn get_actor(&self, addr: &Address) -> anyhow::Result> { match self { From 9965521836591e7a1542f32fa74f42f9c2504d41 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Tue, 16 Sep 2025 19:20:08 +0800 Subject: [PATCH 09/10] revert metrics registry reset changes --- src/daemon/mod.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/daemon/mod.rs b/src/daemon/mod.rs index ad5f1f0a41d6..424b31b93e2b 100644 --- a/src/daemon/mod.rs +++ b/src/daemon/mod.rs @@ -574,8 +574,6 @@ pub(super) async fn start_services( shutdown_send: mpsc::Sender<()>, on_app_context_and_db_initialized: impl Fn(&AppContext), ) -> anyhow::Result<()> { - // Cleanup the default prometheus metrics registry - *crate::metrics::default_registry() = Default::default(); let mut services = JoinSet::new(); let network = config.chain(); let ctx = AppContext::init(opts, &config).await?; From 821be7d98e9326af20293deb6922562d69953c26 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Tue, 16 Sep 2025 20:40:44 +0800 Subject: [PATCH 10/10] collect "head_epoch" metrics with collector --- CHANGELOG.md | 2 +- docs/docs/users/reference/metrics.md | 10 ---------- src/chain/store/chain_store.rs | 6 +----- src/chain_sync/metrics.rs | 11 +---------- src/networks/metrics.rs | 4 ++-- 5 files changed, 5 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5bb28aefc0c..4e8a894670c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,7 +33,7 @@ - [#6000](https://github.com/ChainSafe/forest/pull/6000) Added support for the `Filecoin.StateDecodeParams` API methods to enable decoding actors method params. -- [#6079](https://github.com/ChainSafe/forest/pull/6079) Added prometheus metrics `network_height`, `network_version`, `network_version_revision` and `actor_version`. +- [#6079](https://github.com/ChainSafe/forest/pull/6079) Added prometheus metrics `network_version`, `network_version_revision` and `actor_version`. - [#6068](https://github.com/ChainSafe/forest/issues/6068) Added `--index-backfill-epochs` to `forest-tool api serve`. diff --git a/docs/docs/users/reference/metrics.md b/docs/docs/users/reference/metrics.md index d46ac120abaf..9ad3cb27afb1 100644 --- a/docs/docs/users/reference/metrics.md +++ b/docs/docs/users/reference/metrics.md @@ -16,7 +16,6 @@ title: Metrics | `peer_failure_total` | Counter | Count | Total number of failed peer requests | | `full_peers` | Gauge | Count | Number of healthy peers recognized by the node | | `bad_peers` | Gauge | Count | Number of bad peers recognized by the node | -| `network_height` | Gauge | Count | The current network height | | `expected_network_height` | Gauge | Count | The expected network height based on the current time and the genesis block time | | `network_version` | Gauge | Count | Network version of the current chain head | | `network_version_revision` | Gauge | Count | Network version revision of the current chain head | @@ -283,15 +282,6 @@ head_epoch 2519530 ```
-
- Example `network_height` output -``` -# HELP network_height The current network height -# TYPE network_height gauge -network_height 3020349 -``` -
-
Example `expected_network_height` output ``` diff --git a/src/chain/store/chain_store.rs b/src/chain/store/chain_store.rs index 20f36ee3b5ed..395fe7840c51 100644 --- a/src/chain/store/chain_store.rs +++ b/src/chain/store/chain_store.rs @@ -6,6 +6,7 @@ use super::{ index::{ChainIndex, ResolveNullTipset}, tipset_tracker::TipsetTracker, }; +use crate::db::{EthMappingsStore, EthMappingsStoreExt, IndicesStore, IndicesStoreExt}; use crate::interpreter::{BlockMessages, VMTrace}; use crate::libp2p_bitswap::{BitswapStoreRead, BitswapStoreReadWrite}; use crate::message::{ChainMessage, Message as MessageTrait, SignedMessage}; @@ -22,10 +23,6 @@ use crate::{ blocks::{CachingBlockHeader, Tipset, TipsetKey, TxMeta}, db::HeaviestTipsetKeyProvider, }; -use crate::{ - chain_sync::metrics, - db::{EthMappingsStore, EthMappingsStoreExt, IndicesStore, IndicesStoreExt}, -}; use crate::{fil_cns, utils::cache::SizeTrackingLruCache}; use ahash::{HashMap, HashMapExt, HashSet}; use anyhow::Context as _; @@ -146,7 +143,6 @@ where /// Sets heaviest tipset pub fn set_heaviest_tipset(&self, ts: Arc) -> Result<(), Error> { - metrics::HEAD_EPOCH.set(ts.epoch()); self.heaviest_tipset_key_provider .set_heaviest_tipset_key(ts.key())?; if self.publisher.send(HeadChange::Apply(ts)).is_err() { diff --git a/src/chain_sync/metrics.rs b/src/chain_sync/metrics.rs index 95efe617cd74..e620a92d2185 100644 --- a/src/chain_sync/metrics.rs +++ b/src/chain_sync/metrics.rs @@ -3,7 +3,7 @@ use prometheus_client::{ encoding::{EncodeLabelKey, EncodeLabelSet, EncodeLabelValue, LabelSetEncoder}, - metrics::{counter::Counter, family::Family, gauge::Gauge, histogram::Histogram}, + metrics::{counter::Counter, family::Family, histogram::Histogram}, }; use std::sync::LazyLock; @@ -44,15 +44,6 @@ pub static INVALID_TIPSET_TOTAL: LazyLock = LazyLock::new(|| { ); metric }); -pub static HEAD_EPOCH: LazyLock = LazyLock::new(|| { - let metric = Gauge::default(); - crate::metrics::default_registry().register( - "head_epoch", - "Latest epoch synchronized to the node", - metric.clone(), - ); - metric -}); #[derive(Clone, Debug, Hash, PartialEq, Eq)] pub struct Libp2pMessageKindLabel(&'static str); diff --git a/src/networks/metrics.rs b/src/networks/metrics.rs index b3b71638a4bc..3eb62693e382 100644 --- a/src/networks/metrics.rs +++ b/src/networks/metrics.rs @@ -55,8 +55,8 @@ where let epoch = (self.get_chain_head_height)(); network_height.set(epoch); let metric_encoder = encoder.encode_descriptor( - "network_height", - "The current network height", + "head_epoch", + "Latest epoch synchronized to the node", None, network_height.metric_type(), )?;