From 52df8b2b8a37908823f7a151200bfce34ef0df43 Mon Sep 17 00:00:00 2001 From: Yong Kang Date: Tue, 28 Oct 2025 01:25:23 +0000 Subject: [PATCH 1/7] feat(metrics): improve multiproof worker metrics Replace `inflight_multiproofs_histogram` with active worker metrics and add max worker gauges for better observability of proof worker pool utilization. --- .../src/tree/payload_processor/multiproof.rs | 37 ++++++++++++++++--- crates/trie/parallel/src/proof_task.rs | 32 ++++++++++++++++ 2 files changed, 63 insertions(+), 6 deletions(-) diff --git a/crates/engine/tree/src/tree/payload_processor/multiproof.rs b/crates/engine/tree/src/tree/payload_processor/multiproof.rs index a000e7a5adf..d25b58ee37e 100644 --- a/crates/engine/tree/src/tree/payload_processor/multiproof.rs +++ b/crates/engine/tree/src/tree/payload_processor/multiproof.rs @@ -9,7 +9,7 @@ use alloy_primitives::{ use crossbeam_channel::{unbounded, Receiver as CrossbeamReceiver, Sender as CrossbeamSender}; use dashmap::DashMap; use derive_more::derive::Deref; -use metrics::Histogram; +use metrics::{Gauge, Histogram}; use reth_metrics::Metrics; use reth_revm::state::EvmState; use reth_trie::{ @@ -349,6 +349,10 @@ impl MultiproofManager { proof_worker_handle: ProofWorkerHandle, proof_result_tx: CrossbeamSender, ) -> Self { + // Initialize the max worker gauges with the worker pool sizes + metrics.max_storage_workers.set(proof_worker_handle.total_storage_workers() as f64); + metrics.max_account_workers.set(proof_worker_handle.total_account_workers() as f64); + Self { inflight: 0, metrics, @@ -433,7 +437,12 @@ impl MultiproofManager { } self.inflight += 1; - self.metrics.inflight_multiproofs_histogram.record(self.inflight as f64); + self.metrics + .active_storage_workers_histogram + .record(self.proof_worker_handle.active_storage_workers() as f64); + self.metrics + .active_account_workers_histogram + .record(self.proof_worker_handle.active_account_workers() as f64); self.metrics .pending_storage_multiproofs_histogram .record(self.proof_worker_handle.pending_storage_tasks() as f64); @@ -445,7 +454,12 @@ impl MultiproofManager { /// Signals that a multiproof calculation has finished. fn on_calculation_complete(&mut self) { self.inflight = self.inflight.saturating_sub(1); - self.metrics.inflight_multiproofs_histogram.record(self.inflight as f64); + self.metrics + .active_storage_workers_histogram + .record(self.proof_worker_handle.active_storage_workers() as f64); + self.metrics + .active_account_workers_histogram + .record(self.proof_worker_handle.active_account_workers() as f64); self.metrics .pending_storage_multiproofs_histogram .record(self.proof_worker_handle.pending_storage_tasks() as f64); @@ -507,7 +521,12 @@ impl MultiproofManager { } self.inflight += 1; - self.metrics.inflight_multiproofs_histogram.record(self.inflight as f64); + self.metrics + .active_storage_workers_histogram + .record(self.proof_worker_handle.active_storage_workers() as f64); + self.metrics + .active_account_workers_histogram + .record(self.proof_worker_handle.active_account_workers() as f64); self.metrics .pending_storage_multiproofs_histogram .record(self.proof_worker_handle.pending_storage_tasks() as f64); @@ -520,8 +539,14 @@ impl MultiproofManager { #[derive(Metrics, Clone)] #[metrics(scope = "tree.root")] pub(crate) struct MultiProofTaskMetrics { - /// Histogram of inflight multiproofs. - pub inflight_multiproofs_histogram: Histogram, + /// Histogram of active storage workers processing proofs. + pub active_storage_workers_histogram: Histogram, + /// Histogram of active account workers processing proofs. + pub active_account_workers_histogram: Histogram, + /// Gauge for the maximum number of storage workers in the pool. + pub max_storage_workers: Gauge, + /// Gauge for the maximum number of account workers in the pool. + pub max_account_workers: Gauge, /// Histogram of pending storage multiproofs in the queue. pub pending_storage_multiproofs_histogram: Histogram, /// Histogram of pending account multiproofs in the queue. diff --git a/crates/trie/parallel/src/proof_task.rs b/crates/trie/parallel/src/proof_task.rs index 7e453cbc7c3..caca8687534 100644 --- a/crates/trie/parallel/src/proof_task.rs +++ b/crates/trie/parallel/src/proof_task.rs @@ -1017,6 +1017,10 @@ pub struct ProofWorkerHandle { /// Counter tracking available account workers. Workers decrement when starting work, /// increment when finishing. Used to determine whether to chunk multiproofs. account_available_workers: Arc, + /// Total number of storage workers spawned + storage_worker_count: usize, + /// Total number of account workers spawned + account_worker_count: usize, } impl ProofWorkerHandle { @@ -1118,6 +1122,8 @@ impl ProofWorkerHandle { account_work_tx, storage_available_workers, account_available_workers, + storage_worker_count, + account_worker_count, } } @@ -1141,6 +1147,32 @@ impl ProofWorkerHandle { self.account_work_tx.len() } + /// Returns the total number of storage workers in the pool. + pub const fn total_storage_workers(&self) -> usize { + self.storage_worker_count + } + + /// Returns the total number of account workers in the pool. + pub const fn total_account_workers(&self) -> usize { + self.account_worker_count + } + + /// Returns the number of storage workers currently processing tasks. + /// + /// This is calculated as total workers minus available workers. + pub fn active_storage_workers(&self) -> usize { + self.storage_worker_count + .saturating_sub(self.storage_available_workers.load(Ordering::Relaxed)) + } + + /// Returns the number of account workers currently processing tasks. + /// + /// This is calculated as total workers minus available workers. + pub fn active_account_workers(&self) -> usize { + self.account_worker_count + .saturating_sub(self.account_available_workers.load(Ordering::Relaxed)) + } + /// Dispatch a storage proof computation to storage worker pool /// /// The result will be sent via the `proof_result_sender` channel. From d85bd3033ab5081aaeb8430558e17b87b16b2784 Mon Sep 17 00:00:00 2001 From: Yong Kang Date: Tue, 28 Oct 2025 14:11:11 +0800 Subject: [PATCH 2/7] feat(metrics): replace in-flight multiproof panel with active worker metrics Replaces the "In-flight MultiProof requests" panel with "Active MultiProof Workers" panel that provides better visibility into multiproof worker pool utilization. The new panel displays on dual y-axes: - Left: Active worker counts for storage and account workers - Right: Utilization percentage (active/max) for each worker type - This gives us clear insight into whether worker pools are saturated, idle, or have capacity, and helps identify bottlenecks between storage vs account multiproof processing. --- etc/grafana/dashboards/overview.json | 207 ++++++++++++++++++++++++++- 1 file changed, 202 insertions(+), 5 deletions(-) diff --git a/etc/grafana/dashboards/overview.json b/etc/grafana/dashboards/overview.json index 46a465ca4a4..b0615183623 100644 --- a/etc/grafana/dashboards/overview.json +++ b/etc/grafana/dashboards/overview.json @@ -4308,7 +4308,168 @@ }, "unit": "none" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Storage utilization 0 percentile" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "percent" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Storage utilization 0.5 percentile" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "percent" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Storage utilization 0.9 percentile" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "percent" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Storage utilization 0.95 percentile" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "percent" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Storage utilization 1 percentile" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "percent" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Account utilization 0 percentile" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "percent" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Account utilization 0.5 percentile" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "percent" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Account utilization 0.9 percentile" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "percent" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Account utilization 0.95 percentile" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "percent" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Account utilization 1 percentile" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "percent" + } + ] + } + ] }, "gridPos": { "h": 8, @@ -4338,14 +4499,50 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "reth_tree_root_inflight_multiproofs_histogram{$instance_label=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", + "expr": "reth_tree_root_active_storage_workers_histogram{$instance_label=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", "instant": false, - "legendFormat": "{{quantile}} percentile", + "legendFormat": "Storage workers {{quantile}} percentile", "range": true, - "refId": "Branch Nodes" + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "reth_tree_root_active_account_workers_histogram{$instance_label=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", + "instant": false, + "legendFormat": "Account workers {{quantile}} percentile", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "(reth_tree_root_active_storage_workers_histogram{$instance_label=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"} / reth_tree_root_max_storage_workers{$instance_label=\"$instance\"}) * 100", + "instant": false, + "legendFormat": "Storage utilization {{quantile}} percentile", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "(reth_tree_root_active_account_workers_histogram{$instance_label=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"} / reth_tree_root_max_account_workers{$instance_label=\"$instance\"}) * 100", + "instant": false, + "legendFormat": "Account utilization {{quantile}} percentile", + "range": true, + "refId": "D" } ], - "title": "In-flight MultiProof requests", + "title": "Active MultiProof Workers", "type": "timeseries" }, { From 2be81e272ddac38f567912d8688c62916f407f64 Mon Sep 17 00:00:00 2001 From: Yong Kang Date: Tue, 28 Oct 2025 14:56:17 +0800 Subject: [PATCH 3/7] removed inflight --- .../engine/tree/src/tree/payload_processor/multiproof.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/crates/engine/tree/src/tree/payload_processor/multiproof.rs b/crates/engine/tree/src/tree/payload_processor/multiproof.rs index d25b58ee37e..2b9e6e5e02d 100644 --- a/crates/engine/tree/src/tree/payload_processor/multiproof.rs +++ b/crates/engine/tree/src/tree/payload_processor/multiproof.rs @@ -319,8 +319,6 @@ impl MultiproofInput { /// `ProofSequencer`. #[derive(Debug)] pub struct MultiproofManager { - /// Currently running calculations. - inflight: usize, /// Handle to the proof worker pools (storage and account). proof_worker_handle: ProofWorkerHandle, /// Cached storage proof roots for missed leaves; this maps @@ -354,7 +352,6 @@ impl MultiproofManager { metrics.max_account_workers.set(proof_worker_handle.total_account_workers() as f64); Self { - inflight: 0, metrics, proof_worker_handle, missed_leaves_storage_roots: Default::default(), @@ -436,7 +433,6 @@ impl MultiproofManager { return; } - self.inflight += 1; self.metrics .active_storage_workers_histogram .record(self.proof_worker_handle.active_storage_workers() as f64); @@ -453,7 +449,6 @@ impl MultiproofManager { /// Signals that a multiproof calculation has finished. fn on_calculation_complete(&mut self) { - self.inflight = self.inflight.saturating_sub(1); self.metrics .active_storage_workers_histogram .record(self.proof_worker_handle.active_storage_workers() as f64); @@ -520,7 +515,6 @@ impl MultiproofManager { return; } - self.inflight += 1; self.metrics .active_storage_workers_histogram .record(self.proof_worker_handle.active_storage_workers() as f64); @@ -608,7 +602,6 @@ pub(crate) struct MultiProofTaskMetrics { /// ▼ │ /// ┌──────────────────────────────────────────────────────────────┐ │ /// │ MultiproofManager │ │ -/// │ - Tracks inflight calculations │ │ /// │ - Deduplicates against fetched_proof_targets │ │ /// │ - Routes to appropriate worker pool │ │ /// └──┬───────────────────────────────────────────────────────────┘ │ @@ -649,7 +642,6 @@ pub(crate) struct MultiProofTaskMetrics { /// /// - **[`MultiproofManager`]**: Calculation orchestrator /// - Decides between fast path ([`EmptyProof`]) and worker dispatch -/// - Tracks inflight calculations /// - Routes storage-only vs full multiproofs to appropriate workers /// - Records metrics for monitoring /// From 5f93d04004b20d733cb1657af61150dc9c60deb4 Mon Sep 17 00:00:00 2001 From: Yong Kang Date: Tue, 28 Oct 2025 15:15:43 +0800 Subject: [PATCH 4/7] refactor(metrics): update Grafana dashboard to replace storage and account utilization percentiles with max worker metrics - Dashboard now shows the quantile curves plus dotted horizontal lines at the configured max worker pools --- etc/grafana/dashboards/overview.json | 162 +++------------------------ 1 file changed, 16 insertions(+), 146 deletions(-) diff --git a/etc/grafana/dashboards/overview.json b/etc/grafana/dashboards/overview.json index b0615183623..14dc2b35e17 100644 --- a/etc/grafana/dashboards/overview.json +++ b/etc/grafana/dashboards/overview.json @@ -4312,160 +4312,30 @@ { "matcher": { "id": "byName", - "options": "Storage utilization 0 percentile" + "options": "Max storage workers" }, "properties": [ { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "percent" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Storage utilization 0.5 percentile" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "percent" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Storage utilization 0.9 percentile" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "percent" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Storage utilization 0.95 percentile" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "percent" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Storage utilization 1 percentile" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "percent" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Account utilization 0 percentile" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "percent" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Account utilization 0.5 percentile" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "percent" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Account utilization 0.9 percentile" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "percent" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Account utilization 0.95 percentile" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "percent" + "id": "custom.lineStyle", + "value": { + "dash": [10, 10], + "fill": "dash" + } } ] }, { "matcher": { "id": "byName", - "options": "Account utilization 1 percentile" + "options": "Max account workers" }, "properties": [ { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "percent" + "id": "custom.lineStyle", + "value": { + "dash": [10, 10], + "fill": "dash" + } } ] } @@ -4523,9 +4393,9 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "(reth_tree_root_active_storage_workers_histogram{$instance_label=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"} / reth_tree_root_max_storage_workers{$instance_label=\"$instance\"}) * 100", + "expr": "reth_tree_root_max_storage_workers{$instance_label=\"$instance\"}", "instant": false, - "legendFormat": "Storage utilization {{quantile}} percentile", + "legendFormat": "Max storage workers", "range": true, "refId": "C" }, @@ -4535,9 +4405,9 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "(reth_tree_root_active_account_workers_histogram{$instance_label=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"} / reth_tree_root_max_account_workers{$instance_label=\"$instance\"}) * 100", + "expr": "reth_tree_root_max_account_workers{$instance_label=\"$instance\"}", "instant": false, - "legendFormat": "Account utilization {{quantile}} percentile", + "legendFormat": "Max account workers", "range": true, "refId": "D" } From b8c94720a536143b2c04df7e61bfa2374dff7912 Mon Sep 17 00:00:00 2001 From: Yong Kang Date: Tue, 28 Oct 2025 15:15:56 +0800 Subject: [PATCH 5/7] fix clippy --- crates/engine/tree/src/tree/payload_processor/multiproof.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/engine/tree/src/tree/payload_processor/multiproof.rs b/crates/engine/tree/src/tree/payload_processor/multiproof.rs index 2b9e6e5e02d..a249df56ad3 100644 --- a/crates/engine/tree/src/tree/payload_processor/multiproof.rs +++ b/crates/engine/tree/src/tree/payload_processor/multiproof.rs @@ -382,7 +382,7 @@ impl MultiproofManager { } /// Dispatches a single storage proof calculation to worker pool. - fn dispatch_storage_proof(&mut self, storage_multiproof_input: StorageMultiproofInput) { + fn dispatch_storage_proof(&self, storage_multiproof_input: StorageMultiproofInput) { let StorageMultiproofInput { hashed_state_update, hashed_address, @@ -448,7 +448,7 @@ impl MultiproofManager { } /// Signals that a multiproof calculation has finished. - fn on_calculation_complete(&mut self) { + fn on_calculation_complete(&self) { self.metrics .active_storage_workers_histogram .record(self.proof_worker_handle.active_storage_workers() as f64); @@ -464,7 +464,7 @@ impl MultiproofManager { } /// Dispatches a single multiproof calculation to worker pool. - fn dispatch_multiproof(&mut self, multiproof_input: MultiproofInput) { + fn dispatch_multiproof(&self, multiproof_input: MultiproofInput) { let MultiproofInput { source, hashed_state_update, From ac3291c3089103208b9e812877d9d79a81ad6206 Mon Sep 17 00:00:00 2001 From: Yong Kang Date: Tue, 28 Oct 2025 15:25:50 +0800 Subject: [PATCH 6/7] fix clippy --- crates/engine/tree/src/tree/payload_processor/multiproof.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/engine/tree/src/tree/payload_processor/multiproof.rs b/crates/engine/tree/src/tree/payload_processor/multiproof.rs index a249df56ad3..ca3bd380d4d 100644 --- a/crates/engine/tree/src/tree/payload_processor/multiproof.rs +++ b/crates/engine/tree/src/tree/payload_processor/multiproof.rs @@ -360,7 +360,7 @@ impl MultiproofManager { } /// Dispatches a new multiproof calculation to worker pools. - fn dispatch(&mut self, input: PendingMultiproofTask) { + fn dispatch(&self, input: PendingMultiproofTask) { // If there are no proof targets, we can just send an empty multiproof back immediately if input.proof_targets_is_empty() { debug!( From bca2b92b5e992fa06d491d1c214550fcbe45586b Mon Sep 17 00:00:00 2001 From: Yong Kang Date: Tue, 28 Oct 2025 16:45:44 +0800 Subject: [PATCH 7/7] added desc --- etc/grafana/dashboards/overview.json | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/grafana/dashboards/overview.json b/etc/grafana/dashboards/overview.json index 14dc2b35e17..aba888dba07 100644 --- a/etc/grafana/dashboards/overview.json +++ b/etc/grafana/dashboards/overview.json @@ -4347,6 +4347,7 @@ "x": 12, "y": 104 }, + "description": "The max metrics (Max storage workers and Max account workers) are displayed as dotted lines to highlight the configured upper limits.", "id": 256, "options": { "legend": {