diff --git a/.gitignore b/.gitignore index b7aa064..0b44918 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,5 @@ /docs CLAUDE.md - -.worktrees/ .claude/ -.omc/ \ No newline at end of file +.worktrees/ diff --git a/crates/engine-api/Cargo.toml b/crates/engine-api/Cargo.toml index 58cc46a..d47aaf4 100644 --- a/crates/engine-api/Cargo.toml +++ b/crates/engine-api/Cargo.toml @@ -17,6 +17,7 @@ morph-payload-types.workspace = true morph-primitives = { workspace = true, features = ["reth-codec"] } # reth +reth-metrics.workspace = true reth-node-api.workspace = true reth-payload-builder.workspace = true reth-payload-primitives.workspace = true @@ -30,6 +31,9 @@ alloy-eips.workspace = true alloy-primitives.workspace = true alloy-rpc-types-engine.workspace = true +# metrics +metrics.workspace = true + # misc async-trait.workspace = true auto_impl.workspace = true diff --git a/crates/engine-api/src/builder.rs b/crates/engine-api/src/builder.rs index 4117236..9e40c84 100644 --- a/crates/engine-api/src/builder.rs +++ b/crates/engine-api/src/builder.rs @@ -2,7 +2,9 @@ //! //! This module provides the concrete Morph L2 Engine API implementation and supporting helpers. -use crate::{EngineApiResult, MorphEngineApiError, MorphL2EngineApi}; +use crate::{ + EngineApiResult, MorphEngineApiError, MorphL2EngineApi, metrics::MorphEngineApiMetrics, +}; use alloy_consensus::{ BlockHeader, EMPTY_OMMER_ROOT_HASH, Header, proofs::calculate_transaction_root, }; @@ -49,6 +51,9 @@ pub struct RealMorphL2EngineApi { /// Engine-state tracker updated from consensus engine events (authoritative) and local FCU /// success hints (fast path). engine_state_tracker: Arc, + + /// Prometheus metrics for custom Morph L2 Engine API endpoints and chain head health. + metrics: MorphEngineApiMetrics, } #[derive(Debug, Clone, Copy, PartialEq)] @@ -151,9 +156,21 @@ impl RealMorphL2EngineApi { chain_spec, engine_handle, engine_state_tracker, + metrics: MorphEngineApiMetrics::default(), } } + /// Updates `head_block_timegap_seconds` gauge after a successful block import. + fn record_head_metrics(&self, block_timestamp: u64) { + let now_secs = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + self.metrics + .head_block_timegap_seconds + .set(now_secs.saturating_sub(block_timestamp) as f64); + } + /// Returns a reference to the provider. pub fn provider(&self) -> &Provider { &self.provider @@ -186,7 +203,15 @@ where &self, params: AssembleL2BlockParams, ) -> EngineApiResult { - let built_payload = self.build_l2_payload(params, None, None).await?; + let started = Instant::now(); + let result = self.build_l2_payload(params, None, None).await; + self.metrics + .assemble_l2_block_duration_seconds + .record(started.elapsed()); + + let built_payload = result.inspect_err(|_| { + self.metrics.assemble_l2_block_failures_total.increment(1); + })?; let executable_data = built_payload.executable_data; tracing::debug!( @@ -220,6 +245,10 @@ where actual = data.number, "cannot validate block with discontinuous block number" ); + self.metrics.validate_l2_block_failures_total.increment(1); + self.metrics + .validate_l2_block_duration_seconds + .record(validate_started.elapsed()); return Err(MorphEngineApiError::DiscontinuousBlockNumber { expected: current_head.number + 1, actual: data.number, @@ -233,6 +262,10 @@ where actual = %data.parent_hash, "parent hash mismatch" ); + self.metrics.validate_l2_block_failures_total.increment(1); + self.metrics + .validate_l2_block_duration_seconds + .record(validate_started.elapsed()); return Err(MorphEngineApiError::WrongParentHash { expected: current_head.hash, actual: data.parent_hash, @@ -250,6 +283,10 @@ where error = %err, "failed to convert executable data for validation" ); + self.metrics.validate_l2_block_failures_total.increment(1); + self.metrics + .validate_l2_block_duration_seconds + .record(validate_started.elapsed()); return Ok(GenericResponse { success: false }); } }; @@ -265,6 +302,10 @@ where error = %err, "engine new_payload failed during validate_l2_block" ); + self.metrics.validate_l2_block_failures_total.increment(1); + self.metrics + .validate_l2_block_duration_seconds + .record(validate_started.elapsed()); return Ok(GenericResponse { success: false }); } }; @@ -294,10 +335,18 @@ where "validate_l2_block timing" ); + self.metrics + .validate_l2_block_duration_seconds + .record(validate_started.elapsed()); + if !success { + self.metrics.validate_l2_block_failures_total.increment(1); + } + Ok(GenericResponse { success }) } async fn new_l2_block(&self, data: ExecutableL2Data) -> EngineApiResult<()> { + let started = Instant::now(); tracing::debug!( target: "morph::engine", block_number = data.number, @@ -321,6 +370,9 @@ where current_number = current_number, "ignoring past block number" ); + self.metrics + .new_l2_block_duration_seconds + .record(started.elapsed()); return Ok(()); } // Discontinuous block number @@ -330,6 +382,10 @@ where actual_number = data.number, "cannot new block with discontinuous block number" ); + self.metrics.new_l2_block_failures_total.increment(1); + self.metrics + .new_l2_block_duration_seconds + .record(started.elapsed()); return Err(MorphEngineApiError::DiscontinuousBlockNumber { expected: expected_number, actual: data.number, @@ -344,6 +400,10 @@ where actual = %data.parent_hash, "wrong parent hash" ); + self.metrics.new_l2_block_failures_total.increment(1); + self.metrics + .new_l2_block_duration_seconds + .record(started.elapsed()); return Err(MorphEngineApiError::WrongParentHash { expected: current_head.hash, actual: data.parent_hash, @@ -352,7 +412,20 @@ where let block_hash = data.hash; let block_number = data.number; - self.import_l2_block_via_engine(data).await?; + let block_timestamp = data.timestamp; + self.import_l2_block_via_engine(data) + .await + .inspect_err(|_| { + self.metrics.new_l2_block_failures_total.increment(1); + self.metrics + .new_l2_block_duration_seconds + .record(started.elapsed()); + })?; + + self.metrics + .new_l2_block_duration_seconds + .record(started.elapsed()); + self.record_head_metrics(block_timestamp); tracing::debug!( target: "morph::engine", @@ -365,6 +438,7 @@ where } async fn new_safe_l2_block(&self, mut data: SafeL2Data) -> EngineApiResult { + let started = Instant::now(); tracing::debug!( target: "morph::engine", block_number = data.number, @@ -375,12 +449,18 @@ where let latest_number = self.current_head()?.number; if data.number != latest_number + 1 { + self.metrics.new_safe_l2_block_failures_total.increment(1); + self.metrics + .new_safe_l2_block_duration_seconds + .record(started.elapsed()); return Err(MorphEngineApiError::DiscontinuousBlockNumber { expected: latest_number + 1, actual: data.number, }); } + let block_timestamp = data.timestamp; + // 2. Assemble the block from SafeL2Data inputs. let assemble_params = AssembleL2BlockParams { number: data.number, @@ -391,14 +471,33 @@ where let built_payload = self .build_l2_payload(assemble_params, Some(data.gas_limit), data.base_fee_per_gas) - .await?; + .await + .inspect_err(|_| { + self.metrics.new_safe_l2_block_failures_total.increment(1); + self.metrics + .new_safe_l2_block_duration_seconds + .record(started.elapsed()); + })?; let executable_data = built_payload.executable_data; // Save hash before moving executable_data into the import call. let block_hash = executable_data.hash; // 3. Import the block through reth engine tree and return the in-path header // (do not rely on immediate DB visibility after FCU). - let header = self.import_l2_block_via_engine(executable_data).await?; + let header = self + .import_l2_block_via_engine(executable_data) + .await + .inspect_err(|_| { + self.metrics.new_safe_l2_block_failures_total.increment(1); + self.metrics + .new_safe_l2_block_duration_seconds + .record(started.elapsed()); + })?; + + self.metrics + .new_safe_l2_block_duration_seconds + .record(started.elapsed()); + self.record_head_metrics(block_timestamp); // Update safe block tag and seed finalized for memory cleanup. // diff --git a/crates/engine-api/src/lib.rs b/crates/engine-api/src/lib.rs index 65d229b..931cb27 100644 --- a/crates/engine-api/src/lib.rs +++ b/crates/engine-api/src/lib.rs @@ -23,6 +23,7 @@ mod api; mod builder; mod error; +mod metrics; mod rpc; mod validator; diff --git a/crates/engine-api/src/metrics.rs b/crates/engine-api/src/metrics.rs new file mode 100644 index 0000000..20b95e5 --- /dev/null +++ b/crates/engine-api/src/metrics.rs @@ -0,0 +1,65 @@ +//! Metrics for the Morph L2 Engine API. +//! +//! Tracks per-method latency and failure counts for custom Morph Engine API +//! endpoints, plus a chain head health gauge analogous to geth's +//! `chain/head/timegap`. + +use reth_metrics::{ + Metrics, + metrics::{Counter, Gauge, Histogram}, +}; + +/// Metrics for the custom Morph L2 Engine API endpoints. +/// +/// Each method tracks: +/// - A latency histogram (`*_duration_seconds`) +/// - A failure counter (`*_failures_total`) +/// +/// Additionally, a chain-head gauge is updated after each successful block +/// import (equivalent to geth's `chain/head/timegap`): +/// - `head_block_timegap_seconds` +#[derive(Metrics, Clone)] +#[metrics(scope = "morph.engine")] +pub(crate) struct MorphEngineApiMetrics { + // ------------------------------------------------------------------------- + // assembleL2Block + // ------------------------------------------------------------------------- + /// Latency for `engine_assembleL2Block` calls. + pub(crate) assemble_l2_block_duration_seconds: Histogram, + /// Number of `engine_assembleL2Block` calls that returned an error. + pub(crate) assemble_l2_block_failures_total: Counter, + + // ------------------------------------------------------------------------- + // newL2Block + // ------------------------------------------------------------------------- + /// Latency for `engine_newL2Block` calls. + pub(crate) new_l2_block_duration_seconds: Histogram, + /// Number of `engine_newL2Block` calls that returned an error. + pub(crate) new_l2_block_failures_total: Counter, + + // ------------------------------------------------------------------------- + // validateL2Block + // ------------------------------------------------------------------------- + /// Latency for `engine_validateL2Block` calls. + pub(crate) validate_l2_block_duration_seconds: Histogram, + /// Number of `engine_validateL2Block` calls that returned `success: false`. + pub(crate) validate_l2_block_failures_total: Counter, + + // ------------------------------------------------------------------------- + // newSafeL2Block + // ------------------------------------------------------------------------- + /// Latency for `engine_newSafeL2Block` calls. + pub(crate) new_safe_l2_block_duration_seconds: Histogram, + /// Number of `engine_newSafeL2Block` calls that returned an error. + pub(crate) new_safe_l2_block_failures_total: Counter, + + // ------------------------------------------------------------------------- + // Chain head health + // ------------------------------------------------------------------------- + /// Seconds elapsed since the latest imported block's timestamp. + /// + /// A large value indicates the node is behind the chain tip. + /// Updated on every successful block import. + /// Analogous to geth's `chain/head/timegap` gauge. + pub(crate) head_block_timegap_seconds: Gauge, +} diff --git a/crates/payload/builder/Cargo.toml b/crates/payload/builder/Cargo.toml index 9266ce0..ab42ef7 100644 --- a/crates/payload/builder/Cargo.toml +++ b/crates/payload/builder/Cargo.toml @@ -40,6 +40,10 @@ alloy-rlp.workspace = true # Revm revm.workspace = true +# Metrics +metrics.workspace = true +reth-metrics.workspace = true + # Utils tracing.workspace = true thiserror.workspace = true diff --git a/crates/payload/builder/src/builder.rs b/crates/payload/builder/src/builder.rs index 7741a12..5de02b8 100644 --- a/crates/payload/builder/src/builder.rs +++ b/crates/payload/builder/src/builder.rs @@ -1,5 +1,6 @@ //! Morph payload builder implementation. +use crate::metrics::MorphPayloadBuilderMetrics; use crate::{MorphBuilderConfig, MorphPayloadBuilderError, config::PayloadBuildingBreaker}; use alloy_consensus::{BlockHeader, Transaction, Typed2718}; use alloy_eips::eip2718::Encodable2718; @@ -31,7 +32,7 @@ use reth_revm::{database::StateProviderDatabase, db::State}; use reth_storage_api::{StateProvider, StateProviderFactory}; use reth_transaction_pool::{BestTransactionsAttributes, PoolTransaction, TransactionPool}; use revm::context_interface::Block as RevmBlock; -use std::sync::Arc; +use std::{sync::Arc, time::Instant}; /// Reads the withdraw trie root from the L2MessageQueue contract storage. fn read_withdraw_trie_root(db: &mut DB) -> Result { @@ -177,6 +178,7 @@ where cancel, best_payload, builder_config: self.config.clone(), + metrics: MorphPayloadBuilderMetrics::default(), }; let state_provider = self.client.state_by_block_hash(ctx.parent().hash())?; @@ -247,6 +249,8 @@ struct MorphPayloadBuilderCtx { best_payload: Option, /// Builder configuration with limits. builder_config: MorphBuilderConfig, + /// Prometheus metrics for this payload build job. + metrics: MorphPayloadBuilderMetrics, } impl MorphPayloadBuilderCtx { @@ -311,6 +315,14 @@ impl MorphPayloadBuilderCtx { // Check if adding this transaction would exceed block gas limit if info.cumulative_gas_used + tx_gas > block_gas_limit { + tracing::warn!( + target: "payload_builder", + tx_index = tx_idx, + tx_gas, + cumulative_gas_used = info.cumulative_gas_used, + block_gas_limit, + "L1 message transaction would exceed block gas limit; aborting build" + ); gas_spent_by_transactions.push(tx_gas); return Err(PayloadBuilderError::other( MorphPayloadBuilderError::BlockGasLimitExceededBySequencerTransactions { @@ -320,7 +332,8 @@ impl MorphPayloadBuilderCtx { )); } - // Execute the transaction + // Execute the transaction and record EVM execution time. + let apply_started = Instant::now(); let gas_used = match builder.execute_transaction(recovered_tx.clone()) { Ok(gas_used) => gas_used, Err(BlockExecutionError::Validation(BlockValidationError::InvalidTx { @@ -356,9 +369,19 @@ impl MorphPayloadBuilderCtx { } Err(err) => { // Fatal error - this is a bug or misconfiguration + tracing::error!( + target: "payload_builder", + tx_index = tx_idx, + %err, + ?recovered_tx, + "fatal EVM execution error on L1 message transaction" + ); return Err(PayloadBuilderError::EvmExecutionError(Box::new(err))); } }; + self.metrics + .commit_tx_apply_duration_seconds + .record(apply_started.elapsed()); // For L1 messages, track the next L1 message index. // L1 gas is prepaid on L1, so no fees are collected here. @@ -444,33 +467,60 @@ impl MorphPayloadBuilderCtx { let tx = tx.into_consensus(); - // Skip blob transactions and L1 messages from pool + // Skip blob transactions and L1 messages from pool. These should + // never reach the pool under normal operation (pool filters them + // out at admission), so their presence here indicates either a + // bug in the admission path or a node configuration mismatch — + // warn loudly. if tx.is_eip4844() || tx.is_l1_msg() { + tracing::warn!( + target: "payload_builder", + signer = %tx.signer(), + nonce = tx.nonce(), + is_blob = tx.is_eip4844(), + is_l1_msg = tx.is_l1_msg(), + "unexpected blob or L1-message transaction in the pool; skipping" + ); best_txs.mark_invalid(tx.signer(), tx.nonce()); continue; } - // Check if the transaction exceeds block limits + // Check if the transaction exceeds block limits (gas or DA size). + // Rare in practice; logged at debug to avoid pool-skip noise. if info.is_tx_over_limits( tx.gas_limit(), tx.length() as u64, block_gas_limit, self.builder_config.max_da_block_size, ) { + tracing::debug!( + target: "payload_builder", + signer = %tx.signer(), + nonce = tx.nonce(), + tx_gas_limit = tx.gas_limit(), + tx_size = tx.length(), + block_gas_limit, + max_da_block_size = self.builder_config.max_da_block_size, + "pool transaction exceeds block limits; skipping" + ); best_txs.mark_invalid(tx.signer(), tx.nonce()); continue; } - // Execute the transaction + let apply_started = Instant::now(); let gas_used = match builder.execute_transaction(tx.clone()) { Ok(gas_used) => gas_used, Err(BlockExecutionError::Validation(BlockValidationError::InvalidTx { error, .. })) => { + // These three variants fire on the fast path of every + // pool sweep and can be extremely noisy under load. Keep + // them at `trace` so default operation stays quiet; turn + // on `RUST_LOG=morph_payload_builder=trace` to diagnose + // pool-skip rates. if error.is_nonce_too_low() { - // If the nonce is too low, we can skip this transaction - // but don't mark as invalid - the sender may have other valid txs + // Nonce too low: sender may have other valid txs. tracing::trace!( target: "payload_builder", %error, @@ -478,8 +528,7 @@ impl MorphPayloadBuilderCtx { "skipping nonce too low transaction" ); } else { - // If the transaction is invalid for other reasons, - // skip it and all of its descendants from this sender + // Other invalid: skip this tx AND its descendants. tracing::trace!( target: "payload_builder", %error, @@ -491,7 +540,7 @@ impl MorphPayloadBuilderCtx { continue; } Err(BlockExecutionError::Validation(err)) => { - // Other validation errors - skip transaction and descendants + // Other validation errors - skip transaction and descendants. tracing::trace!( target: "payload_builder", %err, @@ -502,10 +551,20 @@ impl MorphPayloadBuilderCtx { continue; } Err(err) => { - // Fatal error - should not continue + // Fatal error - should not continue. + tracing::error!( + target: "payload_builder", + signer = %tx.signer(), + nonce = tx.nonce(), + %err, + "fatal EVM execution error on pool transaction; aborting build" + ); return Err(PayloadBuilderError::EvmExecutionError(Box::new(err))); } }; + self.metrics + .commit_tx_apply_duration_seconds + .record(apply_started.elapsed()); // Update execution info info.cumulative_gas_used += gas_used; @@ -583,6 +642,7 @@ where DB: Database, BestTxs: PayloadTransactions> + 'a, { + let build_started = Instant::now(); let attributes = ctx.attributes(); tracing::debug!( @@ -633,6 +693,7 @@ where let breaker = ctx.builder_config.breaker(block_gas_limit); // Execute L1 message transactions (must be first, with sequential queue indices) + let txs_all_started = Instant::now(); let mut executed_txs = ctx.execute_l1_messages(&mut builder, &mut info)?; // Always execute pool transactions (L2 transactions from mempool) @@ -663,6 +724,11 @@ where ); } + // Record total transaction execution time. + ctx.metrics + .commit_txs_all_duration_seconds + .record(txs_all_started.elapsed()); + // Check if this payload is better than the previous one if !ctx.is_better_payload(info.total_fees) { return Ok(BuildOutcomeKind::Aborted { @@ -748,6 +814,14 @@ where Some(executed), ); + // Only record block_transactions for successfully built payloads (not Aborted or Cancelled). + ctx.metrics + .block_transactions + .set(info.transaction_count as f64); + ctx.metrics + .payload_build_duration_seconds + .record(build_started.elapsed()); + Ok(BuildOutcomeKind::Better { payload }) } @@ -957,6 +1031,7 @@ mod tests { cancel: Default::default(), best_payload, builder_config: MorphBuilderConfig::default(), + metrics: MorphPayloadBuilderMetrics::default(), } } diff --git a/crates/payload/builder/src/lib.rs b/crates/payload/builder/src/lib.rs index e240068..0f9ebf1 100644 --- a/crates/payload/builder/src/lib.rs +++ b/crates/payload/builder/src/lib.rs @@ -29,6 +29,7 @@ mod builder; mod config; mod error; +mod metrics; pub use builder::{MorphPayloadBuilder, MorphPayloadTransactions}; pub use config::{MorphBuilderConfig, PayloadBuildingBreaker}; diff --git a/crates/payload/builder/src/metrics.rs b/crates/payload/builder/src/metrics.rs new file mode 100644 index 0000000..b647356 --- /dev/null +++ b/crates/payload/builder/src/metrics.rs @@ -0,0 +1,68 @@ +//! Metrics for the Morph payload builder. +//! +//! Tracks per-transaction and per-block timing analogous to geth's +//! `miner/commit/*` and `processor/block/transactions`. +//! +//! L1 message / pool transaction skip events are intentionally NOT exposed +//! as metrics. They are rare enough (L1) or high-frequency enough (pool) +//! that logging is a more useful observability tool; see the log sites in +//! `builder.rs`. + +use reth_metrics::{ + Metrics, + metrics::{Gauge, Histogram}, +}; + +/// Metrics for the Morph payload builder. +/// +/// Scope: `morph.payload_builder` +/// +/// | Metric | Type | geth equivalent | +/// |--------|------|-----------------| +/// | `commit_txs_all_duration_seconds` | Histogram | `miner/commit/txs_all` | +/// | `commit_tx_apply_duration_seconds` | Histogram | `miner/commit/tx_apply` | +/// | `payload_build_duration_seconds` | Histogram | tempo-inspired | +/// | `block_transactions` | Gauge | `processor/block/transactions` | +#[derive(Metrics, Clone)] +#[metrics(scope = "morph.payload_builder")] +pub(crate) struct MorphPayloadBuilderMetrics { + // ------------------------------------------------------------------------- + // Block-level timing + // ------------------------------------------------------------------------- + /// Total time to execute all transactions (L1 messages + pool) for one block, + /// in seconds. + /// + /// Measured from before L1 message execution to after pool transaction execution. + /// Analogous to geth's `miner/commit/txs_all`. + pub(crate) commit_txs_all_duration_seconds: Histogram, + + // ------------------------------------------------------------------------- + // Per-transaction timing + // ------------------------------------------------------------------------- + /// Time for the EVM execution of a single transaction (`execute_transaction` + /// call only, excluding pre/post checks), in seconds. + /// + /// Analogous to geth's `miner/commit/tx_apply`. + pub(crate) commit_tx_apply_duration_seconds: Histogram, + + // ------------------------------------------------------------------------- + // End-to-end payload build timing + // ------------------------------------------------------------------------- + /// Total time for the entire payload build pipeline (state setup + tx + /// execution + finalize/state-root + seal), in seconds. + /// + /// Broader than `commit_txs_all_duration_seconds` which only covers the + /// transaction execution phase. + /// Inspired by tempo's `payload_build_duration_seconds`. + pub(crate) payload_build_duration_seconds: Histogram, + + // ------------------------------------------------------------------------- + // Block summary gauges + // ------------------------------------------------------------------------- + /// Number of transactions included in the most recently successfully built block. + /// + /// Only updated when a payload reaches `BuildOutcomeKind::Better`; Aborted and + /// Cancelled builds do not update this gauge. + /// Analogous to geth's `processor/block/transactions`. + pub(crate) block_transactions: Gauge, +} diff --git a/etc/.gitignore b/etc/.gitignore new file mode 100644 index 0000000..9aceb4e --- /dev/null +++ b/etc/.gitignore @@ -0,0 +1 @@ +jwttoken diff --git a/etc/README.md b/etc/README.md new file mode 100644 index 0000000..b2669a0 --- /dev/null +++ b/etc/README.md @@ -0,0 +1,91 @@ +## Miscellaneous + +This directory contains miscellaneous files, such as example Grafana dashboards and Prometheus configuration. + +The files in this directory may undergo a lot of changes while reth is unstable, so do not expect them to necessarily be +up to date. + +### Overview + +- [**Prometheus**](./prometheus/prometheus.yml): An example Prometheus configuration. +- [**Grafana**](./grafana/): Example Grafana dashboards & data sources. + +### Docker Compose + +The [`docker-compose.yml`](./docker-compose.yml) in this directory brings up a +local morph-reth node together with Prometheus and Grafana. The `morph-reth` +service runs with `--chain morph` and exposes metrics on port `9001`, +which Prometheus scrapes with the labels defined in +[`prometheus/prometheus.yml`](./prometheus/prometheus.yml). + +```bash +./etc/generate-jwt.sh +docker compose -f etc/docker-compose.yml up -d +``` + +Grafana is available at (default admin/admin) and +will auto-provision the unified Morph dashboard from +[`grafana/dashboards/`](./grafana/dashboards/). + +### Grafana + +#### Adding a new metric to Grafana + +To set up a new metric in Reth and its Grafana dashboard (this assumes running Reth and Grafana instances): + +1. Add the metric to the codebase using `#[derive(Metrics)]` from `reth-metrics`. + See `crates/engine-api/src/metrics.rs` or `crates/payload/builder/src/metrics.rs` + for examples. + +1. Access Grafana: + + - Open `http://localhost:3000/` in a browser + - Log in with username and password `admin` + - Navigate to the `Dashboards` tab + +1. Create or modify a dashboard: + + - Select an existing dashboard or create a new one + - Click `Add` > `Visualization` to create a new panel + +1. Configure your metric panel: + + - Set a panel title and description + - Select metric(s) from the `Metrics browser` or use the `PromQL` terminal + - Document your metric(s) by setting units, legends, etc. + - When adding multiple metrics, use field overwrites if needed + +1. Save and arrange: + + - Click `Apply` to save the panel + - Drag the panel to the desired position on the dashboard + +1. Export the dashboard: + + - Click `Share` > `Export` + - Toggle `Export for sharing externally` + - Click `Save to file` + +1. Update dashboard file: + - Replace the content of the corresponding file in the [dashboards folder](./grafana/dashboards) with the exported + JSON + +Your new metric is now integrated into the Reth Grafana dashboard. + +#### Import Grafana dashboards + +If you are running Reth and Grafana outside of Docker, and wish to import new Grafana dashboards or update a dashboard: + +1. Go to `Home` > `Dashboards` + +1. Click `New` > `Import` + +1. Drag the JSON dashboard file to import it + +1. If updating an existing dashboard, you will need to change the name and UID of the imported dashboard in order to + avoid conflict + +1. Delete the old dashboard + +If you are running Reth and Grafana using Docker, after having pulled the updated dashboards from `main`, restart the +Grafana service. This will update all dashboards. diff --git a/etc/docker-compose.yml b/etc/docker-compose.yml new file mode 100644 index 0000000..b11f236 --- /dev/null +++ b/etc/docker-compose.yml @@ -0,0 +1,73 @@ +name: morph-reth + +services: + morph-reth: + restart: unless-stopped + build: + context: .. + dockerfile: Dockerfile + ports: + - "9001:9001" # metrics + - "30303:30303" # p2p + - "8545:8545" # rpc + - "8551:8551" # engine (authrpc) + volumes: + # Dockerfile: USER morph-reth, WORKDIR /var/lib/morph-reth + - morph_data:/var/lib/morph-reth/data + - logs:/var/lib/morph-reth/logs + - ./jwttoken:/var/lib/morph-reth/jwt:ro + pid: host + command: > + node + --chain morph + --datadir /var/lib/morph-reth/data + --metrics 0.0.0.0:9001 + --log.file.directory /var/lib/morph-reth/logs + --authrpc.addr 0.0.0.0 + --authrpc.port 8551 + --authrpc.jwtsecret /var/lib/morph-reth/jwt/jwt.hex + --http --http.addr 0.0.0.0 --http.port 8545 + --http.api "eth,net,web3" + + prometheus: + restart: unless-stopped + image: prom/prometheus + depends_on: + - morph-reth + ports: + - 9090:9090 + volumes: + - ./prometheus/:/etc/prometheus/ + - prometheus_data:/prometheus + command: + - --config.file=/etc/prometheus/prometheus.yml + + grafana: + restart: unless-stopped + image: grafana/grafana:latest + depends_on: + - morph-reth + - prometheus + ports: + - 3000:3000 + environment: + PROMETHEUS_URL: ${PROMETHEUS_URL:-http://prometheus:9090} + volumes: + - grafana_data:/var/lib/grafana + - ./grafana/datasources:/etc/grafana/provisioning/datasources + - ./grafana/dashboards:/etc/grafana/provisioning_temp/dashboards + entrypoint: > + sh -c "cp -r /etc/grafana/provisioning_temp/dashboards/. /etc/grafana/provisioning/dashboards && + find /etc/grafana/provisioning/dashboards/ -name '*.json' -exec sed -i 's/$${DS_PROMETHEUS}/Prometheus/g' {} \+ && + find /etc/grafana/provisioning/dashboards/ -name '*.json' -exec sed -i 's/$${datasource}/Prometheus/g' {} \+ && + /run.sh" + +volumes: + morph_data: + driver: local + logs: + driver: local + prometheus_data: + driver: local + grafana_data: + driver: local diff --git a/etc/generate-jwt.sh b/etc/generate-jwt.sh new file mode 100755 index 0000000..3ec5dad --- /dev/null +++ b/etc/generate-jwt.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# Borrowed from EthStaker's prepare for the merge guide +# See https://github.com/eth-educators/ethstaker-guides/blob/main/docs/prepare-for-the-merge.md#configuring-a-jwt-token-file + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +mkdir -p "${SCRIPT_DIR}/jwttoken" +if [[ ! -f "${SCRIPT_DIR}/jwttoken/jwt.hex" ]] +then + openssl rand -hex 32 | tr -d "\n" > "${SCRIPT_DIR}/jwttoken/jwt.hex" +else + echo "${SCRIPT_DIR}/jwttoken/jwt.hex already exists!" +fi diff --git a/etc/grafana/dashboards/dashboard.yml b/etc/grafana/dashboards/dashboard.yml new file mode 100644 index 0000000..87b13ec --- /dev/null +++ b/etc/grafana/dashboards/dashboard.yml @@ -0,0 +1,7 @@ +apiVersion: 1 + +providers: + - name: 'Folder' + allowUiUpdates: true + options: + path: /etc/grafana/provisioning/dashboards diff --git a/etc/grafana/dashboards/morph-reth.json b/etc/grafana/dashboards/morph-reth.json new file mode 100644 index 0000000..949d908 --- /dev/null +++ b/etc/grafana/dashboards/morph-reth.json @@ -0,0 +1,13223 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Morph-Reth unified: Engine API, Payload Builder, DB, P2P, TxPool, Sync, System", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "valueSize": 14 + }, + "textMode": "name", + "wideLayout": false + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": false, + "expr": "max by (version) (reth_info{chain=\"$chain\", role=~\"$role\", service=~\"$service\"})", + "instant": true, + "legendFormat": "Version: {{version}}", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": false, + "expr": "max by (build_timestamp) (reth_info{chain=\"$chain\", role=~\"$role\", service=~\"$service\"})", + "instant": true, + "legendFormat": "Timestamp: {{build_timestamp}}", + "range": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": false, + "expr": "max by (git_sha) (reth_info{chain=\"$chain\", role=~\"$role\", service=~\"$service\"})", + "instant": true, + "legendFormat": "Git SHA: {{git_sha}}", + "range": false, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": false, + "expr": "max by (build_profile) (reth_info{chain=\"$chain\", role=~\"$role\", service=~\"$service\"})", + "instant": true, + "legendFormat": "Profile: {{build_profile}}", + "range": false, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": false, + "expr": "max by (target_triple) (reth_info{chain=\"$chain\", role=~\"$role\", service=~\"$service\"})", + "instant": true, + "legendFormat": "Target: {{target_triple}}", + "range": false, + "refId": "E" + } + ], + "title": "Build Info", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "max": 10, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "green", + "value": 3 + } + ] + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 3, + "options": { + "displayMode": "gradient", + "minVizHeight": 14, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "valueMode": "color", + "namePlacement": "left" + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "reth_network_connected_peers{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "instant": true, + "legendFormat": "{{service}}", + "refId": "A" + } + ], + "title": "Connected peers", + "transparent": true, + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "x": 12, + "y": 6, + "w": 3, + "h": 6 + }, + "id": 128, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": false + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(reth_db_table_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\"})", + "legendFormat": "Database", + "range": true, + "refId": "A" + } + ], + "title": "Database", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "x": 15, + "y": 6, + "w": 3, + "h": 6 + }, + "id": 129, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": false + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(reth_db_freelist{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} * reth_db_page_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\"})", + "hide": false, + "instant": false, + "legendFormat": "Freelist", + "range": true, + "refId": "B" + } + ], + "title": "Freelist", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "x": 18, + "y": 6, + "w": 3, + "h": 6 + }, + "id": 130, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": false + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(reth_static_files_segment_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\"})", + "hide": false, + "instant": false, + "legendFormat": "Static Files", + "range": true, + "refId": "C" + } + ], + "title": "Static Files", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "x": 21, + "y": 6, + "w": 3, + "h": 6 + }, + "id": 131, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": false + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(reth_db_table_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}) + sum(reth_db_freelist{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} * reth_db_page_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}) + sum(reth_static_files_segment_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\"})", + "hide": false, + "instant": false, + "legendFormat": "Total", + "range": true, + "refId": "D" + } + ], + "title": "Total", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The throughput of the node's executor. The metric is the amount of gas processed in a block, divided by the time it took to process the block.\n\nNote: For mainnet, the block range 2,383,397-2,620,384 will be slow because of the 2016 DoS attack.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "si: gas/s", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_sync_execution_gas_per_second{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "legendFormat": "{{service}} · Gas/s", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "avg_over_time(reth_sync_execution_gas_per_second{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[1m])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "{{service}} · Avg Gas/s (1m)", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "avg_over_time(reth_sync_execution_gas_per_second{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[5m])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "{{service}} · Avg Gas/s (5m)", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "avg_over_time(reth_sync_execution_gas_per_second{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[10m])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "{{service}} · Avg Gas/s (10m)", + "range": true, + "refId": "D", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "avg_over_time(reth_sync_execution_gas_per_second{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[30m])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "{{service}} · Avg Gas/s (30m)", + "range": true, + "refId": "E", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "avg_over_time(reth_sync_execution_gas_per_second{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[1h])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "{{service}} · Avg Gas/s (1h)", + "range": true, + "refId": "F", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "avg_over_time(reth_sync_execution_gas_per_second{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[24h])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "{{service}} · Avg Gas/s (24h)", + "range": true, + "refId": "G", + "useBackend": false + } + ], + "title": "Execution throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false, + "axisCenteredZero": false + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 6, + "options": { + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "title": "Block Import Rate (blocks/s)", + "type": "timeseries", + "targets": [ + { + "expr": "rate(reth_morph_engine_new_l2_block_duration_seconds_count{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}[$__rate_interval]) > 0", + "legendFormat": "{{service}} newL2Block" + }, + { + "expr": "rate(reth_morph_engine_new_safe_l2_block_duration_seconds_count{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}[$__rate_interval]) > 0", + "legendFormat": "{{service}} newSafe" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "red", + "value": 30 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 7, + "options": { + "colorMode": "background", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "title": "Head Block Timegap", + "description": "Seconds since last imported block. >30s = node is falling behind.", + "type": "stat", + "targets": [ + { + "expr": "reth_morph_engine_head_block_timegap_seconds{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}", + "legendFormat": "{{service}}" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false, + "axisCenteredZero": false, + "axisSoftMax": 5 + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 8, + "options": { + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "title": "Head Block Timegap By Role", + "type": "timeseries", + "targets": [ + { + "expr": "reth_morph_engine_head_block_timegap_seconds{chain=\"$chain\", role=~\"$role\"}", + "legendFormat": "{{service}}" + } + ] + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 9, + "panels": [], + "title": "Engine API", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false, + "axisCenteredZero": false + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 34 + }, + "id": 10, + "options": { + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "title": "newL2Block Latency", + "type": "timeseries", + "targets": [ + { + "expr": "rate(reth_morph_engine_new_l2_block_duration_seconds_sum{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}[$__rate_interval]) / rate(reth_morph_engine_new_l2_block_duration_seconds_count{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}[$__rate_interval]) > 0", + "legendFormat": "{{service}} avg" + }, + { + "expr": "reth_morph_engine_new_l2_block_duration_seconds{role=~\"$role\", chain=\"$chain\", service=~\"$service\", quantile=\"$quantile\"} > 0", + "legendFormat": "{{service}} p{{quantile}}" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false, + "axisCenteredZero": false + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 34 + }, + "id": 11, + "options": { + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "title": "newSafeL2Block Latency", + "type": "timeseries", + "targets": [ + { + "expr": "rate(reth_morph_engine_new_safe_l2_block_duration_seconds_sum{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}[$__rate_interval]) / rate(reth_morph_engine_new_safe_l2_block_duration_seconds_count{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}[$__rate_interval]) > 0", + "legendFormat": "{{service}} avg" + }, + { + "expr": "reth_morph_engine_new_safe_l2_block_duration_seconds{role=~\"$role\", chain=\"$chain\", service=~\"$service\", quantile=\"$quantile\"} > 0", + "legendFormat": "{{service}} p{{quantile}}" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false, + "axisCenteredZero": false + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 12, + "options": { + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "title": "assembleL2Block Latency", + "type": "timeseries", + "targets": [ + { + "expr": "rate(reth_morph_engine_assemble_l2_block_duration_seconds_sum{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}[$__rate_interval]) / rate(reth_morph_engine_assemble_l2_block_duration_seconds_count{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}[$__rate_interval]) > 0", + "legendFormat": "{{service}} avg" + }, + { + "expr": "reth_morph_engine_assemble_l2_block_duration_seconds{role=~\"$role\", chain=\"$chain\", service=~\"$service\", quantile=\"$quantile\"} > 0", + "legendFormat": "{{service}} p{{quantile}}" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false, + "axisCenteredZero": false + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 13, + "options": { + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "title": "validateL2Block Latency", + "type": "timeseries", + "targets": [ + { + "expr": "rate(reth_morph_engine_validate_l2_block_duration_seconds_sum{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}[$__rate_interval]) / rate(reth_morph_engine_validate_l2_block_duration_seconds_count{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}[$__rate_interval]) > 0", + "legendFormat": "{{service}} avg" + }, + { + "expr": "reth_morph_engine_validate_l2_block_duration_seconds{role=~\"$role\", chain=\"$chain\", service=~\"$service\", quantile=\"$quantile\"} > 0", + "legendFormat": "{{service}} p{{quantile}}" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false, + "axisCenteredZero": false + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 52 + }, + "id": 14, + "options": { + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "title": "Engine API Failures (per second)", + "type": "timeseries", + "targets": [ + { + "expr": "rate(reth_morph_engine_assemble_l2_block_failures_total{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}[$__rate_interval])", + "legendFormat": "{{service}} assemble" + }, + { + "expr": "rate(reth_morph_engine_validate_l2_block_failures_total{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}[$__rate_interval])", + "legendFormat": "{{service}} validate" + }, + { + "expr": "rate(reth_morph_engine_new_l2_block_failures_total{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}[$__rate_interval])", + "legendFormat": "{{service}} newL2Block" + }, + { + "expr": "rate(reth_morph_engine_new_safe_l2_block_failures_total{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}[$__rate_interval])", + "legendFormat": "{{service}} newSafe" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false, + "axisCenteredZero": false + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 52 + }, + "id": 15, + "options": { + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "title": "Engine API Failures By Role", + "type": "timeseries", + "targets": [ + { + "expr": "sum by (service) (rate(reth_morph_engine_assemble_l2_block_failures_total{chain=\"$chain\", role=~\"$role\"}[$__rate_interval]) + rate(reth_morph_engine_validate_l2_block_failures_total{chain=\"$chain\", role=~\"$role\"}[$__rate_interval]) + rate(reth_morph_engine_new_l2_block_failures_total{chain=\"$chain\", role=~\"$role\"}[$__rate_interval]) + rate(reth_morph_engine_new_safe_l2_block_failures_total{chain=\"$chain\", role=~\"$role\"}[$__rate_interval]))", + "legendFormat": "{{service}}" + } + ] + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 61 + }, + "id": 16, + "panels": [], + "title": "Payload Builder", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false, + "axisCenteredZero": false + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 62 + }, + "id": 17, + "options": { + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "title": "Payload Build Duration", + "description": "End-to-end payload build duration (state setup + tx execution + finalize + seal).", + "type": "timeseries", + "targets": [ + { + "expr": "rate(reth_morph_payload_builder_payload_build_duration_seconds_sum{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}[$__rate_interval]) / rate(reth_morph_payload_builder_payload_build_duration_seconds_count{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}[$__rate_interval]) > 0", + "legendFormat": "{{service}} avg" + }, + { + "expr": "reth_morph_payload_builder_payload_build_duration_seconds{role=~\"$role\", chain=\"$chain\", service=~\"$service\", quantile=\"$quantile\"} > 0", + "legendFormat": "{{service}} p{{quantile}}" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false, + "axisCenteredZero": false + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 62 + }, + "id": 18, + "options": { + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "title": "Tx Execution Duration", + "description": "Total time to execute all transactions (L1 messages + pool) for one block.", + "type": "timeseries", + "targets": [ + { + "expr": "rate(reth_morph_payload_builder_commit_txs_all_duration_seconds_sum{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}[$__rate_interval]) / rate(reth_morph_payload_builder_commit_txs_all_duration_seconds_count{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}[$__rate_interval]) > 0", + "legendFormat": "{{service}} avg" + }, + { + "expr": "reth_morph_payload_builder_commit_txs_all_duration_seconds{role=~\"$role\", chain=\"$chain\", service=~\"$service\", quantile=\"$quantile\"} > 0", + "legendFormat": "{{service}} p{{quantile}}" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false, + "axisCenteredZero": false + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 71 + }, + "id": 19, + "options": { + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "title": "Per-Tx Apply Duration", + "description": "Time for the EVM execution of a single transaction.", + "type": "timeseries", + "targets": [ + { + "expr": "rate(reth_morph_payload_builder_commit_tx_apply_duration_seconds_sum{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}[$__rate_interval]) / rate(reth_morph_payload_builder_commit_tx_apply_duration_seconds_count{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}[$__rate_interval]) > 0", + "legendFormat": "{{service}} avg" + }, + { + "expr": "reth_morph_payload_builder_commit_tx_apply_duration_seconds{role=~\"$role\", chain=\"$chain\", service=~\"$service\", quantile=\"$quantile\"} > 0", + "legendFormat": "{{service}} p{{quantile}}" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false, + "axisCenteredZero": false + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 71 + }, + "id": 20, + "options": { + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "title": "Block Transactions", + "type": "timeseries", + "targets": [ + { + "expr": "reth_morph_payload_builder_block_transactions{role=~\"$role\", chain=\"$chain\", service=~\"$service\"} > 0", + "legendFormat": "{{service}}" + } + ] + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 80 + }, + "id": 23, + "panels": [], + "title": "Execution & State Root", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 24, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 81 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_sync_block_validation_state_root_duration{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · State Root Duration", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_sync_execution_execution_duration{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Execution Duration", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Block Processing Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 81 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "reth_sync_caching_account_cache_hits{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} / (reth_sync_caching_account_cache_hits{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} + reth_sync_caching_account_cache_misses{chain=\"$chain\", role=~\"$role\", service=~\"$service\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Account cache hits", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "reth_sync_caching_storage_cache_hits{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} / (reth_sync_caching_storage_cache_hits{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} + reth_sync_caching_storage_cache_misses{chain=\"$chain\", role=~\"$role\", service=~\"$service\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Storage cache hits", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "reth_sync_caching_code_cache_hits{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} / (reth_sync_caching_code_cache_hits{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} + reth_sync_caching_code_cache_misses{chain=\"$chain\", role=~\"$role\", service=~\"$service\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Code cache hits", + "range": true, + "refId": "C", + "useBackend": false + } + ], + "title": "Execution cache hitrate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The time it takes for operations that are part of block validation, but not execution or state root, to complete.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 89 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "reth_sync_block_validation_trie_input_duration{chain=\"$chain\", role=~\"$role\", service=~\"$service\", quantile=~\"(0|0.5|0.9|0.95|1)\"} > 0", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Trie input creation duration p{{quantile}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Block validation overhead", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit", + "min": 0 + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Precompile cache hits" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": true, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 89 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "reth_sync_caching_precompile_cache_hits{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} / (reth_sync_caching_precompile_cache_hits{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} + reth_sync_caching_precompile_cache_misses{chain=\"$chain\", role=~\"$role\", service=~\"$service\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Precompile cache hits", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Precompile cache hitrate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 97 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_tree_root_proofs_processed_histogram{chain=\"$chain\", role=~\"$role\", service=~\"$service\",quantile=~\"(0|0.5|0.9|0.95|1)\"} > 0", + "instant": false, + "legendFormat": "{{service}} · {{quantile}} percentile", + "range": true, + "refId": "Branch Nodes" + } + ], + "title": "Proofs Processed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 97 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_tree_root_proof_calculation_duration_histogram{chain=\"$chain\", role=~\"$role\", service=~\"$service\",quantile=~\"(0|0.5|0.9|0.95|1)\"} > 0", + "instant": false, + "legendFormat": "{{service}} · {{quantile}} percentile", + "range": true, + "refId": "Branch Nodes" + } + ], + "title": "Proof calculation duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 105 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_tree_root_pending_account_multiproofs_histogram{chain=\"$chain\", role=~\"$role\", service=~\"$service\", quantile=\"0.5\"} > 0", + "instant": false, + "legendFormat": "{{service}} · accounts p50", + "range": true, + "refId": "Branch Nodes" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_tree_root_pending_account_multiproofs_histogram{chain=\"$chain\", role=~\"$role\", service=~\"$service\", quantile=\"0.9\"} > 0", + "hide": false, + "instant": false, + "legendFormat": "{{service}} · accounts p90", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_tree_root_pending_account_multiproofs_histogram{chain=\"$chain\", role=~\"$role\", service=~\"$service\", quantile=\"0.95\"} > 0", + "hide": false, + "instant": false, + "legendFormat": "{{service}} · accounts p95", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_tree_root_pending_account_multiproofs_histogram{chain=\"$chain\", role=~\"$role\", service=~\"$service\", quantile=\"0.99\"} > 0", + "hide": false, + "instant": false, + "legendFormat": "{{service}} · accounts p99", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_tree_root_pending_storage_multiproofs_histogram{chain=\"$chain\", role=~\"$role\", service=~\"$service\", quantile=\"0.5\"} > 0", + "hide": false, + "instant": false, + "legendFormat": "{{service}} · storage p50", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_tree_root_pending_storage_multiproofs_histogram{chain=\"$chain\", role=~\"$role\", service=~\"$service\", quantile=\"0.9\"} > 0", + "hide": false, + "instant": false, + "legendFormat": "{{service}} · storage p90", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_tree_root_pending_storage_multiproofs_histogram{chain=\"$chain\", role=~\"$role\", service=~\"$service\", quantile=\"0.95\"} > 0", + "hide": false, + "instant": false, + "legendFormat": "{{service}} · storage p95", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_tree_root_pending_storage_multiproofs_histogram{chain=\"$chain\", role=~\"$role\", service=~\"$service\", quantile=\"0.99\"} > 0", + "hide": false, + "instant": false, + "legendFormat": "{{service}} · storage p99", + "range": true, + "refId": "G" + } + ], + "title": "Pending MultiProof requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 105 + }, + "id": 31, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_tree_root_active_account_workers_histogram{chain=\"$chain\", role=~\"$role\", service=~\"$service\",quantile=\"0.5\"} > 0", + "instant": false, + "legendFormat": "{{service}} · accounts p50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_tree_root_active_account_workers_histogram{chain=\"$chain\", role=~\"$role\", service=~\"$service\",quantile=\"0.9\"} > 0", + "hide": false, + "instant": false, + "legendFormat": "{{service}} · accounts p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_tree_root_active_account_workers_histogram{chain=\"$chain\", role=~\"$role\", service=~\"$service\",quantile=\"0.95\"} > 0", + "hide": false, + "instant": false, + "legendFormat": "{{service}} · accounts p95", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_tree_root_active_account_workers_histogram{chain=\"$chain\", role=~\"$role\", service=~\"$service\",quantile=\"0.99\"} > 0", + "hide": false, + "instant": false, + "legendFormat": "{{service}} · accounts p99", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_tree_root_active_storage_workers_histogram{chain=\"$chain\", role=~\"$role\", service=~\"$service\",quantile=\"0.5\"} > 0", + "instant": false, + "legendFormat": "{{service}} · storages p50", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_tree_root_active_storage_workers_histogram{chain=\"$chain\", role=~\"$role\", service=~\"$service\",quantile=\"0.9\"} > 0", + "hide": false, + "instant": false, + "legendFormat": "{{service}} · storages p90", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_tree_root_active_storage_workers_histogram{chain=\"$chain\", role=~\"$role\", service=~\"$service\",quantile=\"0.95\"} > 0", + "hide": false, + "instant": false, + "legendFormat": "{{service}} · storages p95", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_tree_root_active_storage_workers_histogram{chain=\"$chain\", role=~\"$role\", service=~\"$service\",quantile=\"0.99\"} > 0", + "hide": false, + "instant": false, + "legendFormat": "{{service}} · storages p99", + "range": true, + "refId": "H" + } + ], + "title": "Active multiproof workers", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 113 + }, + "id": 32, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_sparse_state_trie_multiproof_total_account_nodes{chain=\"$chain\", role=~\"$role\", service=~\"$service\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", + "instant": false, + "legendFormat": "{{service}} · Account {{quantile}} percentile", + "range": true, + "refId": "Branch Nodes" + } + ], + "title": "Total multiproof account nodes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 113 + }, + "id": 33, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_sparse_state_trie_multiproof_total_storage_nodes{chain=\"$chain\", role=~\"$role\", service=~\"$service\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", + "instant": false, + "legendFormat": "{{service}} · Storage {{quantile}} percentile", + "range": true, + "refId": "Branch Nodes" + } + ], + "title": "Total multiproof storage nodes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 121 + }, + "id": 34, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_sparse_state_trie_multiproof_skipped_account_nodes{chain=\"$chain\", role=~\"$role\", service=~\"$service\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", + "hide": false, + "instant": false, + "legendFormat": "{{service}} · Account {{quantile}} percentile", + "range": true, + "refId": "A" + } + ], + "title": "Redundant multiproof account nodes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 121 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_sparse_state_trie_multiproof_skipped_storage_nodes{chain=\"$chain\", role=~\"$role\", service=~\"$service\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", + "instant": false, + "legendFormat": "{{service}} · Storage {{quantile}} percentile", + "range": true, + "refId": "Branch Nodes" + } + ], + "title": "Redundant multiproof storage nodes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "How much time is spent in the multiproof task", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 129 + }, + "id": 36, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_tree_root_multiproof_task_total_duration_histogram{chain=\"$chain\", role=~\"$role\", service=~\"$service\",quantile=~\"(0|0.5|0.9|0.95|1)\"} > 0", + "hide": false, + "instant": false, + "legendFormat": "{{service}} · Task duration {{quantile}} percentile", + "range": true, + "refId": "A" + } + ], + "title": "Proof fetching total duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Histogram for state root latency, the time spent blocked waiting for the state root.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 129 + }, + "id": 37, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_sync_block_validation_state_root_histogram{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · State Root Duration p{{quantile}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "State root latency", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 137 + }, + "id": 38, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The average commit time for database transactions. Generally, this should not be a limiting factor in syncing.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name", + "seriesBy": "last" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 138 + }, + "id": 39, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg by (service) (rate(reth_database_transaction_close_duration_seconds_sum{chain=\"$chain\", role=~\"$role\", service=~\"$service\", outcome=\"commit\"}[$__rate_interval]) / clamp_min(rate(reth_database_transaction_close_duration_seconds_count{chain=\"$chain\", role=~\"$role\", service=~\"$service\", outcome=\"commit\"}[$__rate_interval]), 1))", + "format": "time_series", + "instant": false, + "legendFormat": "{{service}} · Commit time", + "range": true, + "refId": "A" + } + ], + "title": "DB Average Commit Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 138 + }, + "id": 40, + "maxDataPoints": 25, + "options": { + "calculate": false, + "cellGap": 1, + "cellValues": { + "unit": "s" + }, + "color": { + "exponent": 0.2, + "fill": "dark-orange", + "min": 0, + "mode": "opacity", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto", + "value": "Commit time" + }, + "tooltip": { + "mode": "multi", + "showColorScale": false, + "yHistogram": false, + "sort": "desc" + }, + "yAxis": { + "axisLabel": "Quantile", + "axisPlacement": "left", + "reverse": false, + "unit": "percentunit" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg by (service, quantile) (max_over_time(reth_database_transaction_close_duration_seconds{chain=\"$chain\", role=~\"$role\", service=~\"$service\", outcome=\"commit\"}[$__rate_interval]))", + "format": "time_series", + "instant": false, + "legendFormat": "{{service}} · {{quantile}}", + "range": true, + "refId": "A" + } + ], + "title": "DB Commit Time Heatmap", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The average time a database transaction was open.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name", + "seriesBy": "last" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 146 + }, + "id": 41, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (service, outcome, mode) (rate(reth_database_transaction_open_duration_seconds_sum{chain=\"$chain\", role=~\"$role\", service=~\"$service\", outcome!=\"\"}[$__rate_interval]) / rate(reth_database_transaction_open_duration_seconds_count{chain=\"$chain\", role=~\"$role\", service=~\"$service\", outcome!=\"\"}[$__rate_interval])) > 0", + "format": "time_series", + "instant": false, + "legendFormat": "{{service}} · {{mode}}, {{outcome}}", + "range": true, + "refId": "A" + } + ], + "title": "DB Average Transaction Open Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The maximum time the database transaction was open.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 146 + }, + "id": 42, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max by (service, outcome, mode) (max_over_time(reth_database_transaction_open_duration_seconds{chain=\"$chain\", role=~\"$role\", service=~\"$service\", outcome!=\"\", quantile=\"1.0\"}[$__interval])) > 0", + "format": "time_series", + "instant": false, + "legendFormat": "{{service}} · {{mode}}, {{outcome}}", + "range": true, + "refId": "A" + } + ], + "title": "DB Max Transaction Open Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "txs", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Diff(opened-closed)" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 0, + 10 + ], + "fill": "dot" + } + }, + { + "id": "custom.axisLabel", + "value": "diff" + }, + { + "id": "displayName", + "value": "${__field.labels.service} · Diff (opened − closed)" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 154 + }, + "id": 43, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (service) (reth_database_transaction_opened_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\", mode=\"read-write\"})", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Opened", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (service) (reth_database_transaction_closed_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\", mode=\"read-write\"})", + "format": "time_series", + "instant": false, + "legendFormat": "{{service}} · Closed", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "__expr__", + "uid": "${DS_EXPRESSION}" + }, + "expression": "${A} - ${B}", + "hide": false, + "refId": "Diff(opened-closed)", + "type": "math" + } + ], + "title": "DB Read-Write Transactions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "txs", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Diff(opened, closed)" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 0, + 10 + ], + "fill": "dot" + } + }, + { + "id": "custom.axisLabel", + "value": "diff" + }, + { + "id": "displayName", + "value": "${__field.labels.service} · Diff (opened − closed)" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 154 + }, + "id": 44, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by (service) (reth_database_transaction_opened_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\", mode=\"read-only\"})", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Opened", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by (service) (reth_database_transaction_closed_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\", mode=\"read-only\"})", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Closed", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "__expr__", + "uid": "${DS_EXPRESSION}" + }, + "expression": "${A} - ${B}", + "hide": false, + "refId": "Diff(opened, closed)", + "type": "math" + } + ], + "title": "DB Read-Only Transactions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The size of tables in the database", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 162 + }, + "id": 45, + "options": { + "displayLabels": [ + "name" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "sort": "desc", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_db_table_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", + "interval": "", + "legendFormat": "{{service}} · {{table}}", + "range": true, + "refId": "A" + } + ], + "title": "DB Tables", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The maximum time the database transaction operation which inserts a large value took.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 162 + }, + "id": 46, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(max_over_time(reth_database_operation_large_value_duration_seconds{chain=\"$chain\", role=~\"$role\", service=~\"$service\", quantile=\"1.0\"}[$__interval])) by (service, table) > 0", + "format": "time_series", + "instant": false, + "legendFormat": "{{service}} · {{table}}", + "range": true, + "refId": "A" + } + ], + "title": "DB Max Insertion Operation Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The type of the pages in the database:\n\n- **Leaf** pages contain KV pairs.\n- **Branch** pages contain information about keys in the leaf pages\n- **Overflow** pages store large values and should generally be avoided if possible", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 170 + }, + "id": 47, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "sort": "desc", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "sum by (service, type) ( reth_db_table_pages{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} )", + "legendFormat": "{{service}} · {{type}}", + "range": true, + "refId": "A" + } + ], + "title": "DB Pages", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The size of the database over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 4, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 170 + }, + "id": 48, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (service) ( reth_db_table_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} )", + "legendFormat": "{{service}}", + "range": true, + "refId": "A" + } + ], + "title": "DB Growth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The number of pages on the MDBX freelist", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 178 + }, + "id": 49, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(reth_db_freelist{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}) by (service, job)", + "legendFormat": "{{service}} · Pages ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "DB Freelist", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hideFrom.viz", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "__name__" + }, + "properties": [ + { + "id": "custom.hideFrom.viz", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "custom.hideFrom.viz", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "custom.hideFrom.viz", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "type" + }, + "properties": [ + { + "id": "custom.hideFrom.viz", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "unit", + "value": "locale" + }, + { + "id": "displayName", + "value": "Overflow pages" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "table" + }, + "properties": [ + { + "id": "displayName", + "value": "Table" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 178 + }, + "id": 50, + "options": { + "cellHeight": "sm", + "showHeader": true + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sort_desc(reth_db_table_pages{chain=\"$chain\", role=~\"$role\", service=~\"$service\", type=\"overflow\"} != 0)", + "format": "table", + "instant": true, + "legendFormat": "{{service}}", + "range": false, + "refId": "A" + } + ], + "title": "DB Overflow Pages By Table", + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The size of segments in the static files", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 186 + }, + "id": 51, + "options": { + "displayLabels": [ + "name" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "sort": "desc", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_static_files_segment_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", + "interval": "", + "legendFormat": "{{service}} · {{segment}}", + "range": true, + "refId": "A" + } + ], + "title": "Static Files Segment Size", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "unit", + "value": "locale" + }, + { + "id": "displayName", + "value": "Entries" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "segment" + }, + "properties": [ + { + "id": "displayName", + "value": "Segment" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hideFrom.viz", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "custom.hideFrom.viz", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "custom.hideFrom.viz", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "__name__" + }, + "properties": [ + { + "id": "custom.hideFrom.viz", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 186 + }, + "id": 52, + "options": { + "cellHeight": "sm", + "showHeader": true + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "reth_static_files_segment_entries{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "format": "table", + "instant": true, + "legendFormat": "{{service}}", + "range": false, + "refId": "A" + } + ], + "title": "Static Files Entries Per Segment", + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "unit", + "value": "locale" + }, + { + "id": "displayName", + "value": "Files" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "segment" + }, + "properties": [ + { + "id": "displayName", + "value": "Segment" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hideFrom.viz", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "custom.hideFrom.viz", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "custom.hideFrom.viz", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "__name__" + }, + "properties": [ + { + "id": "custom.hideFrom.viz", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 186 + }, + "id": 53, + "options": { + "cellHeight": "sm", + "showHeader": true + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "reth_static_files_segment_files{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "format": "table", + "instant": true, + "legendFormat": "{{service}}", + "range": false, + "refId": "A" + } + ], + "title": "Static Files File Count Per Segment", + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The size of the static files over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 194 + }, + "id": 54, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (service) ( reth_static_files_segment_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} )", + "legendFormat": "{{service}}", + "range": true, + "refId": "A" + } + ], + "title": "Static Files Growth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The maximum time the static files operation which commits a writer took.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 194 + }, + "id": 55, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max(max_over_time(reth_static_files_jar_provider_write_duration_seconds{chain=\"$chain\", role=~\"$role\", service=~\"$service\", operation=\"commit-writer\", quantile=\"1.0\"}[$__interval])) by (service, segment) > 0", + "legendFormat": "{{service}} · {{segment}}", + "range": true, + "refId": "A" + } + ], + "title": "Static Files Max Writer Commit Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The block number of the tip of the canonical chain from the blockchain tree.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 202 + }, + "id": 56, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_blockchain_tree_canonical_chain_height{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "hide": false, + "legendFormat": "{{service}} · Canonical chain height", + "range": true, + "refId": "B" + } + ], + "title": "Chain Tree Canonical Height", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total number of blocks in the tree's block buffer", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 202 + }, + "id": 57, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_blockchain_tree_block_buffer_blocks{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "hide": false, + "legendFormat": "{{service}} · Buffered blocks", + "range": true, + "refId": "B" + } + ], + "title": "Chain Tree Block Buffer", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 210 + }, + "id": 58, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "increase(reth_blockchain_tree_reorgs{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "instant": false, + "legendFormat": "{{service}}", + "range": true, + "refId": "A" + } + ], + "title": "Chain Tree Reorgs", + "type": "timeseries", + "description": "Always 0 on morph: tendermint consensus commits one canonical block per height, so reth never reorgs. This panel is kept for visibility in case of unexpected behavior." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 210 + }, + "id": 59, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_blockchain_tree_latest_reorg_depth{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "instant": false, + "legendFormat": "{{service}}", + "range": true, + "refId": "A" + } + ], + "title": "Chain Tree Latest Reorg Depth", + "type": "timeseries", + "description": "Always 0 on morph: tendermint consensus commits one canonical block per height, so reth never reorgs. This panel is kept for visibility in case of unexpected behavior." + } + ], + "title": "Database & Storage", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 138 + }, + "id": 60, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "http" + }, + "properties": [ + { + "id": "displayName", + "value": "HTTP" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ws" + }, + "properties": [ + { + "id": "displayName", + "value": "WebSocket" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 219 + }, + "id": 61, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum(reth_rpc_server_connections_connections_opened_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} - reth_rpc_server_connections_connections_closed_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}) by (service, transport) > 0", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{service}} · {{transport}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Active Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 219 + }, + "id": 62, + "maxDataPoints": 25, + "options": { + "calculate": false, + "cellGap": 1, + "cellValues": { + "unit": "s" + }, + "color": { + "exponent": 0.2, + "fill": "dark-orange", + "min": 0, + "mode": "opacity", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto", + "value": "Latency time" + }, + "tooltip": { + "mode": "multi", + "showColorScale": false, + "yHistogram": false, + "sort": "desc" + }, + "yAxis": { + "axisLabel": "Quantile", + "axisPlacement": "left", + "reverse": false, + "unit": "percentunit" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg by (quantile) (max_over_time(reth_rpc_server_connections_request_time_seconds{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])) > 0", + "format": "time_series", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Request Latency time", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 227 + }, + "id": 63, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max by (service, method) (max_over_time(reth_rpc_server_calls_time_seconds{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])) > 0", + "instant": false, + "legendFormat": "{{service}} · {{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Maximum call latency per method", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 227 + }, + "id": 64, + "maxDataPoints": 25, + "options": { + "calculate": false, + "cellGap": 1, + "cellValues": { + "unit": "s" + }, + "color": { + "exponent": 0.2, + "fill": "dark-orange", + "min": 0, + "mode": "opacity", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto", + "value": "Latency time" + }, + "tooltip": { + "mode": "multi", + "showColorScale": false, + "yHistogram": false, + "sort": "desc" + }, + "yAxis": { + "axisLabel": "Quantile", + "axisPlacement": "left", + "reverse": false, + "unit": "percentunit" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg by (quantile) (max_over_time(reth_rpc_server_calls_time_seconds{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])) > 0", + "format": "time_series", + "instant": false, + "legendFormat": "{{quantile}}", + "range": true, + "refId": "A" + } + ], + "title": "Call Latency time", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*cached items.*/" + }, + "properties": [ + { + "id": "custom.axisLabel", + "value": "Items" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*consumers.*/" + }, + "properties": [ + { + "id": "custom.axisLabel", + "value": "Queued consumers" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.memory usage*/" + }, + "properties": [ + { + "id": "unit", + "value": "decbytes" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 235 + }, + "id": 65, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_rpc_eth_cache_cached_count{chain=\"$chain\", role=~\"$role\", service=~\"$service\", cache=\"headers\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Headers cache cached items", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_rpc_eth_cache_queued_consumers_count{chain=\"$chain\", role=~\"$role\", service=~\"$service\", cache=\"receipts\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Receipts cache queued consumers", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_rpc_eth_cache_queued_consumers_count{chain=\"$chain\", role=~\"$role\", service=~\"$service\", cache=\"headers\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Headers cache queued consumers", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_rpc_eth_cache_queued_consumers_count{chain=\"$chain\", role=~\"$role\", service=~\"$service\", cache=\"blocks\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Block cache queued consumers", + "range": true, + "refId": "D", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_rpc_eth_cache_memory_usage{chain=\"$chain\", role=~\"$role\", service=~\"$service\", cache=\"blocks\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Blocks cache memory usage", + "range": true, + "refId": "E", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_rpc_eth_cache_cached_count{chain=\"$chain\", role=~\"$role\", service=~\"$service\", cache=\"receipts\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Receipts cache cached items", + "range": true, + "refId": "F", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_rpc_eth_cache_memory_usage{chain=\"$chain\", role=~\"$role\", service=~\"$service\", cache=\"receipts\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Receipts cache memory usage", + "range": true, + "refId": "G", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_rpc_eth_cache_cached_count{chain=\"$chain\", role=~\"$role\", service=~\"$service\", cache=\"blocks\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Block cache cached items", + "range": true, + "refId": "H", + "useBackend": false + } + ], + "title": "RPC Cache Metrics", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 235 + }, + "id": 66, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (service, method) (rate(reth_rpc_server_calls_successful_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])) > 0", + "instant": false, + "legendFormat": "{{service}} · {{method}}", + "range": true, + "refId": "A" + } + ], + "title": "RPC Throughput", + "type": "timeseries" + } + ], + "title": "RPC", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 139 + }, + "id": 117, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 441 + }, + "id": 119, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_process_resident_memory_bytes{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "instant": false, + "legendFormat": "{{service}}", + "range": true, + "refId": "A" + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "100% = 1 core", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 441 + }, + "id": 120, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "avg by (service) (rate(reth_process_cpu_seconds_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[1m]))", + "instant": false, + "legendFormat": "{{service}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 449 + }, + "id": 121, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_process_open_fds{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "instant": false, + "legendFormat": "{{service}} · Open", + "range": true, + "refId": "A" + } + ], + "title": "File Descriptors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Tracks the number of critical tasks currently ran by the executor.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "semi-dark-red", + "value": 0 + } + ] + }, + "unit": "tasks", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 449 + }, + "id": 122, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_executor_spawn_critical_tasks_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}- reth_executor_spawn_finished_critical_tasks_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "hide": false, + "instant": false, + "legendFormat": "{{service}} · Tasks running", + "range": true, + "refId": "C" + } + ], + "title": "Task Executor critical tasks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Tracks the number of regular tasks currently ran by the executor.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "semi-dark-red", + "value": 80 + } + ] + }, + "unit": "tasks/s", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "unit", + "value": "tasks" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 457 + }, + "id": 123, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "rate(reth_executor_spawn_regular_tasks_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "{{service}} · Tasks started", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_executor_spawn_regular_tasks_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} - reth_executor_spawn_finished_regular_tasks_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "hide": false, + "instant": false, + "legendFormat": "{{service}} · Tasks running", + "range": true, + "refId": "C" + } + ], + "title": "Task Executor regular tasks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Tracks the number of regular blocking tasks currently ran by the executor.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "semi-dark-red", + "value": 80 + } + ] + }, + "unit": "tasks/s", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "unit", + "value": "tasks" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 457 + }, + "id": 124, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "rate(reth_executor_spawn_regular_blocking_tasks_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "{{service}} · Tasks started", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_executor_spawn_regular_blocking_tasks_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} - reth_executor_spawn_finished_regular_blocking_tasks_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "hide": false, + "instant": false, + "legendFormat": "{{service}} · Tasks running", + "range": true, + "refId": "C" + } + ], + "title": "Task Executor regular blocking tasks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 465 + }, + "id": 125, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(reth_pruner_duration_seconds_sum{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval]) / rate(reth_pruner_duration_seconds_count{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval]) > 0", + "instant": false, + "legendFormat": "{{service}}", + "range": true, + "refId": "A" + } + ], + "title": "Pruner duration, total", + "type": "timeseries" + } + ], + "title": "Process & Maintenance", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 140 + }, + "id": 80, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Tracks a heuristic of the memory footprint of the various transaction pool sub-pools", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 293 + }, + "id": 82, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_transaction_pool_basefee_pool_size_bytes{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", + "legendFormat": "{{service}} · Base Fee Pool Size", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_transaction_pool_pending_pool_size_bytes{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", + "hide": false, + "legendFormat": "{{service}} · Pending Pool Size", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_transaction_pool_queued_pool_size_bytes{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", + "hide": false, + "legendFormat": "{{service}} · Queued Pool Size", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_transaction_pool_blob_pool_size_bytes{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", + "legendFormat": "{{service}} · Blob Pool Size", + "range": true, + "refId": "D" + } + ], + "title": "Subpool Sizes in Bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Transaction pool maintenance metrics", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 293 + }, + "id": 83, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_transaction_pool_dirty_accounts{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Dirty Accounts", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_transaction_pool_drift_count{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Drift Count", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_transaction_pool_reinserted_transactions{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Reinserted Transactions", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_transaction_pool_deleted_tracked_finalized_blobs{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Deleted Tracked Finalized Blobs", + "range": true, + "refId": "D", + "useBackend": false + } + ], + "title": "TxPool Maintenance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Tracks the number of transactions in the various transaction pool sub-pools", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 301 + }, + "id": 84, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_transaction_pool_basefee_pool_transactions{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", + "legendFormat": "{{service}} · Base Fee Pool Transactions", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_transaction_pool_pending_pool_transactions{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", + "hide": false, + "legendFormat": "{{service}} · Pending Pool Transactions", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_transaction_pool_queued_pool_transactions{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", + "hide": false, + "legendFormat": "{{service}} · Queued Pool Transactions", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_transaction_pool_blob_pool_transactions{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", + "legendFormat": "{{service}} · Blob Pool Transactions", + "range": true, + "refId": "D" + } + ], + "title": "Subpool Transaction Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Tracks the number of transactions per second that are inserted and removed from the transaction pool, as well as the number of invalid transactions per second.\n\nBad transactions are a subset of invalid transactions, these will never be successfully imported. The remaining invalid transactions have a chance of being imported, for example transactions with nonce gaps.\n\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 301 + }, + "id": 85, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_transaction_pool_inserted_transactions{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{service}} · Inserted Transactions", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_transaction_pool_removed_transactions{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "{{service}} · Removed Transactions", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_transaction_pool_invalid_transactions{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "{{service}} · Invalid Transactions", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_network_bad_imports{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "{{service}} · Bad Transactions", + "range": true, + "refId": "D", + "useBackend": false + } + ], + "title": "Inserted Transactions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of transactions about to be imported into the pool.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 309 + }, + "id": 86, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_network_pending_pool_imports{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "hide": false, + "legendFormat": "{{service}} · Transactions Pending Import", + "range": true, + "refId": "C" + } + ], + "title": "Pending Pool Imports", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Tracks the number of incoming transaction messages in the channel from the network to the transaction pool.\n\nMempool messages sent over this channel are `GetPooledTransactions` requests, `NewPooledTransactionHashes` announcements (gossip), and `Transactions` (gossip)\n\nTx - `NetworkManager`\n\\nRx - `TransactionsManager`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "mps", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "unit", + "value": "events" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 309 + }, + "id": 87, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "rate(reth_network_pool_transactions_messages_sent_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{service}} · Tx", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "rate(reth_network_pool_transactions_messages_received_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "hide": false, + "legendFormat": "{{service}} · Rx", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_network_pool_transactions_messages_sent_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} - reth_network_pool_transactions_messages_received_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "hide": false, + "legendFormat": "{{service}} · Messages in Channel", + "range": true, + "refId": "C" + } + ], + "title": "Incoming Gossip and Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Measures the message send rate (MPS) for queued outgoing messages. Outgoing messages are added to the queue before being sent to other peers, and this metric helps track the rate of message dispatch.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "mps", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 317 + }, + "id": 88, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_network_queued_outgoing_messages{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Queued Messages per Second", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Queued Outgoing Messages", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "All Transactions metrics", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 317 + }, + "id": 89, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_transaction_pool_all_transactions_by_hash{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · All transactions by hash", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_transaction_pool_all_transactions_by_id{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · All transactions by id", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_transaction_pool_all_transactions_by_all_senders{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · All transactions by all senders", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_transaction_pool_blob_transactions_nonce_gaps{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Blob transactions nonce gaps", + "range": true, + "refId": "D", + "useBackend": false + } + ], + "title": "All Transactions metrics", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Egress RLPx capability traffic (default only `eth` capability)\n\nDropped - session channels are bounded, if there's no capacity, the message will be dropped.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "mps", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 325 + }, + "id": 90, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_network_total_outgoing_peer_messages_dropped{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Dropped", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Outgoing Capability Messages", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total number of times a transaction is sent/announced that is already in the local pool.\n\nThis reflects the redundancy in the mempool.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 325 + }, + "id": 91, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_network_occurrences_hashes_already_in_pool{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "{{service}} · Freq Announced Transactions Already in Pool", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_network_occurrences_transactions_already_in_pool{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "{{service}} · Freq Received Transactions Already in Pool ", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Frequency of Transactions Already in Pool", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Currently active outgoing GetPooledTransactions requests.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 333 + }, + "id": 92, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_network_inflight_transaction_requests{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "hide": false, + "legendFormat": "{{service}} · Inflight Transaction Requests", + "range": true, + "refId": "C" + } + ], + "title": "Inflight Transaction Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Duration of one call to poll `TransactionsManager` future, and its nested function calls.\n\nNetwork Events - stream peer session updates from `NetworkManager`;\nTransaction Events - stream txns gossip from `NetworkManager`;\nPending Transactions - stream hashes of txns successfully inserted into pending set in `TransactionPool`;\nPending Pool Imports - flush txns to pool from `TransactionsManager`;\nFetch Events - stream fetch txn events (success case wraps a tx) from `TransactionFetcher`;\nFetch Pending Hashes - search for hashes announced by an idle peer in cache for hashes pending fetch;\n(Transactions Commands - stream commands from testnet to fetch/serve/propagate txns)\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 333 + }, + "id": 93, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_network_acc_duration_poll_network_events{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Network Events", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_network_acc_duration_poll_transaction_events{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Transaction Events", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "reth_network_acc_duration_poll_imported_transactions{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Pending Transactions", + "range": true, + "refId": "D", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_network_acc_duration_poll_pending_pool_imports{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Pending Pool Imports", + "range": true, + "refId": "E", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_network_acc_duration_poll_fetch_events{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Fetch Events", + "range": true, + "refId": "F", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_network_acc_duration_poll_commands{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Commands", + "range": true, + "refId": "G", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_network_acc_duration_fetch_pending_hashes{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Fetch Pending Hashes", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_network_duration_poll_tx_manager{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Total Transactions Manager Future", + "range": true, + "refId": "H", + "useBackend": false + } + ], + "title": "Transactions Manager Poll Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 341 + }, + "id": 94, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "reth_network_hashes_pending_fetch{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Hashes in Pending Fetch Cache", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "reth_network_inflight_transaction_requests{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Hashes in Inflight Requests", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (service) (reth_network_hashes_inflight_transaction_requests{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}) + sum by (service) (reth_network_hashes_pending_fetch{chain=\"$chain\", role=~\"$role\", service=~\"$service\"})", + "hide": false, + "instant": false, + "legendFormat": "{{service}} · Total Hashes in Transaction Fetcher", + "range": true, + "refId": "C" + } + ], + "title": "Transaction Fetcher Hashes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Durations of one call to poll `NetworkManager` future, and its nested function calls.\n\nNetwork Handle Message - stream network handle messages from `TransactionsManager`;\nSwarm Events - stream transaction gossip from `Swarm`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 341 + }, + "id": 95, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_network_acc_duration_poll_network_handle{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Network Handle Messages", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_network_acc_duration_poll_swarm{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Swarm Events", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_network_duration_poll_network_manager{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Total Network Manager Future", + "range": true, + "refId": "C", + "useBackend": false + } + ], + "title": "Network Manager Poll Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Frequency of a peer sending a transaction that has already been marked as seen by that peer. This could for example be the case if a transaction is sent/announced to the peer at the same time that the peer sends/announces the same transaction to us.\n\nThis reflects the latency in the mempool.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 349 + }, + "id": 96, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_network_occurrences_hash_already_seen_by_peer{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "{{service}} · Freq Announced Transactions Already Seen by Peer", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_network_occurrences_of_transaction_already_seen_by_peer{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "{{service}} · Freq Received Transactions Already Seen by Peer", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Frequency of Transactions Already Marked as Seen by Peer", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of all transactions of all sub-pools by type", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 349 + }, + "id": 97, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "reth_transaction_pool_total_legacy_transactions{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "{{service}} · Legacy", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "reth_transaction_pool_total_eip2930_transactions{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "{{service}} · EIP-2930", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "reth_transaction_pool_total_eip1559_transactions{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "{{service}} · EIP-1559", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "reth_transaction_pool_total_eip4844_transactions{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "{{service}} · EIP-4844", + "range": true, + "refId": "D", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "reth_transaction_pool_total_eip7702_transactions{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "{{service}} · EIP-7702", + "range": true, + "refId": "E", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "reth_transaction_pool_total_other_transactions{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "{{service}} · Morph (0x7F/other)", + "range": true, + "refId": "F", + "useBackend": false + } + ], + "title": "Transactions by Type in Pool", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Duration of one call to `TransactionFetcher::on_fetch_pending_hashes`.\n\nFind Peer - find an idle fallback peer for a hash pending fetch.\n\nFill Request - fill `GetPooledTransactions` request, for the found peer, with more hashes from cache of hashes pending fetch. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 357 + }, + "id": 98, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_network_duration_find_idle_fallback_peer_for_any_pending_hash{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Find Idle Peer", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_network_duration_fill_request_from_hashes_pending_fetch{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} · Fill Request", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Fetch Hashes Pending Fetch Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Frequency of transaction types seen in announcements", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 357 + }, + "id": 99, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_network_transaction_fetcher_legacy_sum{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "{{service}} · Legacy", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_network_transaction_fetcher_eip2930_sum{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "{{service}} · Eip2930", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_network_transaction_fetcher_eip1559_sum{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "{{service}} · Eip1559", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_network_transaction_fetcher_eip4844_sum{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "{{service}} · Eip4844", + "range": true, + "refId": "D", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_network_transaction_fetcher_eip7702_sum{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "{{service}} · Eip7702", + "range": true, + "refId": "E", + "useBackend": false + } + ], + "title": "Announced Transactions by Type", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of transactions evicted in each pool", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 365 + }, + "id": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "disableTextWrap": false, + "editorMode": "code", + "expr": "reth_transaction_pool_pending_transactions_evicted{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{service}} · PendingPool", + "range": true, + "refId": "A", + "useBackend": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_transaction_pool_basefee_transactions_evicted{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", + "hide": false, + "instant": false, + "legendFormat": "{{service}} · BasefeePool", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_transaction_pool_blob_transactions_evicted{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", + "hide": false, + "instant": false, + "legendFormat": "{{service}} · BlobPool", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_transaction_pool_queued_transactions_evicted{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", + "hide": false, + "instant": false, + "legendFormat": "{{service}} · QueuedPool", + "range": true, + "refId": "D" + } + ], + "title": "Evicted Transactions", + "type": "timeseries" + } + ], + "title": "TxPool", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 141 + }, + "id": 132, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The number of tracked peers in the discovery modules (dnsdisc and discv4)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "y": 293, + "x": 0, + "w": 12, + "h": 8 + }, + "id": 101, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_network_tracked_peers{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "legendFormat": "{{service}} · Tracked Peers", + "range": true, + "refId": "A" + } + ], + "title": "Discovery: Tracked peers", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The number of incoming and outgoing connections, as well as the number of peers we are currently connected to. Outgoing and incoming connections also count peers we are trying to connect to.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "y": 293, + "x": 12, + "w": 12, + "h": 8 + }, + "id": 102, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_network_outgoing_connections{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "legendFormat": "{{service}} · Outgoing Connections", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_network_incoming_connections{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "hide": false, + "legendFormat": "{{service}} · Incoming Connections", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_network_connected_peers{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "hide": false, + "legendFormat": "{{service}} · Connected Peers", + "range": true, + "refId": "E" + } + ], + "title": "P2P Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Internal errors in the P2P module. These are expected to happen from time to time. High error rates should not cause alarm if the node is peering otherwise.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "y": 301, + "x": 0, + "w": 12, + "h": 8 + }, + "id": 103, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "rate(reth_p2pstream_disconnected_errors{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "legendFormat": "{{service}} · P2P Stream Disconnected", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "rate(reth_network_pending_session_failures{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "hide": false, + "legendFormat": "{{service}} · Failed Pending Sessions", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "rate(reth_network_invalid_messages_received_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "hide": false, + "legendFormat": "{{service}} · Invalid Messages", + "range": true, + "refId": "C" + } + ], + "title": "P2P Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "y": 301, + "x": 12, + "w": 12, + "h": 8 + }, + "id": 104, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_network_useless_peer{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", + "legendFormat": "{{service}} · UselessPeer", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_network_subprotocol_specific{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", + "hide": false, + "legendFormat": "{{service}} · SubprotocolSpecific", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_network_already_connected{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", + "hide": false, + "legendFormat": "{{service}} · AlreadyConnected", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_network_client_quitting{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", + "hide": false, + "legendFormat": "{{service}} · ClientQuitting", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_network_unexpected_identity{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", + "hide": false, + "legendFormat": "{{service}} · UnexpectedHandshakeIdentity", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_network_disconnect_requested{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", + "hide": false, + "legendFormat": "{{service}} · DisconnectRequested", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_network_null_node_identity{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", + "hide": false, + "legendFormat": "{{service}} · NullNodeIdentity", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_network_tcp_subsystem_error{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", + "hide": false, + "legendFormat": "{{service}} · TCPSubsystemError", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_network_incompatible{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", + "hide": false, + "legendFormat": "{{service}} · IncompatibleP2PVersion", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_network_protocol_breach{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", + "hide": false, + "legendFormat": "{{service}} · ProtocolBreach", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_network_too_many_peers{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", + "hide": false, + "legendFormat": "{{service}} · TooManyPeers", + "range": true, + "refId": "K" + } + ], + "title": "P2P Peer Disconnect Reasons", + "type": "piechart", + "description": "每种 disconnect reason 的计数。空 slice 表示该类断连没发生过(健康)。只要节点稳定运行,这个图应该为空。" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of successful outgoing dial attempts.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "y": 309, + "x": 0, + "w": 24, + "h": 8 + }, + "id": 105, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_network_total_dial_successes{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "legendFormat": "{{service}} · Total Dial Successes", + "range": true, + "refId": "A" + } + ], + "title": "P2P Total Dial Success", + "type": "timeseries" + } + ], + "title": "P2P", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 142 + }, + "id": 106, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 390 + }, + "id": 107, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg by (service) (delta(reth_db_table_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", table=\"PlainAccountState\"}[$interval]))", + "instant": false, + "interval": "$interval", + "legendFormat": "{{service}} · Account", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg by (service) (delta(reth_db_table_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", table=\"PlainStorageState\"}[$interval]))", + "hide": false, + "instant": false, + "interval": "$interval", + "legendFormat": "{{service}} · Storage", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg by (service) (delta(reth_db_table_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", table=\"Bytecodes\"}[$interval]))", + "hide": false, + "instant": false, + "interval": "$interval", + "legendFormat": "{{service}} · Bytecodes", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg by (service) (delta(reth_db_table_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", table=\"PlainAccountState\"}[$interval])) + avg by (service) (delta(reth_db_table_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", table=\"PlainStorageState\"}[$interval])) + avg by (service) (delta(reth_db_table_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", table=\"Bytecodes\"}[$interval]))", + "hide": false, + "instant": false, + "interval": "$interval", + "legendFormat": "{{service}} · Total", + "range": true, + "refId": "D" + } + ], + "title": "State Growth (interval = ${interval})", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 390 + }, + "id": 108, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (service) (reth_db_table_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", table=\"PlainAccountState\"})", + "instant": false, + "interval": "10m", + "legendFormat": "{{service}} · Account", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (service) (reth_db_table_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", table=\"PlainStorageState\"})", + "hide": false, + "instant": false, + "interval": "10m", + "legendFormat": "{{service}} · Storage", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (service) (reth_db_table_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", table=\"Bytecodes\"})", + "hide": false, + "instant": false, + "interval": "10m", + "legendFormat": "{{service}} · Bytecodes", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (service) (reth_db_table_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", table=~\"PlainAccountState|PlainStorageState|Bytecodes\"})", + "hide": false, + "instant": false, + "interval": "10m", + "legendFormat": "{{service}} · Total", + "range": true, + "refId": "D" + } + ], + "title": "State Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 400 + }, + "id": 109, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg by (service) (delta(reth_db_table_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", table=\"PlainAccountState\"}[$interval]))", + "instant": false, + "interval": "$interval", + "legendFormat": "{{service}}", + "range": true, + "refId": "A" + } + ], + "title": "Account State Growth (interval = ${interval})", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 400 + }, + "id": 110, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg by (service) (delta(reth_db_table_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", table=\"PlainStorageState\"}[$interval]))", + "instant": false, + "interval": "$interval", + "legendFormat": "{{service}}", + "range": true, + "refId": "A" + } + ], + "title": "Storage State Growth (interval = ${interval})", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 410 + }, + "id": 111, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg by (service) (delta(reth_db_table_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", table=\"Bytecodes\"}[$interval]))", + "instant": false, + "interval": "$interval", + "legendFormat": "{{service}}", + "range": true, + "refId": "A" + } + ], + "title": "Bytecodes Growth (interval = ${interval})", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 410 + }, + "id": 112, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg by (service) (delta(reth_db_table_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", table=\"Headers\"}[$interval])) + avg by (service) (delta(reth_static_files_segment_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", segment=\"headers\"}[$interval]))", + "instant": false, + "interval": "$interval", + "legendFormat": "{{service}} · Headers", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg by (service) (delta(reth_db_table_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", table=\"Receipts\"}[$interval])) + avg by (service) (delta(reth_static_files_segment_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", segment=\"receipts\"}[$interval]))", + "hide": false, + "instant": false, + "interval": "$interval", + "legendFormat": "{{service}} · Receipts", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg by (service) (delta(reth_db_table_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", table=\"Transactions\"}[$interval])) + avg by (service) (delta(reth_static_files_segment_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", segment=\"transactions\"}[$interval]))", + "hide": false, + "instant": false, + "interval": "$interval", + "legendFormat": "{{service}} · Transactions", + "range": true, + "refId": "C" + } + ], + "title": "History Growth (interval = ${interval})", + "transformations": [ + { + "id": "calculateField", + "options": { + "binary": { + "left": "Headers", + "reducer": "sum", + "right": "Receipts" + }, + "mode": "reduceRow", + "reduce": { + "include": [ + "Headers", + "Receipts", + "Transactions" + ], + "reducer": "sum" + } + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 420 + }, + "id": 113, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (service) (reth_db_table_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", table=\"Headers\"}) + sum by (service) (reth_static_files_segment_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", segment=\"headers\"})", + "instant": false, + "interval": "10m", + "legendFormat": "{{service}} · Headers", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (service) (reth_db_table_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", table=\"Receipts\"}) + sum by (service) (reth_static_files_segment_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", segment=\"receipts\"})", + "hide": false, + "instant": false, + "interval": "10m", + "legendFormat": "{{service}} · Receipts", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (service) (reth_db_table_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", table=\"Transactions\"}) + sum by (service) (reth_static_files_segment_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", segment=\"transactions\"})", + "hide": false, + "instant": false, + "interval": "10m", + "legendFormat": "{{service}} · Transactions", + "range": true, + "refId": "C" + } + ], + "title": "History Size", + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reduce": { + "include": [ + "Headers", + "Receipts", + "Transactions" + ], + "reducer": "sum" + } + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 420 + }, + "id": 114, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg by (service) (delta(reth_db_table_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", table=\"Headers\"}[$interval])) + avg by (service) (delta(reth_static_files_segment_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", segment=\"headers\"}[$interval]))", + "instant": false, + "interval": "$interval", + "legendFormat": "{{service}}", + "range": true, + "refId": "A" + } + ], + "title": "Headers Growth (interval = ${interval})", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 430 + }, + "id": 115, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg by (service) (delta(reth_db_table_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", table=\"Receipts\"}[$interval])) + avg by (service) (delta(reth_static_files_segment_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", segment=\"receipts\"}[$interval]))", + "instant": false, + "interval": "$interval", + "legendFormat": "{{service}}", + "range": true, + "refId": "A" + } + ], + "title": "Receipts Growth (interval = ${interval})", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 430 + }, + "id": 116, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg by (service) (delta(reth_db_table_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", table=\"Transactions\"}[$interval])) + avg by (service) (delta(reth_static_files_segment_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", segment=\"transactions\"}[$interval]))", + "instant": false, + "interval": "$interval", + "legendFormat": "{{service}}", + "range": true, + "refId": "A" + } + ], + "title": "Transactions Growth (interval = ${interval})", + "type": "timeseries" + } + ], + "title": "State & History", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 143 + }, + "id": 67, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "D" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 244 + }, + "id": 68, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_downloaders_headers_total_downloaded{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "legendFormat": "{{service}} · Downloaded", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_downloaders_headers_total_flushed{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "hide": false, + "legendFormat": "{{service}} · Flushed", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "rate(reth_downloaders_headers_total_downloaded{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{service}} · Downloaded/s", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "rate(reth_downloaders_headers_total_flushed{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "hide": false, + "legendFormat": "{{service}} · Flushed/s", + "range": true, + "refId": "D" + } + ], + "title": "Headers I/O", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Internal errors in the header downloader. These are expected to happen from time to time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 244 + }, + "id": 69, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "rate(reth_downloaders_headers_timeout_errors{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "legendFormat": "{{service}} · Request timed out", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "rate(reth_downloaders_headers_unexpected_errors{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "hide": false, + "legendFormat": "{{service}} · Unexpected error", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "rate(reth_downloaders_headers_validation_errors{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "hide": false, + "legendFormat": "{{service}} · Invalid response", + "range": true, + "refId": "C" + } + ], + "title": "Headers Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The number of connected peers and in-progress requests for headers.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 252 + }, + "id": 70, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_downloaders_headers_in_flight_requests{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "legendFormat": "{{service}} · In flight requests", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_network_connected_peers{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "hide": false, + "legendFormat": "{{service}} · Connected peers", + "range": true, + "refId": "B" + } + ], + "title": "Headers Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The internal state of the headers downloader: the number of downloaded headers, and the number of headers sent to the header stage.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "locale", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "ops" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "D" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "ops" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 252 + }, + "id": 71, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_downloaders_bodies_total_downloaded{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "legendFormat": "{{service}} · Downloaded", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_downloaders_bodies_total_flushed{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "hide": false, + "legendFormat": "{{service}} · Flushed", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "rate(reth_downloaders_bodies_total_flushed{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "hide": false, + "legendFormat": "{{service}} · Flushed/s", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "rate(reth_downloaders_bodies_total_downloaded{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "hide": false, + "legendFormat": "{{service}} · Downloaded/s", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_downloaders_bodies_buffered_responses{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "hide": false, + "legendFormat": "{{service}} · Buffered responses", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_downloaders_bodies_buffered_blocks{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "hide": false, + "legendFormat": "{{service}} · Buffered blocks", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_downloaders_bodies_queued_blocks{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "hide": false, + "legendFormat": "{{service}} · Queued blocks", + "range": true, + "refId": "G" + } + ], + "title": "Bodies I/O", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Internal errors in the bodies downloader. These are expected to happen from time to time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 260 + }, + "id": 72, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "rate(reth_downloaders_bodies_timeout_errors{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "legendFormat": "{{service}} · Request timed out", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "rate(reth_downloaders_bodies_unexpected_errors{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "hide": false, + "legendFormat": "{{service}} · Unexpected error", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "rate(reth_downloaders_bodies_validation_errors{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "hide": false, + "legendFormat": "{{service}} · Invalid response", + "range": true, + "refId": "C" + } + ], + "title": "Bodies Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The number of connected peers and in-progress requests for bodies.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 260 + }, + "id": 73, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_downloaders_bodies_in_flight_requests{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "legendFormat": "{{service}} · In flight requests", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_network_connected_peers{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "hide": false, + "legendFormat": "{{service}} · Connected peers", + "range": true, + "refId": "B" + } + ], + "title": "Bodies Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The number of blocks and size in bytes of those blocks", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "blocks" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 268 + }, + "id": 74, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_downloaders_bodies_buffered_blocks_size_bytes{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "hide": false, + "legendFormat": "{{service}} · Buffered blocks size ", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_downloaders_bodies_buffered_blocks{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "hide": false, + "legendFormat": "{{service}} · Buffered blocks", + "range": true, + "refId": "B" + } + ], + "title": "Downloader buffer", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "http" + }, + "properties": [ + { + "id": "displayName", + "value": "HTTP" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ws" + }, + "properties": [ + { + "id": "displayName", + "value": "WebSocket" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 268 + }, + "id": 76, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_network_eth_headers_requests_received_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{service}} · Headers Requests/s", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Headers Requests Received", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "http" + }, + "properties": [ + { + "id": "displayName", + "value": "HTTP" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ws" + }, + "properties": [ + { + "id": "displayName", + "value": "WebSocket" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 276 + }, + "id": 77, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_network_eth_receipts_requests_received_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{service}} · Receipts Requests/s", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Receipts Requests Received", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "http" + }, + "properties": [ + { + "id": "displayName", + "value": "HTTP" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ws" + }, + "properties": [ + { + "id": "displayName", + "value": "WebSocket" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 276 + }, + "id": 78, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_network_eth_bodies_requests_received_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{service}} · Bodies Requests/s", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Bodies Requests Received", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "http" + }, + "properties": [ + { + "id": "displayName", + "value": "HTTP" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ws" + }, + "properties": [ + { + "id": "displayName", + "value": "WebSocket" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 284 + }, + "id": 79, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_network_eth_node_data_requests_received_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{service}} · Node Data Requests/s", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Node Data Requests Received", + "type": "timeseries" + } + ], + "title": "Sync & Downloaders", + "type": "row" + } + ], + "schemaVersion": 39, + "tags": [ + "morph", + "reth", + "engine-api", + "payload-builder" + ], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "name": "datasource", + "query": "prometheus", + "refresh": 1, + "type": "datasource" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(reth_info, chain)", + "hide": 0, + "includeAll": false, + "label": "chain", + "name": "chain", + "query": "label_values(reth_info, chain)", + "refresh": 2, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "0.95", + "value": "0.95" + }, + "hide": 0, + "includeAll": false, + "label": "quantile", + "name": "quantile", + "options": [ + { + "selected": false, + "text": "0.5", + "value": "0.5" + }, + { + "selected": false, + "text": "0.9", + "value": "0.9" + }, + { + "selected": true, + "text": "0.95", + "value": "0.95" + }, + { + "selected": false, + "text": "0.99", + "value": "0.99" + }, + { + "selected": false, + "text": "0.999", + "value": "0.999" + } + ], + "query": "0.5, 0.9, 0.95, 0.99, 0.999", + "type": "custom" + }, + { + "allValue": ".*", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(reth_info{chain=\"$chain\", role=~\"$role\"}, service)", + "hide": 0, + "includeAll": true, + "multi": true, + "label": "service", + "name": "service", + "query": "label_values(reth_info{chain=\"$chain\", role=~\"$role\"}, service)", + "refresh": 2, + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(reth_info{chain=\"$chain\"}, role)", + "hide": 0, + "includeAll": true, + "multi": false, + "label": "role", + "name": "role", + "query": "label_values(reth_info{chain=\"$chain\"}, role)", + "refresh": 2, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "5m", + "value": "5m" + }, + "hide": 0, + "includeAll": false, + "label": "Interval", + "multi": false, + "name": "interval", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1m,5m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "type": "custom" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "title": "Morph Reth", + "uid": "morph-reth-unified", + "version": 1 +} diff --git a/etc/grafana/datasources/prometheus.yml b/etc/grafana/datasources/prometheus.yml new file mode 100644 index 0000000..6561365 --- /dev/null +++ b/etc/grafana/datasources/prometheus.yml @@ -0,0 +1,8 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: $PROMETHEUS_URL + editable: true diff --git a/etc/prometheus/prometheus.yml b/etc/prometheus/prometheus.yml new file mode 100644 index 0000000..a443a71 --- /dev/null +++ b/etc/prometheus/prometheus.yml @@ -0,0 +1,14 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s + +scrape_configs: + - job_name: morph-reth + metrics_path: "/" + scrape_interval: 5s + static_configs: + - targets: ['morph-reth:9001'] + labels: + service: morph-reth + role: sequencer + chain: morph