Skip to content

fix: fix and cleanup metrics (sequencers + block) #2904

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tac0turtle merged 5 commits into from
Dec 3, 2025
Merged

fix: fix and cleanup metrics (sequencers + block) #2904

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4138425
fix: fix and cleanup metrics (sequencers + block)
julienrbrt Dec 2, 2025
ffc898a
clean-up
julienrbrt Dec 2, 2025
6fa6446
simplify ticket
julienrbrt Dec 2, 2025
aeb9867
properly track
julienrbrt Dec 2, 2025
08cb5c6
add changelog
tac0turtle Dec 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 0 additions & 6 deletions apps/evm/cmd/run.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,19 +111,13 @@ func createSequencer(
nodeConfig config.Config,
genesis genesis.Genesis,
) (coresequencer.Sequencer, error) {
singleMetrics, err := single.NopMetrics()
if err != nil {
return nil, fmt.Errorf("failed to create single sequencer metrics: %w", err)
}

sequencer, err := single.NewSequencer(
ctx,
logger,
datastore,
da,
[]byte(genesis.ChainID),
nodeConfig.Node.BlockTime.Duration,
singleMetrics,
nodeConfig.Node.Aggregator,
)
if err != nil {
Expand Down
6 changes: 0 additions & 6 deletions apps/grpc/cmd/run.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,19 +119,13 @@ func createSequencer(
nodeConfig config.Config,
genesis genesis.Genesis,
) (coresequencer.Sequencer, error) {
singleMetrics, err := single.NopMetrics()
if err != nil {
return nil, fmt.Errorf("failed to create single sequencer metrics: %w", err)
}

sequencer, err := single.NewSequencer(
ctx,
logger,
datastore,
da,
[]byte(genesis.ChainID),
nodeConfig.Node.BlockTime.Duration,
singleMetrics,
nodeConfig.Node.Aggregator,
)
if err != nil {
Expand Down
6 changes: 0 additions & 6 deletions apps/testapp/cmd/run.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,11 +68,6 @@ var RunCmd = &cobra.Command{
return err
}

singleMetrics, err := single.NopMetrics()
if err != nil {
return err
}

// Start the KV executor HTTP server
if kvEndpoint != "" { // Only start if endpoint is provided
httpServer := kvexecutor.NewHTTPServer(executor, kvEndpoint)
Expand Down Expand Up @@ -101,7 +96,6 @@ var RunCmd = &cobra.Command{
&daJrpc.DA,
[]byte(genesis.ChainID),
nodeConfig.Node.BlockTime.Duration,
singleMetrics,
nodeConfig.Node.Aggregator,
)
if err != nil {
Expand Down
262 changes: 12 additions & 250 deletions block/internal/common/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,54 +49,22 @@ type Metrics struct {
BlockSizeBytes metrics.Gauge // Size of the latest block
TotalTxs metrics.Gauge // Total number of transactions
CommittedHeight metrics.Gauge `metrics_name:"latest_block_height"` // The latest block height

// Channel metrics
ChannelBufferUsage map[string]metrics.Gauge
DroppedSignals metrics.Counter

// Error metrics
ErrorsByType map[string]metrics.Counter
RecoverableErrors metrics.Counter
NonRecoverableErrors metrics.Counter
TxsPerBlock metrics.Histogram

// Performance metrics
OperationDuration map[string]metrics.Histogram
GoroutineCount metrics.Gauge

// DA metrics
DASubmissionAttempts metrics.Counter
DASubmissionSuccesses metrics.Counter
DASubmissionFailures metrics.Counter
DARetrievalAttempts metrics.Counter
DARetrievalSuccesses metrics.Counter
DARetrievalFailures metrics.Counter
DAInclusionHeight metrics.Gauge
PendingHeadersCount metrics.Gauge
PendingDataCount metrics.Gauge

// Sync metrics
SyncLag metrics.Gauge
HeadersSynced metrics.Counter
DataSynced metrics.Counter
BlocksApplied metrics.Counter
InvalidHeadersCount metrics.Counter

// Block production metrics
BlockProductionTime metrics.Histogram
EmptyBlocksProduced metrics.Counter
LazyBlocksProduced metrics.Counter
NormalBlocksProduced metrics.Counter
TxsPerBlock metrics.Histogram

// State transition metrics
StateTransitions map[string]metrics.Counter
InvalidTransitions metrics.Counter

// DA Submitter metrics
DASubmitterFailures map[DASubmitterFailureReason]metrics.Counter // Counter with reason label
DASubmitterLastFailure map[DASubmitterFailureReason]metrics.Gauge // Timestamp gauge with reason label
DASubmitterPendingBlobs metrics.Gauge // Total number of blobs awaiting submission (backlog)
DASubmitterResends metrics.Counter // Number of resend attempts
DARetrievalAttempts metrics.Counter
DARetrievalSuccesses metrics.Counter
DARetrievalFailures metrics.Counter
DAInclusionHeight metrics.Gauge
PendingHeadersCount metrics.Gauge
PendingDataCount metrics.Gauge
}

// PrometheusMetrics returns Metrics built using Prometheus client library
Expand All @@ -107,10 +75,7 @@ func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
}

m := &Metrics{
ChannelBufferUsage: make(map[string]metrics.Gauge),
ErrorsByType: make(map[string]metrics.Counter),
OperationDuration: make(map[string]metrics.Histogram),
StateTransitions: make(map[string]metrics.Counter),
DASubmitterFailures: make(map[DASubmitterFailureReason]metrics.Counter),
DASubmitterLastFailure: make(map[DASubmitterFailureReason]metrics.Gauge),
}
Expand Down Expand Up @@ -151,63 +116,12 @@ func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
Help: "The latest block height.",
}, labels).With(labelsAndValues...)

// Channel metrics
m.DroppedSignals = prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "dropped_signals_total",
Help: "Total number of dropped channel signals",
}, labels).With(labelsAndValues...)

// Initialize channel buffer usage gauges
channelNames := []string{"height_in", "header_store", "data_store", "retrieve", "da_includer", "tx_notify"}
for _, name := range channelNames {
m.ChannelBufferUsage[name] = prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "channel_buffer_usage",
Help: "Current buffer usage of channels",
ConstLabels: map[string]string{
"channel": name,
},
}, labels).With(labelsAndValues...)
}

// Error metrics
m.RecoverableErrors = prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "recoverable_errors_total",
Help: "Total number of recoverable errors",
}, labels).With(labelsAndValues...)

m.NonRecoverableErrors = prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "non_recoverable_errors_total",
Help: "Total number of non-recoverable errors",
}, labels).With(labelsAndValues...)

// Initialize error type counters
errorTypes := []string{"block_production", "da_submission", "sync", "validation", "state_update"}
for _, errType := range errorTypes {
m.ErrorsByType[errType] = prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "errors_by_type_total",
Help: "Total number of errors by type",
ConstLabels: map[string]string{
"error_type": errType,
},
}, labels).With(labelsAndValues...)
}

// Performance metrics
m.GoroutineCount = prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
m.TxsPerBlock = prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "goroutines_count",
Help: "Current number of goroutines",
Name: "txs_per_block",
Help: "Number of transactions per block",
Buckets: []float64{0, 1, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000},
}, labels).With(labelsAndValues...)

// Initialize operation duration histograms
Expand All @@ -226,27 +140,6 @@ func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
}

// DA metrics
m.DASubmissionAttempts = prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "da_submission_attempts_total",
Help: "Total number of DA submission attempts",
}, labels).With(labelsAndValues...)

m.DASubmissionSuccesses = prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "da_submission_successes_total",
Help: "Total number of successful DA submissions",
}, labels).With(labelsAndValues...)

m.DASubmissionFailures = prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "da_submission_failures_total",
Help: "Total number of failed DA submissions",
}, labels).With(labelsAndValues...)

m.DARetrievalAttempts = prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Expand Down Expand Up @@ -289,102 +182,6 @@ func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
Help: "Number of data blocks pending DA submission",
}, labels).With(labelsAndValues...)

// Sync metrics
m.SyncLag = prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "sync_lag_blocks",
Help: "Number of blocks behind the head",
}, labels).With(labelsAndValues...)

m.HeadersSynced = prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "headers_synced_total",
Help: "Total number of headers synced",
}, labels).With(labelsAndValues...)

m.DataSynced = prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "data_synced_total",
Help: "Total number of data blocks synced",
}, labels).With(labelsAndValues...)

m.BlocksApplied = prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "blocks_applied_total",
Help: "Total number of blocks applied to state",
}, labels).With(labelsAndValues...)

m.InvalidHeadersCount = prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "invalid_headers_total",
Help: "Total number of invalid headers rejected",
}, labels).With(labelsAndValues...)

// Block production metrics
m.BlockProductionTime = prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "block_production_duration_seconds",
Help: "Time taken to produce a block",
Buckets: []float64{.001, .005, .01, .025, .05, .1, .25, .5, 1},
}, labels).With(labelsAndValues...)

m.EmptyBlocksProduced = prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "empty_blocks_produced_total",
Help: "Total number of empty blocks produced",
}, labels).With(labelsAndValues...)

m.LazyBlocksProduced = prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "lazy_blocks_produced_total",
Help: "Total number of blocks produced in lazy mode",
}, labels).With(labelsAndValues...)

m.NormalBlocksProduced = prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "normal_blocks_produced_total",
Help: "Total number of blocks produced in normal mode",
}, labels).With(labelsAndValues...)

m.TxsPerBlock = prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "txs_per_block",
Help: "Number of transactions per block",
Buckets: []float64{0, 1, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000},
}, labels).With(labelsAndValues...)

// State transition metrics
m.InvalidTransitions = prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "invalid_state_transitions_total",
Help: "Total number of invalid state transitions attempted",
}, labels).With(labelsAndValues...)

// Initialize state transition counters
transitions := []string{"pending_to_submitted", "submitted_to_included", "included_to_finalized"}
for _, transition := range transitions {
m.StateTransitions[transition] = prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "state_transitions_total",
Help: "Total number of state transitions",
ConstLabels: map[string]string{
"transition": transition,
},
}, labels).With(labelsAndValues...)
}

// DA Submitter metrics
m.DASubmitterPendingBlobs = prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Expand Down Expand Up @@ -435,63 +232,28 @@ func NopMetrics() *Metrics {
BlockSizeBytes: discard.NewGauge(),
TotalTxs: discard.NewGauge(),
CommittedHeight: discard.NewGauge(),
TxsPerBlock: discard.NewHistogram(),

// Extended metrics
ChannelBufferUsage: make(map[string]metrics.Gauge),
ErrorsByType: make(map[string]metrics.Counter),
OperationDuration: make(map[string]metrics.Histogram),
StateTransitions: make(map[string]metrics.Counter),
DroppedSignals: discard.NewCounter(),
RecoverableErrors: discard.NewCounter(),
NonRecoverableErrors: discard.NewCounter(),
GoroutineCount: discard.NewGauge(),
DASubmissionAttempts: discard.NewCounter(),
DASubmissionSuccesses: discard.NewCounter(),
DASubmissionFailures: discard.NewCounter(),
DARetrievalAttempts: discard.NewCounter(),
DARetrievalSuccesses: discard.NewCounter(),
DARetrievalFailures: discard.NewCounter(),
DAInclusionHeight: discard.NewGauge(),
PendingHeadersCount: discard.NewGauge(),
PendingDataCount: discard.NewGauge(),
SyncLag: discard.NewGauge(),
HeadersSynced: discard.NewCounter(),
DataSynced: discard.NewCounter(),
BlocksApplied: discard.NewCounter(),
InvalidHeadersCount: discard.NewCounter(),
BlockProductionTime: discard.NewHistogram(),
EmptyBlocksProduced: discard.NewCounter(),
LazyBlocksProduced: discard.NewCounter(),
NormalBlocksProduced: discard.NewCounter(),
TxsPerBlock: discard.NewHistogram(),
InvalidTransitions: discard.NewCounter(),
DASubmitterFailures: make(map[DASubmitterFailureReason]metrics.Counter),
DASubmitterLastFailure: make(map[DASubmitterFailureReason]metrics.Gauge),
DASubmitterPendingBlobs: discard.NewGauge(),
DASubmitterResends: discard.NewCounter(),
}

// Initialize maps with no-op metrics
channelNames := []string{"height_in", "header_store", "data_store", "retrieve", "da_includer", "tx_notify"}
for _, name := range channelNames {
m.ChannelBufferUsage[name] = discard.NewGauge()
}

errorTypes := []string{"block_production", "da_submission", "sync", "validation", "state_update"}
for _, errType := range errorTypes {
m.ErrorsByType[errType] = discard.NewCounter()
}

operations := []string{"block_production", "da_submission", "block_retrieval", "block_validation", "state_update"}
for _, op := range operations {
m.OperationDuration[op] = discard.NewHistogram()
}

transitions := []string{"pending_to_submitted", "submitted_to_included", "included_to_finalized"}
for _, transition := range transitions {
m.StateTransitions[transition] = discard.NewCounter()
}

// Initialize DA submitter failure maps with no-op metrics
for _, reason := range AllDASubmitterFailureReasons() {
m.DASubmitterFailures[reason] = discard.NewCounter()
Expand Down
Loading
Loading