diff --git a/spartan/metrics/grafana/alerts/rules.yaml b/spartan/metrics/grafana/alerts/rules.yaml index cbea5beb931a..bfe88fa23d11 100644 --- a/spartan/metrics/grafana/alerts/rules.yaml +++ b/spartan/metrics/grafana/alerts/rules.yaml @@ -273,7 +273,7 @@ groups: datasourceUid: spartan-metrics-prometheus model: editorMode: code - expr: sum by (k8s_namespace_name, aztec_error_type) (increase(aztec_sequencer_block_proposal_precheck_failed_count{k8s_namespace_name=~".*(fisherman|mainnet).*"}[$__rate_interval])) + expr: sum by (k8s_namespace_name, aztec_error_type) (increase(aztec_sequencer_checkpoint_precheck_failed_count{k8s_namespace_name=~".*(fisherman|mainnet).*"}[$__rate_interval])) instant: true intervalMs: 60000 legendFormat: __auto diff --git a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts index e7b84bb9cddc..3336ba871168 100644 --- a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts +++ b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts @@ -129,7 +129,7 @@ export class CheckpointProposalJob implements Traceable { await Promise.all(votesPromises); if (checkpoint) { - this.metrics.recordBlockProposalSuccess(); + this.metrics.recordCheckpointProposalSuccess(); } // Do not post anything to L1 if we are fishermen, but do perform L1 fee analysis @@ -221,6 +221,7 @@ export class CheckpointProposalJob implements Traceable { let blocksInCheckpoint: L2Block[] = []; let blockPendingBroadcast: { block: L2Block; txs: Tx[] } | undefined = undefined; + const checkpointBuildTimer = new Timer(); try { // Main loop: build blocks for the checkpoint @@ -253,6 +254,14 @@ export class CheckpointProposalJob implements Traceable { this.setStateFn(SequencerState.ASSEMBLING_CHECKPOINT, this.slot); const checkpoint = await checkpointBuilder.completeCheckpoint(); + // Record checkpoint-level build metrics + this.metrics.recordCheckpointBuild( + checkpointBuildTimer.ms(), + blocksInCheckpoint.length, + checkpoint.getStats().txCount, + Number(checkpoint.header.totalManaUsed.toBigInt()), + ); + // Do not collect attestations nor publish to L1 in fisherman mode if (this.config.fishermanMode) { this.log.info( @@ -826,7 +835,7 @@ export class CheckpointProposalJob implements Traceable { slot: this.slot, feeAnalysisId: feeAnalysis?.id, }); - this.metrics.recordBlockProposalFailed('block_build_failed'); + this.metrics.recordCheckpointProposalFailed('block_build_failed'); } this.publisher.clearPendingRequests(); diff --git a/yarn-project/sequencer-client/src/sequencer/metrics.ts b/yarn-project/sequencer-client/src/sequencer/metrics.ts index 2758074d3bdd..a6d774d8be3c 100644 --- a/yarn-project/sequencer-client/src/sequencer/metrics.ts +++ b/yarn-project/sequencer-client/src/sequencer/metrics.ts @@ -18,7 +18,6 @@ import { type Hex, formatUnits } from 'viem'; import type { SequencerState } from './utils.js'; -// TODO(palla/mbps): Review all metrics and add any missing ones per checkpoint export class SequencerMetrics { public readonly tracer: Tracer; private meter: Meter; @@ -40,11 +39,16 @@ export class SequencerMetrics { private filledSlots: UpDownCounter; private blockProposalFailed: UpDownCounter; - private blockProposalSuccess: UpDownCounter; - private blockProposalPrecheckFailed: UpDownCounter; + private checkpointProposalSuccess: UpDownCounter; + private checkpointPrecheckFailed: UpDownCounter; + private checkpointProposalFailed: UpDownCounter; private checkpointSuccess: UpDownCounter; private slashingAttempts: UpDownCounter; private checkpointAttestationDelay: Histogram; + private checkpointBuildDuration: Histogram; + private checkpointBlockCount: Gauge; + private checkpointTxCount: Gauge; + private checkpointTotalMana: Gauge; // Fisherman fee analysis metrics private fishermanWouldBeIncluded: UpDownCounter; @@ -83,7 +87,7 @@ export class SequencerMetrics { this.checkpointAttestationDelay = this.meter.createHistogram(Metrics.SEQUENCER_CHECKPOINT_ATTESTATION_DELAY); - this.rewards = this.meter.createGauge(Metrics.SEQUENCER_CURRENT_BLOCK_REWARDS); + this.rewards = this.meter.createGauge(Metrics.SEQUENCER_CURRENT_SLOT_REWARDS); this.slots = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_SLOT_COUNT); @@ -106,16 +110,16 @@ export class SequencerMetrics { Metrics.SEQUENCER_BLOCK_PROPOSAL_FAILED_COUNT, ); - this.blockProposalSuccess = createUpDownCounterWithDefault( + this.checkpointProposalSuccess = createUpDownCounterWithDefault( this.meter, - Metrics.SEQUENCER_BLOCK_PROPOSAL_SUCCESS_COUNT, + Metrics.SEQUENCER_CHECKPOINT_PROPOSAL_SUCCESS_COUNT, ); this.checkpointSuccess = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_CHECKPOINT_SUCCESS_COUNT); - this.blockProposalPrecheckFailed = createUpDownCounterWithDefault( + this.checkpointPrecheckFailed = createUpDownCounterWithDefault( this.meter, - Metrics.SEQUENCER_BLOCK_PROPOSAL_PRECHECK_FAILED_COUNT, + Metrics.SEQUENCER_CHECKPOINT_PRECHECK_FAILED_COUNT, { [Attributes.ERROR_TYPE]: [ 'slot_already_taken', @@ -126,6 +130,16 @@ export class SequencerMetrics { }, ); + this.checkpointProposalFailed = createUpDownCounterWithDefault( + this.meter, + Metrics.SEQUENCER_CHECKPOINT_PROPOSAL_FAILED_COUNT, + ); + + this.checkpointBuildDuration = this.meter.createHistogram(Metrics.SEQUENCER_CHECKPOINT_BUILD_DURATION); + this.checkpointBlockCount = this.meter.createGauge(Metrics.SEQUENCER_CHECKPOINT_BLOCK_COUNT); + this.checkpointTxCount = this.meter.createGauge(Metrics.SEQUENCER_CHECKPOINT_TX_COUNT); + this.checkpointTotalMana = this.meter.createGauge(Metrics.SEQUENCER_CHECKPOINT_TOTAL_MANA); + this.slashingAttempts = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_SLASHING_ATTEMPTS_COUNT); // Fisherman fee analysis metrics @@ -258,18 +272,30 @@ export class SequencerMetrics { }); } - recordBlockProposalSuccess() { - this.blockProposalSuccess.add(1); + recordCheckpointProposalSuccess() { + this.checkpointProposalSuccess.add(1); } - recordBlockProposalPrecheckFailed( + recordCheckpointPrecheckFailed( checkType: 'slot_already_taken' | 'rollup_contract_check_failed' | 'slot_mismatch' | 'block_number_mismatch', ) { - this.blockProposalPrecheckFailed.add(1, { - [Attributes.ERROR_TYPE]: checkType, + this.checkpointPrecheckFailed.add(1, { [Attributes.ERROR_TYPE]: checkType }); + } + + recordCheckpointProposalFailed(reason?: string) { + this.checkpointProposalFailed.add(1, { + ...(reason && { [Attributes.ERROR_TYPE]: reason }), }); } + /** Records aggregate metrics for a completed checkpoint build. */ + recordCheckpointBuild(durationMs: number, blockCount: number, txCount: number, totalMana: number) { + this.checkpointBuildDuration.record(Math.ceil(durationMs)); + this.checkpointBlockCount.record(blockCount); + this.checkpointTxCount.record(txCount); + this.checkpointTotalMana.record(totalMana); + } + recordSlashingAttempt(actionCount: number) { this.slashingAttempts.add(actionCount); } diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.ts index de3dd62cd897..0f981bcda8dc 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.ts @@ -306,7 +306,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter TypedEventEmitter TypedEventEmitter TypedEventEmitter