From 1b49a3db775e955fcb963c7bdf0acda25fa968bd Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Mon, 14 Apr 2025 07:45:20 +0000 Subject: [PATCH 1/4] feat: track rewards --- yarn-project/prover-node/src/metrics.ts | 93 +++++++++++++++---- .../prover-node/src/prover-node-publisher.ts | 4 + yarn-project/prover-node/src/prover-node.ts | 12 ++- .../src/publisher/sequencer-publisher.ts | 4 + .../sequencer-client/src/sequencer/metrics.ts | 75 +++++++++++---- .../src/sequencer/sequencer.ts | 9 ++ .../telemetry-client/src/attributes.ts | 2 + yarn-project/telemetry-client/src/index.ts | 1 + yarn-project/telemetry-client/src/metrics.ts | 3 + 9 files changed, 170 insertions(+), 33 deletions(-) diff --git a/yarn-project/prover-node/src/metrics.ts b/yarn-project/prover-node/src/metrics.ts index fe5a21bda55f..478d41d9d70d 100644 --- a/yarn-project/prover-node/src/metrics.ts +++ b/yarn-project/prover-node/src/metrics.ts @@ -1,16 +1,22 @@ +import type { RollupContract } from '@aztec/ethereum'; +import type { EthAddress } from '@aztec/foundation/eth-address'; import { createLogger } from '@aztec/foundation/log'; import type { L1PublishProofStats, L1PublishStats } from '@aztec/stdlib/stats'; import { Attributes, + type BatchObservableResult, type Gauge, type Histogram, + type Meter, Metrics, + type ObservableGauge, + type ObservableUpDownCounter, type TelemetryClient, type UpDownCounter, ValueType, } from '@aztec/telemetry-client'; -import { formatEther } from 'viem'; +import { formatEther, formatUnits } from 'viem'; export class ProverNodeMetrics { proverEpochExecutionDuration: Histogram; @@ -30,78 +36,88 @@ export class ProverNodeMetrics { private senderBalance: Gauge; + private rewards: ObservableGauge; + private accumulatedRewards: ObservableUpDownCounter; + private prevEpoch = -1n; + private proofSubmissionWindow = 0n; + + private meter: Meter; + constructor( public readonly client: TelemetryClient, + private coinbase: EthAddress, + private rollup: RollupContract, name = 'ProverNode', private logger = createLogger('prover-node:publisher:metrics'), ) { - const meter = client.getMeter(name); - this.proverEpochExecutionDuration = meter.createHistogram(Metrics.PROVER_NODE_EXECUTION_DURATION, { + this.meter = client.getMeter(name); + + this.proverEpochExecutionDuration = this.meter.createHistogram(Metrics.PROVER_NODE_EXECUTION_DURATION, { description: 'Duration of execution of an epoch by the prover', unit: 'ms', valueType: ValueType.INT, }); - this.provingJobDuration = meter.createHistogram(Metrics.PROVER_NODE_JOB_DURATION, { + this.provingJobDuration = this.meter.createHistogram(Metrics.PROVER_NODE_JOB_DURATION, { description: 'Duration of proving job', unit: 's', valueType: ValueType.DOUBLE, }); - this.provingJobBlocks = meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS, { + this.provingJobBlocks = this.meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS, { description: 'Number of blocks in a proven epoch', valueType: ValueType.INT, }); - this.provingJobTransactions = meter.createGauge(Metrics.PROVER_NODE_JOB_TRANSACTIONS, { + this.provingJobTransactions = this.meter.createGauge(Metrics.PROVER_NODE_JOB_TRANSACTIONS, { description: 'Number of transactions in a proven epoch', valueType: ValueType.INT, }); - this.gasPrice = meter.createHistogram(Metrics.L1_PUBLISHER_GAS_PRICE, { + this.gasPrice = this.meter.createHistogram(Metrics.L1_PUBLISHER_GAS_PRICE, { description: 'The gas price used for transactions', unit: 'gwei', valueType: ValueType.DOUBLE, }); - this.txCount = meter.createUpDownCounter(Metrics.L1_PUBLISHER_TX_COUNT, { + this.txCount = this.meter.createUpDownCounter(Metrics.L1_PUBLISHER_TX_COUNT, { description: 'The number of transactions processed', }); - this.txDuration = meter.createHistogram(Metrics.L1_PUBLISHER_TX_DURATION, { + this.txDuration = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_DURATION, { description: 'The duration of transaction processing', unit: 'ms', valueType: ValueType.INT, }); - this.txGas = meter.createHistogram(Metrics.L1_PUBLISHER_TX_GAS, { + this.txGas = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_GAS, { description: 'The gas consumed by transactions', unit: 'gas', valueType: ValueType.INT, }); - this.txCalldataSize = meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_SIZE, { + this.txCalldataSize = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_SIZE, { description: 'The size of the calldata in transactions', unit: 'By', valueType: ValueType.INT, }); - this.txCalldataGas = meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_GAS, { + this.txCalldataGas = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_GAS, { description: 'The gas consumed by the calldata in transactions', unit: 'gas', valueType: ValueType.INT, }); - this.txBlobDataGasUsed = meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_USED, { + this.txBlobDataGasUsed = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_USED, { description: 'The amount of blob gas used in transactions', unit: 'gas', valueType: ValueType.INT, }); - this.txBlobDataGasCost = meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_COST, { + this.txBlobDataGasCost = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_COST, { description: 'The gas cost of blobs in transactions', unit: 'gwei', valueType: ValueType.INT, }); - this.txTotalFee = meter.createHistogram(Metrics.L1_PUBLISHER_TX_TOTAL_FEE, { + this.txTotalFee = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_TOTAL_FEE, { description: 'How much L1 tx costs', unit: 'gwei', valueType: ValueType.DOUBLE, @@ -112,13 +128,58 @@ export class ProverNodeMetrics { }, }); - this.senderBalance = meter.createGauge(Metrics.L1_PUBLISHER_BALANCE, { + this.senderBalance = this.meter.createGauge(Metrics.L1_PUBLISHER_BALANCE, { unit: 'eth', description: 'The balance of the sender address', valueType: ValueType.DOUBLE, }); + + this.rewards = this.meter.createObservableGauge(Metrics.L1_REWARDS_BALANCE, { + valueType: ValueType.DOUBLE, + description: 'The rewards earned', + }); + + this.accumulatedRewards = this.meter.createObservableUpDownCounter(Metrics.L1_REWARDS_BALANCE_SUM, { + valueType: ValueType.DOUBLE, + description: 'The rewards earned (total)', + }); + } + + public async start() { + this.proofSubmissionWindow = await this.rollup.getProofSubmissionWindow(); + this.meter.addBatchObservableCallback(this.observe, [this.rewards, this.accumulatedRewards]); } + public stop() { + this.meter.removeBatchObservableCallback(this.observe, [this.rewards, this.accumulatedRewards]); + } + + private observe = async (observer: BatchObservableResult): Promise => { + const slot = await this.rollup.getSlotNumber(); + + // look at the prev epoch so that we get an accurate value, after proof submission window has closed + if (slot > this.proofSubmissionWindow) { + const closedEpoch = await this.rollup.getEpochNumberForSlotNumber(slot - this.proofSubmissionWindow); + const rewards = await this.rollup.getSpecificProverRewardsForEpoch(closedEpoch, this.coinbase); + + const fmt = parseFloat(formatUnits(rewards, 18)); + + observer.observe(this.rewards, fmt, { + [Attributes.L1_SENDER]: this.coinbase.toString(), + [Attributes.NODE_ROLE]: 'prover', + }); + + // only accumulate once per epoch + if (closedEpoch > this.prevEpoch) { + this.prevEpoch = closedEpoch; + observer.observe(this.accumulatedRewards, fmt, { + [Attributes.L1_SENDER]: this.coinbase.toString(), + [Attributes.NODE_ROLE]: 'prover', + }); + } + } + }; + recordFailedTx() { this.txCount.add(1, { [Attributes.L1_TX_TYPE]: 'submitProof', diff --git a/yarn-project/prover-node/src/prover-node-publisher.ts b/yarn-project/prover-node/src/prover-node-publisher.ts index 260c38e79fa3..e45eac5986a3 100644 --- a/yarn-project/prover-node/src/prover-node-publisher.ts +++ b/yarn-project/prover-node/src/prover-node-publisher.ts @@ -66,6 +66,10 @@ export class ProverNodePublisher { this.l1TxUtils = deps.l1TxUtils; } + public getRollupContract() { + return this.rollupContract; + } + /** * Calling `interrupt` will cause any in progress call to `publishRollup` to return `false` asap. * Be warned, the call may return false even if the tx subsequently gets successfully mined. diff --git a/yarn-project/prover-node/src/prover-node.ts b/yarn-project/prover-node/src/prover-node.ts index f9bf79118d96..f8349d9d50ec 100644 --- a/yarn-project/prover-node/src/prover-node.ts +++ b/yarn-project/prover-node/src/prover-node.ts @@ -1,5 +1,7 @@ +import { RollupContract, getPublicClient } from '@aztec/ethereum'; import { compact } from '@aztec/foundation/collection'; import { memoize } from '@aztec/foundation/decorators'; +import { EthAddress } from '@aztec/foundation/eth-address'; import { createLogger } from '@aztec/foundation/log'; import { RunningPromise } from '@aztec/foundation/running-promise'; import { DateProvider } from '@aztec/foundation/timer'; @@ -92,7 +94,13 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable ...compact(options), }; - this.metrics = new ProverNodeMetrics(telemetryClient, 'ProverNode'); + this.metrics = new ProverNodeMetrics( + telemetryClient, + + EthAddress.fromField(this.prover.getProverId()), + this.publisher.getRollupContract(), + 'ProverNode', + ); this.tracer = telemetryClient.getTracer('ProverNode'); this.txFetcher = new RunningPromise(() => this.checkForTxs(), this.log, this.options.txGatheringIntervalMs); } @@ -143,6 +151,7 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable this.txFetcher.start(); this.epochsMonitor.start(this); this.l1Metrics.start(); + this.metrics.start(); this.log.info(`Started Prover Node with prover id ${this.prover.getProverId().toString()}`, this.options); } @@ -160,6 +169,7 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable await this.worldState.stop(); await tryStop(this.coordination); this.l1Metrics.stop(); + this.metrics.stop(); await this.telemetryClient.stop(); this.log.info('Stopped ProverNode'); } diff --git a/yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts b/yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts index 5bbbd6245b8f..44ef0796803f 100644 --- a/yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts +++ b/yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts @@ -136,6 +136,10 @@ export class SequencerPublisher { this.slashingProposerContract = deps.slashingProposerContract; } + public getRollupContract(): RollupContract { + return this.rollupContract; + } + public registerSlashPayloadGetter(callback: GetSlashPayloadCallBack) { this.getSlashPayload = callback; } diff --git a/yarn-project/sequencer-client/src/sequencer/metrics.ts b/yarn-project/sequencer-client/src/sequencer/metrics.ts index 17d7e0aa07a1..0d6f03058fcc 100644 --- a/yarn-project/sequencer-client/src/sequencer/metrics.ts +++ b/yarn-project/sequencer-client/src/sequencer/metrics.ts @@ -1,18 +1,26 @@ +import type { EthAddress } from '@aztec/aztec.js'; +import type { RollupContract } from '@aztec/ethereum'; import { Attributes, + type BatchObservableResult, type Gauge, type Histogram, + type Meter, Metrics, + type ObservableGauge, type TelemetryClient, type Tracer, type UpDownCounter, ValueType, } from '@aztec/telemetry-client'; +import { formatUnits } from 'viem'; + import { type SequencerState, type SequencerStateCallback, sequencerStateToNumber } from './utils.js'; export class SequencerMetrics { public readonly tracer: Tracer; + private meter: Meter; private blockCounter: UpDownCounter; private blockBuildDuration: Histogram; @@ -24,32 +32,43 @@ export class SequencerMetrics { private timeToCollectAttestations: Gauge; - constructor(client: TelemetryClient, getState: SequencerStateCallback, name = 'Sequencer') { - const meter = client.getMeter(name); + private rewards: ObservableGauge; + + constructor( + client: TelemetryClient, + getState: SequencerStateCallback, + private coinbase: EthAddress, + private rollup: RollupContract, + name = 'Sequencer', + ) { + this.meter = client.getMeter(name); this.tracer = client.getTracer(name); - this.blockCounter = meter.createUpDownCounter(Metrics.SEQUENCER_BLOCK_COUNT); + this.blockCounter = this.meter.createUpDownCounter(Metrics.SEQUENCER_BLOCK_COUNT); - this.blockBuildDuration = meter.createHistogram(Metrics.SEQUENCER_BLOCK_BUILD_DURATION, { + this.blockBuildDuration = this.meter.createHistogram(Metrics.SEQUENCER_BLOCK_BUILD_DURATION, { unit: 'ms', description: 'Duration to build a block', valueType: ValueType.INT, }); - this.blockBuildManaPerSecond = meter.createGauge(Metrics.SEQUENCER_BLOCK_BUILD_MANA_PER_SECOND, { + this.blockBuildManaPerSecond = this.meter.createGauge(Metrics.SEQUENCER_BLOCK_BUILD_MANA_PER_SECOND, { unit: 'mana/s', description: 'Mana per second when building a block', valueType: ValueType.INT, }); - this.stateTransitionBufferDuration = meter.createHistogram(Metrics.SEQUENCER_STATE_TRANSITION_BUFFER_DURATION, { - unit: 'ms', - description: - 'The time difference between when the sequencer needed to transition to a new state and when it actually did.', - valueType: ValueType.INT, - }); + this.stateTransitionBufferDuration = this.meter.createHistogram( + Metrics.SEQUENCER_STATE_TRANSITION_BUFFER_DURATION, + { + unit: 'ms', + description: + 'The time difference between when the sequencer needed to transition to a new state and when it actually did.', + valueType: ValueType.INT, + }, + ); - const currentState = meter.createObservableGauge(Metrics.SEQUENCER_CURRENT_STATE, { + const currentState = this.meter.createObservableGauge(Metrics.SEQUENCER_CURRENT_STATE, { description: 'Current state of the sequencer', }); @@ -57,22 +76,22 @@ export class SequencerMetrics { observer.observe(sequencerStateToNumber(getState())); }); - this.currentBlockNumber = meter.createGauge(Metrics.SEQUENCER_CURRENT_BLOCK_NUMBER, { + this.currentBlockNumber = this.meter.createGauge(Metrics.SEQUENCER_CURRENT_BLOCK_NUMBER, { description: 'Current block number', valueType: ValueType.INT, }); - this.currentBlockSize = meter.createGauge(Metrics.SEQUENCER_CURRENT_BLOCK_SIZE, { + this.currentBlockSize = this.meter.createGauge(Metrics.SEQUENCER_CURRENT_BLOCK_SIZE, { description: 'Current block size', valueType: ValueType.INT, }); - this.timeToCollectAttestations = meter.createGauge(Metrics.SEQUENCER_TIME_TO_COLLECT_ATTESTATIONS, { + this.timeToCollectAttestations = this.meter.createGauge(Metrics.SEQUENCER_TIME_TO_COLLECT_ATTESTATIONS, { description: 'The time spent collecting attestations from committee members', valueType: ValueType.INT, }); - this.blockBuilderInsertions = meter.createHistogram(Metrics.SEQUENCER_BLOCK_BUILD_INSERTION_TIME, { + this.blockBuilderInsertions = this.meter.createHistogram(Metrics.SEQUENCER_BLOCK_BUILD_INSERTION_TIME, { description: 'Timer for tree insertions performed by the block builder', unit: 'us', valueType: ValueType.INT, @@ -89,8 +108,32 @@ export class SequencerMetrics { this.blockCounter.add(0, { [Attributes.STATUS]: 'built', }); + + this.rewards = this.meter.createObservableGauge(Metrics.L1_REWARDS_BALANCE, { + valueType: ValueType.DOUBLE, + description: 'The rewards earned', + }); } + public start() { + this.meter.addBatchObservableCallback(this.observe, [this.rewards]); + } + + public stop() { + this.meter.removeBatchObservableCallback(this.observe, [this.rewards]); + } + + private observe = async (observer: BatchObservableResult): Promise => { + let rewards = 0n; + rewards = await this.rollup.getSequencerRewards(this.coinbase); + + const fmt = parseFloat(formatUnits(rewards, 18)); + observer.observe(this.rewards, fmt, { + [Attributes.L1_SENDER]: this.coinbase.toString(), + [Attributes.NODE_ROLE]: 'sequencer', + }); + }; + startCollectingAttestationsTimer(): () => void { const startTime = Date.now(); const stop = () => { diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.ts index 0a632fb37afd..b18b9a58ec3b 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.ts @@ -36,6 +36,7 @@ import { import { Attributes, L1Metrics, + RewardMetrics, type TelemetryClient, type Tracer, getTelemetryClient, @@ -80,6 +81,7 @@ export class Sequencer { private maxBlockGas: Gas = new Gas(100e9, 100e9); private metrics: SequencerMetrics; private l1Metrics: L1Metrics; + private rewardsMetrics: RewardMetrics; private isFlushing: boolean = false; /** The maximum number of seconds that the sequencer can be into a slot to transition to a particular state. */ @@ -110,6 +112,13 @@ export class Sequencer { publisher.getSenderAddress(), ]); + this.rewardsMetrics = new RewardMetrics( + telemetry.getMeter('ValidatorL1Metrics'), + this._coinbase, + 'sequencer', + this.publisher.getRollupContract(), + ); + // Register the block builder with the validator client for re-execution this.validatorClient?.registerBlockBuilder(this.buildBlock.bind(this)); diff --git a/yarn-project/telemetry-client/src/attributes.ts b/yarn-project/telemetry-client/src/attributes.ts index 8d1f0147c128..ef5761b260c2 100644 --- a/yarn-project/telemetry-client/src/attributes.ts +++ b/yarn-project/telemetry-client/src/attributes.ts @@ -64,6 +64,8 @@ export const ERROR_TYPE = 'aztec.error_type'; export const L1_TX_TYPE = 'aztec.l1.tx_type'; /** The L1 address of the entity that sent a transaction to L1 */ export const L1_SENDER = 'aztec.l1.sender'; +/** The role of node on the network (e.g. sequencer, prover) */ +export const NODE_ROLE = 'aztec.node_role'; /** The phase of the transaction */ export const TX_PHASE_NAME = 'aztec.tx.phase_name'; /** The reason for disconnecting a peer */ diff --git a/yarn-project/telemetry-client/src/index.ts b/yarn-project/telemetry-client/src/index.ts index db9f09d8edfd..cb955ca65981 100644 --- a/yarn-project/telemetry-client/src/index.ts +++ b/yarn-project/telemetry-client/src/index.ts @@ -5,6 +5,7 @@ export * from './with_tracer.js'; export * from './prom_otel_adapter.js'; export * from './lmdb_metrics.js'; export * from './l1_metrics.js'; +export * from './reward_metrics.js'; export * from './wrappers/index.js'; export * from './start.js'; export * from './otel_propagation.js'; diff --git a/yarn-project/telemetry-client/src/metrics.ts b/yarn-project/telemetry-client/src/metrics.ts index a10903f6fb09..a60a0496d045 100644 --- a/yarn-project/telemetry-client/src/metrics.ts +++ b/yarn-project/telemetry-client/src/metrics.ts @@ -83,6 +83,9 @@ export const L1_BALANCE_ETH = 'aztec.l1.balance'; export const L1_GAS_PRICE_WEI = 'aztec.l1.gas_price'; export const L1_BLOB_BASE_FEE_WEI = 'aztec.l1.blob_base_fee'; +export const L1_REWARDS_BALANCE = 'aztec.l1.rewards_balance'; +export const L1_REWARDS_BALANCE_SUM = 'aztec.l1.rewards_balance_sum'; + export const PEER_MANAGER_GOODBYES_SENT = 'aztec.peer_manager.goodbyes_sent'; export const PEER_MANAGER_GOODBYES_RECEIVED = 'aztec.peer_manager.goodbyes_received'; export const PEER_MANAGER_PEER_COUNT = 'aztec.peer_manager.peer_count'; From cd744f9a3ddee77f6aa44ca43d0b5765494b9b93 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Mon, 14 Apr 2025 08:20:40 +0000 Subject: [PATCH 2/4] feat: track attestations --- .../aztec/src/cli/cmds/start_prover_node.ts | 2 +- .../src/e2e_prover/e2e_prover_test.ts | 2 +- yarn-project/end-to-end/src/fixtures/utils.ts | 2 +- .../src/job/epoch-proving-job.test.ts | 9 +- .../prover-node/src/job/epoch-proving-job.ts | 6 +- yarn-project/prover-node/src/metrics.ts | 169 ++++++++++-------- .../prover-node/src/prover-node-publisher.ts | 6 +- yarn-project/prover-node/src/prover-node.ts | 30 ++-- .../sequencer-client/src/sequencer/metrics.ts | 3 +- .../src/sequencer/sequencer.ts | 19 +- .../telemetry-client/src/attributes.ts | 2 - yarn-project/telemetry-client/src/index.ts | 1 - yarn-project/telemetry-client/src/metrics.ts | 8 +- yarn-project/validator-client/src/metrics.ts | 22 +++ .../validator-client/src/validator.ts | 8 + .../world-state/src/synchronizer/errors.ts | 5 + .../server_world_state_synchronizer.ts | 13 +- 17 files changed, 185 insertions(+), 122 deletions(-) create mode 100644 yarn-project/world-state/src/synchronizer/errors.ts diff --git a/yarn-project/aztec/src/cli/cmds/start_prover_node.ts b/yarn-project/aztec/src/cli/cmds/start_prover_node.ts index 01d4163b0bc8..3c46e515e3a7 100644 --- a/yarn-project/aztec/src/cli/cmds/start_prover_node.ts +++ b/yarn-project/aztec/src/cli/cmds/start_prover_node.ts @@ -120,6 +120,6 @@ export async function startProverNode( signalHandlers.push(proverNode.stop.bind(proverNode)); - proverNode.start(); + await proverNode.start(); return { config: proverConfig }; } diff --git a/yarn-project/end-to-end/src/e2e_prover/e2e_prover_test.ts b/yarn-project/end-to-end/src/e2e_prover/e2e_prover_test.ts index 57b66f9a7bd5..912f05008313 100644 --- a/yarn-project/end-to-end/src/e2e_prover/e2e_prover_test.ts +++ b/yarn-project/end-to-end/src/e2e_prover/e2e_prover_test.ts @@ -308,7 +308,7 @@ export class FullProverTest { }, { prefilledPublicData }, ); - this.proverNode.start(); + await this.proverNode.start(); this.logger.warn(`Proofs are now enabled`); return this; diff --git a/yarn-project/end-to-end/src/fixtures/utils.ts b/yarn-project/end-to-end/src/fixtures/utils.ts index 2220cf3fbaa2..fd13d82f9a49 100644 --- a/yarn-project/end-to-end/src/fixtures/utils.ts +++ b/yarn-project/end-to-end/src/fixtures/utils.ts @@ -828,7 +828,7 @@ export async function createAndSyncProverNode( { prefilledPublicData }, ); getLogger().info(`Created and synced prover node`, { publisherAddress: l1TxUtils.walletClient.account.address }); - proverNode.start(); + await proverNode.start(); return proverNode; } diff --git a/yarn-project/prover-node/src/job/epoch-proving-job.test.ts b/yarn-project/prover-node/src/job/epoch-proving-job.test.ts index 76dc3428b5ac..7248c4c02c47 100644 --- a/yarn-project/prover-node/src/job/epoch-proving-job.test.ts +++ b/yarn-project/prover-node/src/job/epoch-proving-job.test.ts @@ -14,7 +14,7 @@ import { getTelemetryClient } from '@aztec/telemetry-client'; import { type MockProxy, mock } from 'jest-mock-extended'; -import { ProverNodeMetrics } from '../metrics.js'; +import { ProverNodeJobMetrics } from '../metrics.js'; import type { ProverNodePublisher } from '../prover-node-publisher.js'; import { EpochProvingJob } from './epoch-proving-job.js'; @@ -26,7 +26,7 @@ describe('epoch-proving-job', () => { let l1ToL2MessageSource: MockProxy; let worldState: MockProxy; let publicProcessorFactory: MockProxy; - let metrics: ProverNodeMetrics; + let metrics: ProverNodeJobMetrics; // Created by a dependency let db: MockProxy; @@ -71,7 +71,10 @@ describe('epoch-proving-job', () => { publicProcessorFactory = mock(); db = mock(); publicProcessor = mock(); - metrics = new ProverNodeMetrics(getTelemetryClient()); + metrics = new ProverNodeJobMetrics( + getTelemetryClient().getMeter('EpochProvingJob'), + getTelemetryClient().getTracer('EpochProvingJob'), + ); publicInputs = RootRollupPublicInputs.random(); proof = Proof.empty(); diff --git a/yarn-project/prover-node/src/job/epoch-proving-job.ts b/yarn-project/prover-node/src/job/epoch-proving-job.ts index bc1a128b4808..5a75428b69fa 100644 --- a/yarn-project/prover-node/src/job/epoch-proving-job.ts +++ b/yarn-project/prover-node/src/job/epoch-proving-job.ts @@ -16,7 +16,7 @@ import { Attributes, type Traceable, type Tracer, trackSpan } from '@aztec/telem import * as crypto from 'node:crypto'; -import type { ProverNodeMetrics } from '../metrics.js'; +import type { ProverNodeJobMetrics } from '../metrics.js'; import type { ProverNodePublisher } from '../prover-node-publisher.js'; /** @@ -45,12 +45,12 @@ export class EpochProvingJob implements Traceable { private publisher: ProverNodePublisher, private l2BlockSource: L2BlockSource, private l1ToL2MessageSource: L1ToL2MessageSource, - private metrics: ProverNodeMetrics, + private metrics: ProverNodeJobMetrics, private deadline: Date | undefined, private config: { parallelBlockLimit: number } = { parallelBlockLimit: 32 }, ) { this.uuid = crypto.randomUUID(); - this.tracer = metrics.client.getTracer('EpochProvingJob'); + this.tracer = metrics.tracer; } public getId(): string { diff --git a/yarn-project/prover-node/src/metrics.ts b/yarn-project/prover-node/src/metrics.ts index 478d41d9d70d..1f8a6862c382 100644 --- a/yarn-project/prover-node/src/metrics.ts +++ b/yarn-project/prover-node/src/metrics.ts @@ -12,46 +12,24 @@ import { type ObservableGauge, type ObservableUpDownCounter, type TelemetryClient, + type Tracer, type UpDownCounter, ValueType, } from '@aztec/telemetry-client'; import { formatEther, formatUnits } from 'viem'; -export class ProverNodeMetrics { +export class ProverNodeJobMetrics { proverEpochExecutionDuration: Histogram; provingJobDuration: Histogram; provingJobBlocks: Gauge; provingJobTransactions: Gauge; - gasPrice: Histogram; - txCount: UpDownCounter; - txDuration: Histogram; - txGas: Histogram; - txCalldataSize: Histogram; - txCalldataGas: Histogram; - txBlobDataGasUsed: Histogram; - txBlobDataGasCost: Histogram; - txTotalFee: Histogram; - - private senderBalance: Gauge; - - private rewards: ObservableGauge; - private accumulatedRewards: ObservableUpDownCounter; - private prevEpoch = -1n; - private proofSubmissionWindow = 0n; - - private meter: Meter; - constructor( - public readonly client: TelemetryClient, - private coinbase: EthAddress, - private rollup: RollupContract, - name = 'ProverNode', + private meter: Meter, + public readonly tracer: Tracer, private logger = createLogger('prover-node:publisher:metrics'), ) { - this.meter = client.getMeter(name); - this.proverEpochExecutionDuration = this.meter.createHistogram(Metrics.PROVER_NODE_EXECUTION_DURATION, { description: 'Duration of execution of an epoch by the prover', unit: 'ms', @@ -70,6 +48,93 @@ export class ProverNodeMetrics { description: 'Number of transactions in a proven epoch', valueType: ValueType.INT, }); + } + + public recordProvingJob(executionTimeMs: number, totalTimeMs: number, numBlocks: number, numTxs: number) { + this.proverEpochExecutionDuration.record(Math.ceil(executionTimeMs)); + this.provingJobDuration.record(totalTimeMs / 1000); + this.provingJobBlocks.record(Math.floor(numBlocks)); + this.provingJobTransactions.record(Math.floor(numTxs)); + } +} + +export class ProverNodeRewardsMetrics { + private rewards: ObservableGauge; + private accumulatedRewards: ObservableUpDownCounter; + private prevEpoch = -1n; + private proofSubmissionWindow = 0n; + + constructor( + private meter: Meter, + private coinbase: EthAddress, + private rollup: RollupContract, + private logger = createLogger('prover-node:publisher:metrics'), + ) { + this.rewards = this.meter.createObservableGauge(Metrics.PROVER_NODE_REWARDS_PER_EPOCH, { + valueType: ValueType.DOUBLE, + description: 'The rewards earned', + }); + + this.accumulatedRewards = this.meter.createObservableUpDownCounter(Metrics.PROVER_NODE_REWARDS_TOTAL, { + valueType: ValueType.DOUBLE, + description: 'The rewards earned (total)', + }); + } + + public async start() { + this.proofSubmissionWindow = await this.rollup.getProofSubmissionWindow(); + this.meter.addBatchObservableCallback(this.observe, [this.rewards, this.accumulatedRewards]); + } + + public stop() { + this.meter.removeBatchObservableCallback(this.observe, [this.rewards, this.accumulatedRewards]); + } + + private observe = async (observer: BatchObservableResult): Promise => { + const slot = await this.rollup.getSlotNumber(); + + // look at the prev epoch so that we get an accurate value, after proof submission window has closed + if (slot > this.proofSubmissionWindow) { + const closedEpoch = await this.rollup.getEpochNumberForSlotNumber(slot - this.proofSubmissionWindow); + const rewards = await this.rollup.getSpecificProverRewardsForEpoch(closedEpoch, this.coinbase); + + const fmt = parseFloat(formatUnits(rewards, 18)); + + observer.observe(this.rewards, fmt, { + [Attributes.L1_SENDER]: this.coinbase.toString(), + }); + + // only accumulate once per epoch + if (closedEpoch > this.prevEpoch) { + this.prevEpoch = closedEpoch; + observer.observe(this.accumulatedRewards, fmt, { + [Attributes.L1_SENDER]: this.coinbase.toString(), + }); + } + } + }; +} + +export class ProverNodePublisherMetrics { + gasPrice: Histogram; + txCount: UpDownCounter; + txDuration: Histogram; + txGas: Histogram; + txCalldataSize: Histogram; + txCalldataGas: Histogram; + txBlobDataGasUsed: Histogram; + txBlobDataGasCost: Histogram; + txTotalFee: Histogram; + + private senderBalance: Gauge; + private meter: Meter; + + constructor( + public readonly client: TelemetryClient, + name = 'ProverNode', + private logger = createLogger('prover-node:publisher:metrics'), + ) { + this.meter = client.getMeter(name); this.gasPrice = this.meter.createHistogram(Metrics.L1_PUBLISHER_GAS_PRICE, { description: 'The gas price used for transactions', @@ -133,53 +198,8 @@ export class ProverNodeMetrics { description: 'The balance of the sender address', valueType: ValueType.DOUBLE, }); - - this.rewards = this.meter.createObservableGauge(Metrics.L1_REWARDS_BALANCE, { - valueType: ValueType.DOUBLE, - description: 'The rewards earned', - }); - - this.accumulatedRewards = this.meter.createObservableUpDownCounter(Metrics.L1_REWARDS_BALANCE_SUM, { - valueType: ValueType.DOUBLE, - description: 'The rewards earned (total)', - }); } - public async start() { - this.proofSubmissionWindow = await this.rollup.getProofSubmissionWindow(); - this.meter.addBatchObservableCallback(this.observe, [this.rewards, this.accumulatedRewards]); - } - - public stop() { - this.meter.removeBatchObservableCallback(this.observe, [this.rewards, this.accumulatedRewards]); - } - - private observe = async (observer: BatchObservableResult): Promise => { - const slot = await this.rollup.getSlotNumber(); - - // look at the prev epoch so that we get an accurate value, after proof submission window has closed - if (slot > this.proofSubmissionWindow) { - const closedEpoch = await this.rollup.getEpochNumberForSlotNumber(slot - this.proofSubmissionWindow); - const rewards = await this.rollup.getSpecificProverRewardsForEpoch(closedEpoch, this.coinbase); - - const fmt = parseFloat(formatUnits(rewards, 18)); - - observer.observe(this.rewards, fmt, { - [Attributes.L1_SENDER]: this.coinbase.toString(), - [Attributes.NODE_ROLE]: 'prover', - }); - - // only accumulate once per epoch - if (closedEpoch > this.prevEpoch) { - this.prevEpoch = closedEpoch; - observer.observe(this.accumulatedRewards, fmt, { - [Attributes.L1_SENDER]: this.coinbase.toString(), - [Attributes.NODE_ROLE]: 'prover', - }); - } - } - }; - recordFailedTx() { this.txCount.add(1, { [Attributes.L1_TX_TYPE]: 'submitProof', @@ -191,13 +211,6 @@ export class ProverNodeMetrics { this.recordTx(durationMs, stats); } - public recordProvingJob(executionTimeMs: number, totalTimeMs: number, numBlocks: number, numTxs: number) { - this.proverEpochExecutionDuration.record(Math.ceil(executionTimeMs)); - this.provingJobDuration.record(totalTimeMs / 1000); - this.provingJobBlocks.record(Math.floor(numBlocks)); - this.provingJobTransactions.record(Math.floor(numTxs)); - } - public recordSenderBalance(wei: bigint, senderAddress: string) { const eth = parseFloat(formatEther(wei, 'wei')); this.senderBalance.record(eth, { diff --git a/yarn-project/prover-node/src/prover-node-publisher.ts b/yarn-project/prover-node/src/prover-node-publisher.ts index e45eac5986a3..dc369f3abf56 100644 --- a/yarn-project/prover-node/src/prover-node-publisher.ts +++ b/yarn-project/prover-node/src/prover-node-publisher.ts @@ -17,7 +17,7 @@ import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-clien import { type Hex, type TransactionReceipt, encodeFunctionData } from 'viem'; -import { ProverNodeMetrics } from './metrics.js'; +import { ProverNodePublisherMetrics } from './metrics.js'; /** * Stats for a sent transaction. @@ -40,7 +40,7 @@ export class ProverNodePublisher { private interruptibleSleep = new InterruptibleSleep(); private sleepTimeMs: number; private interrupted = false; - private metrics: ProverNodeMetrics; + private metrics: ProverNodePublisherMetrics; protected log = createLogger('prover-node:l1-tx-publisher'); @@ -60,7 +60,7 @@ export class ProverNodePublisher { const telemetry = deps.telemetry ?? getTelemetryClient(); - this.metrics = new ProverNodeMetrics(telemetry, 'ProverNode'); + this.metrics = new ProverNodePublisherMetrics(telemetry, 'ProverNode'); this.rollupContract = deps.rollupContract; this.l1TxUtils = deps.l1TxUtils; diff --git a/yarn-project/prover-node/src/prover-node.ts b/yarn-project/prover-node/src/prover-node.ts index f8349d9d50ec..99387eb40448 100644 --- a/yarn-project/prover-node/src/prover-node.ts +++ b/yarn-project/prover-node/src/prover-node.ts @@ -1,4 +1,3 @@ -import { RollupContract, getPublicClient } from '@aztec/ethereum'; import { compact } from '@aztec/foundation/collection'; import { memoize } from '@aztec/foundation/decorators'; import { EthAddress } from '@aztec/foundation/eth-address'; @@ -35,7 +34,7 @@ import { } from '@aztec/telemetry-client'; import { EpochProvingJob, type EpochProvingJobState } from './job/epoch-proving-job.js'; -import { ProverNodeMetrics } from './metrics.js'; +import { ProverNodeJobMetrics, ProverNodeRewardsMetrics } from './metrics.js'; import type { EpochMonitor, EpochMonitorHandler } from './monitors/epoch-monitor.js'; import type { ProverNodePublisher } from './prover-node-publisher.js'; @@ -60,7 +59,8 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable private jobs: Map = new Map(); private options: ProverNodeOptions; - private metrics: ProverNodeMetrics; + private jobMetrics: ProverNodeJobMetrics; + private rewardsMetrics: ProverNodeRewardsMetrics; private l1Metrics: L1Metrics; private txFetcher: RunningPromise; @@ -94,14 +94,17 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable ...compact(options), }; - this.metrics = new ProverNodeMetrics( - telemetryClient, + const meter = telemetryClient.getMeter('ProverNode'); + this.tracer = telemetryClient.getTracer('ProverNode'); + + this.jobMetrics = new ProverNodeJobMetrics(meter, telemetryClient.getTracer('EpochProvingJob')); + this.rewardsMetrics = new ProverNodeRewardsMetrics( + meter, EthAddress.fromField(this.prover.getProverId()), this.publisher.getRollupContract(), - 'ProverNode', ); - this.tracer = telemetryClient.getTracer('ProverNode'); + this.txFetcher = new RunningPromise(() => this.checkForTxs(), this.log, this.options.txGatheringIntervalMs); } @@ -147,11 +150,11 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable * Starts the prover node so it periodically checks for unproven epochs in the unfinalised chain from L1 and * starts proving jobs for them. */ - start() { + async start() { this.txFetcher.start(); this.epochsMonitor.start(this); this.l1Metrics.start(); - this.metrics.start(); + await this.rewardsMetrics.start(); this.log.info(`Started Prover Node with prover id ${this.prover.getProverId().toString()}`, this.options); } @@ -169,14 +172,15 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable await this.worldState.stop(); await tryStop(this.coordination); this.l1Metrics.stop(); - this.metrics.stop(); + this.rewardsMetrics.stop(); await this.telemetryClient.stop(); this.log.info('Stopped ProverNode'); } /** Returns world state status. */ - public getWorldStateSyncStatus(): Promise { - return this.worldState.status().then(s => s.syncSummary); + public async getWorldStateSyncStatus(): Promise { + const { syncSummary } = await this.worldState.status(); + return syncSummary; } /** Returns archiver status. */ @@ -346,7 +350,7 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable this.publisher, this.l2BlockSource, this.l1ToL2MessageSource, - this.metrics, + this.jobMetrics, deadline, { parallelBlockLimit: this.options.maxParallelBlocksPerEpoch }, ); diff --git a/yarn-project/sequencer-client/src/sequencer/metrics.ts b/yarn-project/sequencer-client/src/sequencer/metrics.ts index 0d6f03058fcc..512c1712c12a 100644 --- a/yarn-project/sequencer-client/src/sequencer/metrics.ts +++ b/yarn-project/sequencer-client/src/sequencer/metrics.ts @@ -109,7 +109,7 @@ export class SequencerMetrics { [Attributes.STATUS]: 'built', }); - this.rewards = this.meter.createObservableGauge(Metrics.L1_REWARDS_BALANCE, { + this.rewards = this.meter.createObservableGauge(Metrics.SEQUENCER_CURRENT_BLOCK_REWARDS, { valueType: ValueType.DOUBLE, description: 'The rewards earned', }); @@ -130,7 +130,6 @@ export class SequencerMetrics { const fmt = parseFloat(formatUnits(rewards, 18)); observer.observe(this.rewards, fmt, { [Attributes.L1_SENDER]: this.coinbase.toString(), - [Attributes.NODE_ROLE]: 'sequencer', }); }; diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.ts index b18b9a58ec3b..0f22912379d8 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.ts @@ -36,7 +36,6 @@ import { import { Attributes, L1Metrics, - RewardMetrics, type TelemetryClient, type Tracer, getTelemetryClient, @@ -81,7 +80,6 @@ export class Sequencer { private maxBlockGas: Gas = new Gas(100e9, 100e9); private metrics: SequencerMetrics; private l1Metrics: L1Metrics; - private rewardsMetrics: RewardMetrics; private isFlushing: boolean = false; /** The maximum number of seconds that the sequencer can be into a slot to transition to a particular state. */ @@ -107,17 +105,16 @@ export class Sequencer { telemetry: TelemetryClient = getTelemetryClient(), protected log = createLogger('sequencer'), ) { - this.metrics = new SequencerMetrics(telemetry, () => this.state, 'Sequencer'); - this.l1Metrics = new L1Metrics(telemetry.getMeter('SequencerL1Metrics'), publisher.l1TxUtils.publicClient, [ - publisher.getSenderAddress(), - ]); - - this.rewardsMetrics = new RewardMetrics( - telemetry.getMeter('ValidatorL1Metrics'), + this.metrics = new SequencerMetrics( + telemetry, + () => this.state, this._coinbase, - 'sequencer', this.publisher.getRollupContract(), + 'Sequencer', ); + this.l1Metrics = new L1Metrics(telemetry.getMeter('SequencerL1Metrics'), publisher.l1TxUtils.publicClient, [ + publisher.getSenderAddress(), + ]); // Register the block builder with the validator client for re-execution this.validatorClient?.registerBlockBuilder(this.buildBlock.bind(this)); @@ -206,6 +203,7 @@ export class Sequencer { */ public async start() { await this.updateConfig(this.config); + this.metrics.start(); this.runningPromise = new RunningPromise(this.work.bind(this), this.log, this.pollingIntervalMs); this.setState(SequencerState.IDLE, 0n, true /** force */); this.runningPromise.start(); @@ -218,6 +216,7 @@ export class Sequencer { */ public async stop(): Promise { this.log.debug(`Stopping sequencer`); + this.metrics.stop(); await this.validatorClient?.stop(); await this.runningPromise?.stop(); this.slasherClient.stop(); diff --git a/yarn-project/telemetry-client/src/attributes.ts b/yarn-project/telemetry-client/src/attributes.ts index ef5761b260c2..8d1f0147c128 100644 --- a/yarn-project/telemetry-client/src/attributes.ts +++ b/yarn-project/telemetry-client/src/attributes.ts @@ -64,8 +64,6 @@ export const ERROR_TYPE = 'aztec.error_type'; export const L1_TX_TYPE = 'aztec.l1.tx_type'; /** The L1 address of the entity that sent a transaction to L1 */ export const L1_SENDER = 'aztec.l1.sender'; -/** The role of node on the network (e.g. sequencer, prover) */ -export const NODE_ROLE = 'aztec.node_role'; /** The phase of the transaction */ export const TX_PHASE_NAME = 'aztec.tx.phase_name'; /** The reason for disconnecting a peer */ diff --git a/yarn-project/telemetry-client/src/index.ts b/yarn-project/telemetry-client/src/index.ts index cb955ca65981..db9f09d8edfd 100644 --- a/yarn-project/telemetry-client/src/index.ts +++ b/yarn-project/telemetry-client/src/index.ts @@ -5,7 +5,6 @@ export * from './with_tracer.js'; export * from './prom_otel_adapter.js'; export * from './lmdb_metrics.js'; export * from './l1_metrics.js'; -export * from './reward_metrics.js'; export * from './wrappers/index.js'; export * from './start.js'; export * from './otel_propagation.js'; diff --git a/yarn-project/telemetry-client/src/metrics.ts b/yarn-project/telemetry-client/src/metrics.ts index a60a0496d045..917fc6385f23 100644 --- a/yarn-project/telemetry-client/src/metrics.ts +++ b/yarn-project/telemetry-client/src/metrics.ts @@ -62,6 +62,7 @@ export const SEQUENCER_CURRENT_BLOCK_NUMBER = 'aztec.sequencer.current.block_num export const SEQUENCER_CURRENT_BLOCK_SIZE = 'aztec.sequencer.current.block_size'; export const SEQUENCER_TIME_TO_COLLECT_ATTESTATIONS = 'aztec.sequencer.time_to_collect_attestations'; export const SEQUENCER_BLOCK_BUILD_INSERTION_TIME = 'aztec.sequencer.block_builder_tree_insertion_duration'; +export const SEQUENCER_CURRENT_BLOCK_REWARDS = 'aztec.sequencer.current_block_rewards'; export const L1_PUBLISHER_GAS_PRICE = 'aztec.l1_publisher.gas_price'; export const L1_PUBLISHER_TX_COUNT = 'aztec.l1_publisher.tx_count'; @@ -83,9 +84,6 @@ export const L1_BALANCE_ETH = 'aztec.l1.balance'; export const L1_GAS_PRICE_WEI = 'aztec.l1.gas_price'; export const L1_BLOB_BASE_FEE_WEI = 'aztec.l1.blob_base_fee'; -export const L1_REWARDS_BALANCE = 'aztec.l1.rewards_balance'; -export const L1_REWARDS_BALANCE_SUM = 'aztec.l1.rewards_balance_sum'; - export const PEER_MANAGER_GOODBYES_SENT = 'aztec.peer_manager.goodbyes_sent'; export const PEER_MANAGER_GOODBYES_RECEIVED = 'aztec.peer_manager.goodbyes_received'; export const PEER_MANAGER_PEER_COUNT = 'aztec.peer_manager.peer_count'; @@ -141,6 +139,8 @@ export const PROVER_NODE_EXECUTION_DURATION = 'aztec.prover_node.execution.durat export const PROVER_NODE_JOB_DURATION = 'aztec.prover_node.job_duration'; export const PROVER_NODE_JOB_BLOCKS = 'aztec.prover_node.job_blocks'; export const PROVER_NODE_JOB_TRANSACTIONS = 'aztec.prover_node.job_transactions'; +export const PROVER_NODE_REWARDS_TOTAL = 'aztec.prover_node.rewards_total'; +export const PROVER_NODE_REWARDS_PER_EPOCH = 'aztec.prover_node.rewards_per_epoch'; export const WORLD_STATE_FORK_DURATION = 'aztec.world_state.fork.duration'; export const WORLD_STATE_SYNC_DURATION = 'aztec.world_state.sync.duration'; @@ -160,6 +160,8 @@ export const PROOF_VERIFIER_COUNT = 'aztec.proof_verifier.count'; export const VALIDATOR_RE_EXECUTION_TIME = 'aztec.validator.re_execution_time'; export const VALIDATOR_FAILED_REEXECUTION_COUNT = 'aztec.validator.failed_reexecution_count'; +export const VALIDATOR_ATTESTATION_COUNT = 'aztec.validator.attestation_count'; +export const VALIDATOR_FAILED_ATTESTATION_COUNT = 'aztec.validator.failed_attestation_count'; export const NODEJS_EVENT_LOOP_DELAY_MIN = 'nodejs.eventloop.delay.min'; export const NODEJS_EVENT_LOOP_DELAY_MEAN = 'nodejs.eventloop.delay.mean'; diff --git a/yarn-project/validator-client/src/metrics.ts b/yarn-project/validator-client/src/metrics.ts index 39d296df4474..f8ba850e25a8 100644 --- a/yarn-project/validator-client/src/metrics.ts +++ b/yarn-project/validator-client/src/metrics.ts @@ -11,6 +11,8 @@ import { export class ValidatorMetrics { private reExecutionTime: Gauge; private failedReexecutionCounter: UpDownCounter; + private attestationsCount: UpDownCounter; + private failedAttestationsCount: UpDownCounter; constructor(telemetryClient: TelemetryClient) { const meter = telemetryClient.getMeter('Validator'); @@ -26,6 +28,16 @@ export class ValidatorMetrics { unit: 'ms', valueType: ValueType.DOUBLE, }); + + this.attestationsCount = meter.createUpDownCounter(Metrics.VALIDATOR_ATTESTATION_COUNT, { + description: 'The number of attestations', + valueType: ValueType.INT, + }); + + this.failedAttestationsCount = meter.createUpDownCounter(Metrics.VALIDATOR_FAILED_ATTESTATION_COUNT, { + description: 'The number of failed attestations', + valueType: ValueType.INT, + }); } public reExecutionTimer(): () => void { @@ -46,4 +58,14 @@ export class ValidatorMetrics { [Attributes.BLOCK_PROPOSER]: (await proposal.getSender())?.toString(), }); } + + public incAttestations() { + this.attestationsCount.add(1); + } + + public incFailedAttestations(reason: string) { + this.failedAttestationsCount.add(1, { + [Attributes.ERROR_TYPE]: reason, + }); + } } diff --git a/yarn-project/validator-client/src/validator.ts b/yarn-project/validator-client/src/validator.ts index 20c2fd2f42f8..f579aed5776a 100644 --- a/yarn-project/validator-client/src/validator.ts +++ b/yarn-project/validator-client/src/validator.ts @@ -197,6 +197,7 @@ export class ValidatorClient extends WithTracer implements Validator { const invalidProposal = await this.blockProposalValidator.validate(proposal); if (invalidProposal) { this.log.verbose(`Proposal is not valid, skipping attestation`); + this.metrics.incFailedAttestations('invalid_proposal'); return undefined; } @@ -210,6 +211,12 @@ export class ValidatorClient extends WithTracer implements Validator { await this.reExecuteTransactions(proposal); } } catch (error: any) { + if (error instanceof Error) { + this.metrics.incFailedAttestations(error.name); + } else { + this.metrics.incFailedAttestations('unknown'); + } + // If the transactions are not available, then we should not attempt to attest if (error instanceof TransactionsNotAvailableError) { this.log.error(`Transactions not available, skipping attestation`, error, proposalInfo); @@ -223,6 +230,7 @@ export class ValidatorClient extends WithTracer implements Validator { // Provided all of the above checks pass, we can attest to the proposal this.log.info(`Attesting to proposal for slot ${slotNumber}`, proposalInfo); + this.metrics.incAttestations(); // If the above function does not throw an error, then we can attest to the proposal return this.doAttestToProposal(proposal); diff --git a/yarn-project/world-state/src/synchronizer/errors.ts b/yarn-project/world-state/src/synchronizer/errors.ts new file mode 100644 index 000000000000..b0cb2924e113 --- /dev/null +++ b/yarn-project/world-state/src/synchronizer/errors.ts @@ -0,0 +1,5 @@ +export class WorldStateSynchronizerError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(message, options); + } +} diff --git a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts index 24aba1abc5bd..1568fda30c25 100644 --- a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts +++ b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts @@ -31,6 +31,7 @@ import { WorldStateInstrumentation } from '../instrumentation/instrumentation.js import type { WorldStateStatusFull } from '../native/message.js'; import type { MerkleTreeAdminDatabase } from '../world-state-db/merkle_tree_db.js'; import type { WorldStateConfig } from './config.js'; +import { WorldStateSynchronizerError } from './errors.js'; export type { SnapshotDataKeys }; @@ -202,7 +203,17 @@ export class ServerWorldStateSynchronizer // If we have been given a block number to sync to and we have not reached that number then fail const updatedBlockNumber = await this.getLatestBlockNumber(); if (!skipThrowIfTargetNotReached && targetBlockNumber !== undefined && targetBlockNumber > updatedBlockNumber) { - throw new Error(`Unable to sync to block number ${targetBlockNumber} (last synced is ${updatedBlockNumber})`); + throw new WorldStateSynchronizerError( + `Unable to sync to block number ${targetBlockNumber} (last synced is ${updatedBlockNumber})`, + { + cause: { + reason: 'block_not_available', + previousBlockNumber: currentBlockNumber, + updatedBlockNumber, + targetBlockNumber, + }, + }, + ); } return updatedBlockNumber; From 0c0f3d58be8aa78845efa1171dee43b502087546 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Mon, 14 Apr 2025 14:33:40 +0000 Subject: [PATCH 3/4] feat: track filled slots --- .../src/publisher/sequencer-publisher.ts | 6 ++- .../sequencer-client/src/sequencer/metrics.ts | 49 +++++++++++++++++++ .../src/sequencer/sequencer.ts | 9 +++- yarn-project/telemetry-client/src/metrics.ts | 3 ++ 4 files changed, 65 insertions(+), 2 deletions(-) diff --git a/yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts b/yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts index 44ef0796803f..b91eb6a5a119 100644 --- a/yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts +++ b/yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts @@ -184,6 +184,10 @@ export class SequencerPublisher { const currentL2Slot = this.getCurrentL2Slot(); this.log.debug(`Current L2 slot: ${currentL2Slot}`); const validRequests = requestsToProcess.filter(request => request.lastValidL2Slot >= currentL2Slot); + const validActions = validRequests.map(x => x.action); + const expiredActions = requestsToProcess + .filter(request => request.lastValidL2Slot < currentL2Slot) + .map(x => x.action); if (validRequests.length !== requestsToProcess.length) { this.log.warn(`Some requests were expired for slot ${currentL2Slot}`, { @@ -228,7 +232,7 @@ export class SequencerPublisher { this.log, ); this.callbackBundledTransactions(validRequests, result); - return result; + return { result, expiredActions, validActions }; } catch (err) { const viemError = formatViemError(err); this.log.error(`Failed to publish bundled transactions`, viemError); diff --git a/yarn-project/sequencer-client/src/sequencer/metrics.ts b/yarn-project/sequencer-client/src/sequencer/metrics.ts index 512c1712c12a..e8395f65da8d 100644 --- a/yarn-project/sequencer-client/src/sequencer/metrics.ts +++ b/yarn-project/sequencer-client/src/sequencer/metrics.ts @@ -34,6 +34,12 @@ export class SequencerMetrics { private rewards: ObservableGauge; + private slots: UpDownCounter; + private filledSlots: UpDownCounter; + private missedSlots: UpDownCounter; + + private lastSeenSlot?: bigint; + constructor( client: TelemetryClient, getState: SequencerStateCallback, @@ -113,6 +119,21 @@ export class SequencerMetrics { valueType: ValueType.DOUBLE, description: 'The rewards earned', }); + + this.slots = this.meter.createUpDownCounter(Metrics.SEQUENCER_SLOT_COUNT, { + valueType: ValueType.INT, + description: 'The number of slots this sequencer was selected for', + }); + + this.filledSlots = this.meter.createUpDownCounter(Metrics.SEQUENCER_FILLED_SLOT_COUNT, { + valueType: ValueType.INT, + description: 'The number of slots this sequencer has filled', + }); + + this.missedSlots = this.meter.createUpDownCounter(Metrics.SEQUENCER_MISSED_SLOT_COUNT, { + valueType: ValueType.INT, + description: 'The number of slots this sequencer has missed to fill', + }); } public start() { @@ -182,6 +203,34 @@ export class SequencerMetrics { }); } + observeSlotChange(slot: bigint | undefined, proposer: string) { + // sequencer went through the loop a second time. Noop + if (slot === this.lastSeenSlot) { + return; + } + + if (typeof this.lastSeenSlot === 'bigint') { + this.missedSlots.add(1, { + [Attributes.BLOCK_PROPOSER]: proposer, + }); + } + + if (typeof slot === 'bigint') { + this.slots.add(1, { + [Attributes.BLOCK_PROPOSER]: proposer, + }); + } + + this.lastSeenSlot = slot; + } + + incFilledSlot(proposer: string) { + this.filledSlots.add(1, { + [Attributes.BLOCK_PROPOSER]: proposer, + }); + this.lastSeenSlot = undefined; + } + private setCurrentBlock(blockNumber: number, txCount: number) { this.currentBlockNumber.record(blockNumber); this.currentBlockSize.record(txCount); diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.ts index 0f22912379d8..edb78076bfce 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.ts @@ -274,6 +274,7 @@ export class Sequencer { const chainTipArchive = chainTip.archive; const slot = await this.slotForProposal(chainTipArchive.toBuffer(), BigInt(newBlockNumber)); + this.metrics.observeSlotChange(slot, this.publisher.getSenderAddress().toString()); if (!slot) { this.log.debug(`Cannot propose block ${newBlockNumber}`); return; @@ -341,7 +342,13 @@ export class Sequencer { this.log.error(`Error enqueuing slashing vote`, err, { blockNumber: newBlockNumber, slot }); }); - await this.publisher.sendRequests(); + const resp = await this.publisher.sendRequests(); + if (resp) { + const proposedBlock = resp.validActions.find(a => a === 'propose'); + if (proposedBlock) { + this.metrics.incFilledSlot(this.publisher.getSenderAddress().toString()); + } + } if (finishedFlushing) { this.isFlushing = false; diff --git a/yarn-project/telemetry-client/src/metrics.ts b/yarn-project/telemetry-client/src/metrics.ts index 917fc6385f23..e830b6645735 100644 --- a/yarn-project/telemetry-client/src/metrics.ts +++ b/yarn-project/telemetry-client/src/metrics.ts @@ -63,6 +63,9 @@ export const SEQUENCER_CURRENT_BLOCK_SIZE = 'aztec.sequencer.current.block_size' export const SEQUENCER_TIME_TO_COLLECT_ATTESTATIONS = 'aztec.sequencer.time_to_collect_attestations'; export const SEQUENCER_BLOCK_BUILD_INSERTION_TIME = 'aztec.sequencer.block_builder_tree_insertion_duration'; export const SEQUENCER_CURRENT_BLOCK_REWARDS = 'aztec.sequencer.current_block_rewards'; +export const SEQUENCER_SLOT_COUNT = 'aztec.sequencer.slot.total_count'; +export const SEQUENCER_FILLED_SLOT_COUNT = 'aztec.sequencer.slot.filled_count'; +export const SEQUENCER_MISSED_SLOT_COUNT = 'aztec.sequencer.slot.missed_count'; export const L1_PUBLISHER_GAS_PRICE = 'aztec.l1_publisher.gas_price'; export const L1_PUBLISHER_TX_COUNT = 'aztec.l1_publisher.tx_count'; From fc10f8260e45dc85531eb33b9af520374c909b23 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Mon, 14 Apr 2025 20:02:50 +0000 Subject: [PATCH 4/4] chore: logging --- yarn-project/prover-node/src/metrics.ts | 15 +++++++-------- yarn-project/prover-node/src/prover-node.test.ts | 5 ++++- .../src/publisher/sequencer-publisher.test.ts | 2 +- .../sequencer-client/src/sequencer/metrics.ts | 2 +- yarn-project/telemetry-client/src/attributes.ts | 2 ++ 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/yarn-project/prover-node/src/metrics.ts b/yarn-project/prover-node/src/metrics.ts index 1f8a6862c382..91a69a862e9e 100644 --- a/yarn-project/prover-node/src/metrics.ts +++ b/yarn-project/prover-node/src/metrics.ts @@ -10,7 +10,6 @@ import { type Meter, Metrics, type ObservableGauge, - type ObservableUpDownCounter, type TelemetryClient, type Tracer, type UpDownCounter, @@ -60,7 +59,7 @@ export class ProverNodeJobMetrics { export class ProverNodeRewardsMetrics { private rewards: ObservableGauge; - private accumulatedRewards: ObservableUpDownCounter; + private accumulatedRewards: UpDownCounter; private prevEpoch = -1n; private proofSubmissionWindow = 0n; @@ -75,7 +74,7 @@ export class ProverNodeRewardsMetrics { description: 'The rewards earned', }); - this.accumulatedRewards = this.meter.createObservableUpDownCounter(Metrics.PROVER_NODE_REWARDS_TOTAL, { + this.accumulatedRewards = this.meter.createUpDownCounter(Metrics.PROVER_NODE_REWARDS_TOTAL, { valueType: ValueType.DOUBLE, description: 'The rewards earned (total)', }); @@ -83,11 +82,11 @@ export class ProverNodeRewardsMetrics { public async start() { this.proofSubmissionWindow = await this.rollup.getProofSubmissionWindow(); - this.meter.addBatchObservableCallback(this.observe, [this.rewards, this.accumulatedRewards]); + this.meter.addBatchObservableCallback(this.observe, [this.rewards]); } public stop() { - this.meter.removeBatchObservableCallback(this.observe, [this.rewards, this.accumulatedRewards]); + this.meter.removeBatchObservableCallback(this.observe, [this.rewards]); } private observe = async (observer: BatchObservableResult): Promise => { @@ -101,14 +100,14 @@ export class ProverNodeRewardsMetrics { const fmt = parseFloat(formatUnits(rewards, 18)); observer.observe(this.rewards, fmt, { - [Attributes.L1_SENDER]: this.coinbase.toString(), + [Attributes.COINBASE]: this.coinbase.toString(), }); // only accumulate once per epoch if (closedEpoch > this.prevEpoch) { this.prevEpoch = closedEpoch; - observer.observe(this.accumulatedRewards, fmt, { - [Attributes.L1_SENDER]: this.coinbase.toString(), + this.accumulatedRewards.add(fmt, { + [Attributes.COINBASE]: this.coinbase.toString(), }); } } diff --git a/yarn-project/prover-node/src/prover-node.test.ts b/yarn-project/prover-node/src/prover-node.test.ts index 50acfb22111c..d8729a02e41a 100644 --- a/yarn-project/prover-node/src/prover-node.test.ts +++ b/yarn-project/prover-node/src/prover-node.test.ts @@ -1,5 +1,6 @@ import { timesParallel } from '@aztec/foundation/collection'; import { EthAddress } from '@aztec/foundation/eth-address'; +import { Fr } from '@aztec/foundation/fields'; import { promiseWithResolvers } from '@aztec/foundation/promise'; import { retryUntil } from '@aztec/foundation/retry'; import { sleep } from '@aztec/foundation/sleep'; @@ -67,7 +68,9 @@ describe('prover-node', () => { ); beforeEach(async () => { - prover = mock(); + prover = mock({ + getProverId: () => Fr.random(), + }); publisher = mock(); l2BlockSource = mock(); l1ToL2MessageSource = mock(); diff --git a/yarn-project/sequencer-client/src/publisher/sequencer-publisher.test.ts b/yarn-project/sequencer-client/src/publisher/sequencer-publisher.test.ts index 10bd479e49c6..869326a10a6a 100644 --- a/yarn-project/sequencer-client/src/publisher/sequencer-publisher.test.ts +++ b/yarn-project/sequencer-client/src/publisher/sequencer-publisher.test.ts @@ -281,7 +281,7 @@ describe('SequencerPublisher', () => { expect(enqueued).toEqual(true); const result = await publisher.sendRequests(); - expect(result?.errorMsg).toEqual('Test error'); + expect(result?.result?.errorMsg).toEqual('Test error'); }); it('does not send requests if interrupted', async () => { diff --git a/yarn-project/sequencer-client/src/sequencer/metrics.ts b/yarn-project/sequencer-client/src/sequencer/metrics.ts index e8395f65da8d..eeb2e8925a42 100644 --- a/yarn-project/sequencer-client/src/sequencer/metrics.ts +++ b/yarn-project/sequencer-client/src/sequencer/metrics.ts @@ -150,7 +150,7 @@ export class SequencerMetrics { const fmt = parseFloat(formatUnits(rewards, 18)); observer.observe(this.rewards, fmt, { - [Attributes.L1_SENDER]: this.coinbase.toString(), + [Attributes.COINBASE]: this.coinbase.toString(), }); }; diff --git a/yarn-project/telemetry-client/src/attributes.ts b/yarn-project/telemetry-client/src/attributes.ts index 8d1f0147c128..f5bb8aacf9d8 100644 --- a/yarn-project/telemetry-client/src/attributes.ts +++ b/yarn-project/telemetry-client/src/attributes.ts @@ -64,6 +64,8 @@ export const ERROR_TYPE = 'aztec.error_type'; export const L1_TX_TYPE = 'aztec.l1.tx_type'; /** The L1 address of the entity that sent a transaction to L1 */ export const L1_SENDER = 'aztec.l1.sender'; +/** The L1 address receiving rewards */ +export const COINBASE = 'aztec.coinbase'; /** The phase of the transaction */ export const TX_PHASE_NAME = 'aztec.tx.phase_name'; /** The reason for disconnecting a peer */