diff --git a/yarn-project/end-to-end/src/spartan/n_tps.test.ts b/yarn-project/end-to-end/src/spartan/n_tps.test.ts index 72a3d3002955..d7a774b5fc3b 100644 --- a/yarn-project/end-to-end/src/spartan/n_tps.test.ts +++ b/yarn-project/end-to-end/src/spartan/n_tps.test.ts @@ -90,10 +90,10 @@ const mempoolTxMinedDelayQuery = (perc: string) => const mempoolAttestationMinedDelayQuery = (perc: string) => `histogram_quantile(${perc}, sum(rate(aztec_mempool_attestations_mined_delay_milliseconds_bucket{k8s_namespace_name="${config.NAMESPACE}"}[1m])) by (le))`; -const peerCountQuery = () => `avg(aztec_peer_manager_peer_count{k8s_namespace_name="${config.NAMESPACE}"})`; +const peerCountQuery = () => `avg(aztec_peer_manager_peer_count_peers{k8s_namespace_name="${config.NAMESPACE}"})`; -const peerConnectionDurationQuery = (perc: string) => - `histogram_quantile(${perc}, sum(rate(aztec_peer_manager_peer_connection_duration_milliseconds_bucket{k8s_namespace_name="${config.NAMESPACE}"}[1m])) by (le))`; +const peerConnectionDurationQuery = (perc: string, windowSeconds: number) => + `histogram_quantile(${perc}, sum(rate(aztec_peer_manager_peer_connection_duration_milliseconds_bucket{k8s_namespace_name="${config.NAMESPACE}"}[${windowSeconds}s])) by (le))`; describe('sustained N TPS test', () => { jest.setTimeout(60 * 60 * 1000 * 10); // 10 hours @@ -168,8 +168,8 @@ describe('sustained N TPS test', () => { try { const [avgCount, durationP50, durationP95] = await Promise.all([ prometheusClient.querySingleValue(peerCountQuery()), - prometheusClient.querySingleValue(peerConnectionDurationQuery('0.50')), - prometheusClient.querySingleValue(peerConnectionDurationQuery('0.95')), + prometheusClient.querySingleValue(peerConnectionDurationQuery('0.50', TEST_DURATION_SECONDS + 60)), + prometheusClient.querySingleValue(peerConnectionDurationQuery('0.95', TEST_DURATION_SECONDS + 60)), ]); metrics.recordPeerStats(avgCount, durationP50, durationP95); logger.debug('Scraped peer stats', { avgCount, durationP50, durationP95 }); @@ -384,7 +384,7 @@ describe('sustained N TPS test', () => { const tx = await (config.REAL_VERIFIER ? submitProven(wallet, fee) : submitUnproven(wallet, fee)); const t1 = performance.now(); - metrics.recordSentTx(tx, `high_value_${highValueTps}tps`); + metrics.recordSentTx(tx, 'tx_inclusion_time'); const txHash = await tx.send({ wait: NO_WAIT }); const t2 = performance.now(); @@ -461,8 +461,8 @@ describe('sustained N TPS test', () => { logger.warn(`Failed transaction ${idx + 1}: ${result.error}`); }); - const highValueGroup = `high_value_${highValueTps}tps`; - const inclusionStats = metrics.inclusionTimeInSeconds(highValueGroup); + const txInclusionGroup = 'tx_inclusion_time'; + const inclusionStats = metrics.inclusionTimeInSeconds(txInclusionGroup); logger.info(`Transaction inclusion summary: ${successCount} succeeded, ${failureCount} failed`); logger.info('Inclusion time stats', inclusionStats); }); diff --git a/yarn-project/end-to-end/src/spartan/tx_metrics.ts b/yarn-project/end-to-end/src/spartan/tx_metrics.ts index 6234641a9276..20d536432ceb 100644 --- a/yarn-project/end-to-end/src/spartan/tx_metrics.ts +++ b/yarn-project/end-to-end/src/spartan/tx_metrics.ts @@ -296,7 +296,7 @@ export class TxInclusionMetrics { value: stats.mean, }, { - name: `${group}/median_inclusion`, + name: `${group}/p50_inclusion`, unit: 's', value: stats.median, }, diff --git a/yarn-project/p2p/src/mem_pools/attestation_pool/attestation_pool.ts b/yarn-project/p2p/src/mem_pools/attestation_pool/attestation_pool.ts index 827d91ce84ea..e61456ef5cbf 100644 --- a/yarn-project/p2p/src/mem_pools/attestation_pool/attestation_pool.ts +++ b/yarn-project/p2p/src/mem_pools/attestation_pool/attestation_pool.ts @@ -359,11 +359,10 @@ export class AttestationPool { } const address = sender.toString(); + const ownKey = this.getAttestationKey(slotNumber, proposalId, address); - await this.checkpointAttestations.set( - this.getAttestationKey(slotNumber, proposalId, address), - attestation.toBuffer(), - ); + await this.checkpointAttestations.set(ownKey, attestation.toBuffer()); + this.metrics.trackMempoolItemAdded(ownKey); this.log.debug(`Added own checkpoint attestation for slot ${slotNumber} from ${address}`, { signature: attestation.signature.toString(), @@ -429,6 +428,7 @@ export class AttestationPool { const attestationEndKey = new Fr(oldestSlot).toString(); for await (const key of this.checkpointAttestations.keysAsync({ end: attestationEndKey })) { await this.checkpointAttestations.delete(key); + this.metrics.trackMempoolItemRemoved(key); numberOfAttestations++; } @@ -526,6 +526,7 @@ export class AttestationPool { // Add the attestation await this.checkpointAttestations.set(key, attestation.toBuffer()); + this.metrics.trackMempoolItemAdded(key); // Track this attestation in the per-signer-per-slot index for duplicate detection const slotSignerKey = this.getSlotSignerKey(slotNumber, signerAddress); diff --git a/yarn-project/p2p/src/mem_pools/instrumentation.ts b/yarn-project/p2p/src/mem_pools/instrumentation.ts index bfd3c7b64ac7..d76d2c30ad4a 100644 --- a/yarn-project/p2p/src/mem_pools/instrumentation.ts +++ b/yarn-project/p2p/src/mem_pools/instrumentation.ts @@ -73,7 +73,7 @@ export class PoolInstrumentation { private defaultAttributes; private meter: Meter; - private txAddedTimestamp: Map = new Map(); + private mempoolItemAddedTimestamp: Map = new Map(); constructor( telemetry: TelemetryClient, @@ -114,22 +114,26 @@ export class PoolInstrumentation { } public transactionsAdded(transactions: Tx[]) { - const timestamp = Date.now(); - for (const transaction of transactions) { - this.txAddedTimestamp.set(transaction.txHash.toBigInt(), timestamp); - } + transactions.forEach(tx => this.trackMempoolItemAdded(tx.txHash.toBigInt())); } public transactionsRemoved(hashes: Iterable | Iterable) { - const timestamp = Date.now(); for (const hash of hashes) { - const key = BigInt(hash); - const addedAt = this.txAddedTimestamp.get(key); - if (addedAt !== undefined) { - this.txAddedTimestamp.delete(key); - if (addedAt < timestamp) { - this.minedDelay.record(timestamp - addedAt); - } + this.trackMempoolItemRemoved(BigInt(hash)); + } + } + + public trackMempoolItemAdded(key: bigint | string): void { + this.mempoolItemAddedTimestamp.set(key, Date.now()); + } + + public trackMempoolItemRemoved(key: bigint | string): void { + const timestamp = Date.now(); + const addedAt = this.mempoolItemAddedTimestamp.get(key); + if (addedAt !== undefined) { + this.mempoolItemAddedTimestamp.delete(key); + if (addedAt < timestamp) { + this.minedDelay.record(timestamp - addedAt); } } } diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.ts index 4f65a0d6b0ae..69b91bd95c52 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.ts @@ -58,6 +58,9 @@ export class AztecKVTxPoolV2 extends (EventEmitter as new () => TypedEventEmitte const hashes = txHashes.map(h => (typeof h === 'string' ? TxHash.fromString(h) : TxHash.fromBigInt(h))); this.emit('txs-removed', { txHashes: hashes }); }, + onTxsMined: (txHashes: string[]) => { + this.#metrics?.transactionsRemoved(txHashes); + }, }; // Create the implementation diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts index 88f6e887b9a8..15f6eb4b051e 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts @@ -45,6 +45,7 @@ import { TxPoolIndices } from './tx_pool_indices.js'; export interface TxPoolV2Callbacks { onTxsAdded: (txs: Tx[], opts: { source?: string }) => void; onTxsRemoved: (txHashes: string[] | bigint[]) => void; + onTxsMined: (txHashes: string[]) => void; } /** @@ -498,6 +499,10 @@ export class TxPoolV2Impl { await this.#evictionManager.evictAfterNewBlock(block.header, nullifiers, feePayers); }); + if (found.length > 0) { + this.#callbacks.onTxsMined(found.map(m => m.txHash)); + } + this.#log.info(`Marked ${found.length} txs as mined in block ${blockId.number}`); } diff --git a/yarn-project/p2p/src/services/peer-manager/metrics.ts b/yarn-project/p2p/src/services/peer-manager/metrics.ts index 06cd513727db..2e1f198611a4 100644 --- a/yarn-project/p2p/src/services/peer-manager/metrics.ts +++ b/yarn-project/p2p/src/services/peer-manager/metrics.ts @@ -18,6 +18,7 @@ export class PeerManagerMetrics { private sentGoodbyes: UpDownCounter; private receivedGoodbyes: UpDownCounter; private peerCount: Gauge; + private healthyPeerCount: Gauge; private lowScoreDisconnects: UpDownCounter; private peerConnectionDuration: Histogram; @@ -49,6 +50,7 @@ export class PeerManagerMetrics { goodbyeReasonAttrs, ); this.peerCount = meter.createGauge(Metrics.PEER_MANAGER_PEER_COUNT); + this.healthyPeerCount = meter.createGauge(Metrics.PEER_MANAGER_HEALTHY_PEER_COUNT); this.lowScoreDisconnects = createUpDownCounterWithDefault(meter, Metrics.PEER_MANAGER_LOW_SCORE_DISCONNECTS, { [Attributes.P2P_PEER_SCORE_STATE]: ['Banned', 'Disconnect'], }); @@ -67,6 +69,10 @@ export class PeerManagerMetrics { this.peerCount.record(count); } + public recordHealthyPeerCount(count: number) { + this.healthyPeerCount.record(count); + } + public recordLowScoreDisconnect(scoreState: 'Banned' | 'Disconnect') { this.lowScoreDisconnects.add(1, { [Attributes.P2P_PEER_SCORE_STATE]: scoreState }); } @@ -79,6 +85,7 @@ export class PeerManagerMetrics { const connectedAt = this.peerConnectedAt.get(id.toString()); if (connectedAt) { this.peerConnectionDuration.record(Date.now() - connectedAt); + this.peerConnectedAt.delete(id.toString()); } } } diff --git a/yarn-project/p2p/src/services/peer-manager/peer_manager.ts b/yarn-project/p2p/src/services/peer-manager/peer_manager.ts index 336f726b6dec..669f0e149a9c 100644 --- a/yarn-project/p2p/src/services/peer-manager/peer_manager.ts +++ b/yarn-project/p2p/src/services/peer-manager/peer_manager.ts @@ -515,7 +515,8 @@ export class PeerManager implements PeerManagerInterface { ...this.peerScoring.getStats(), }); - this.metrics.recordPeerCount(healthyConnections.length); + this.metrics.recordPeerCount(connections.length); + this.metrics.recordHealthyPeerCount(healthyConnections.length); // Exit if no peers to connect if (peersToConnect <= 0) { diff --git a/yarn-project/telemetry-client/src/metrics.ts b/yarn-project/telemetry-client/src/metrics.ts index d30a4185e15b..f0065257c1fe 100644 --- a/yarn-project/telemetry-client/src/metrics.ts +++ b/yarn-project/telemetry-client/src/metrics.ts @@ -764,6 +764,12 @@ export const PEER_MANAGER_PEER_COUNT: MetricDefinition = { unit: 'peers', valueType: ValueType.INT, }; +export const PEER_MANAGER_HEALTHY_PEER_COUNT: MetricDefinition = { + name: 'aztec.peer_manager.healthy_peer_count', + description: 'Number of healthy (non-protected, non-banned) peers', + unit: 'peers', + valueType: ValueType.INT, +}; export const PEER_MANAGER_LOW_SCORE_DISCONNECTS: MetricDefinition = { name: 'aztec.peer_manager.low_score_disconnects', description: 'Number of peers disconnected due to low score',