feat: Metrics added to the transaction pool

spartan/aztec-chaos-scenarios/values/network-requirements.yaml

@@ -9,7 +9,7 @@ networkShaping:
         correlation: "75"
     bandwidth:
       enabled: true
-      rate: 25mbps
+      rate: 200mbps
     packetLoss:
       enabled: true
       loss:

spartan/environments/five-tps-long-epoch.env

@@ -6,7 +6,7 @@ DESTROY_ETH_DEVNET=true
 CREATE_ETH_DEVNET=${CREATE_ETH_DEVNET:-true}
 AZTEC_EPOCH_DURATION=32
 AZTEC_SLOT_DURATION=36
-AZTEC_PROOF_SUBMISSION_WINDOW=64
+AZTEC_PROOF_SUBMISSION_EPOCHS=2
 ETHEREUM_CHAIN_ID=1337
 LABS_INFRA_MNEMONIC="test test test test test test test test test test test junk"
 FUNDING_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"

spartan/environments/five-tps-short-epoch.env

@@ -6,7 +6,7 @@ DESTROY_ETH_DEVNET=true
 CREATE_ETH_DEVNET=${CREATE_ETH_DEVNET:-true}
 AZTEC_EPOCH_DURATION=8
 AZTEC_SLOT_DURATION=36
-AZTEC_PROOF_SUBMISSION_WINDOW=16
+AZTEC_PROOF_SUBMISSION_EPOCHS=10
 ETHEREUM_CHAIN_ID=1337
 LABS_INFRA_MNEMONIC="test test test test test test test test test test test junk"
 FUNDING_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"

spartan/environments/ten-tps-long-epoch.env

@@ -6,7 +6,7 @@ DESTROY_ETH_DEVNET=true
 CREATE_ETH_DEVNET=${CREATE_ETH_DEVNET:-true}
 AZTEC_EPOCH_DURATION=32
 AZTEC_SLOT_DURATION=36
-AZTEC_PROOF_SUBMISSION_WINDOW=64
+AZTEC_PROOF_SUBMISSION_EPOCHS=2
 ETHEREUM_CHAIN_ID=1337
 LABS_INFRA_MNEMONIC="test test test test test test test test test test test junk"
 FUNDING_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"

spartan/environments/ten-tps-short-epoch.env

@@ -6,7 +6,7 @@ DESTROY_ETH_DEVNET=true
 CREATE_ETH_DEVNET=${CREATE_ETH_DEVNET:-true}
 AZTEC_EPOCH_DURATION=8
 AZTEC_SLOT_DURATION=36
-AZTEC_PROOF_SUBMISSION_WINDOW=16
+AZTEC_PROOF_SUBMISSION_EPOCHS=2
 ETHEREUM_CHAIN_ID=1337
 LABS_INFRA_MNEMONIC="test test test test test test test test test test test junk"
 FUNDING_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"

spartan/environments/tps-scenario.env

@@ -4,7 +4,7 @@ GCP_REGION=us-west1-a
 
 AZTEC_EPOCH_DURATION=8
 AZTEC_SLOT_DURATION=72
-AZTEC_PROOF_SUBMISSION_WINDOW=16
+AZTEC_PROOF_SUBMISSION_EPOCHS=2
 AZTEC_LAG_IN_EPOCHS=1
 
 CREATE_ETH_DEVNET=false

spartan/scripts/deploy_network.sh

@@ -97,6 +97,7 @@ SEQ_MIN_TX_PER_BLOCK=${SEQ_MIN_TX_PER_BLOCK:-0}
 SEQ_MAX_TX_PER_BLOCK=${SEQ_MAX_TX_PER_BLOCK:-8}
 SEQ_BLOCK_DURATION_MS=${SEQ_BLOCK_DURATION_MS:-}
 SEQ_BUILD_CHECKPOINT_IF_EMPTY=${SEQ_BUILD_CHECKPOINT_IF_EMPTY:-}
+SEQ_SKIP_CHECKPOINT_PUBLISH_PERCENT=${SEQ_SKIP_CHECKPOINT_PUBLISH_PERCENT:0}
 PROVER_REPLICAS=${PROVER_REPLICAS:-4}
 PROVER_AGENTS_PER_PROVER=${PROVER_AGENTS_PER_PROVER:-1}
 R2_ACCESS_KEY_ID=${R2_ACCESS_KEY_ID:-}
@@ -506,6 +507,7 @@ SEQ_MIN_TX_PER_BLOCK = ${SEQ_MIN_TX_PER_BLOCK}
 SEQ_MAX_TX_PER_BLOCK = ${SEQ_MAX_TX_PER_BLOCK}
 SEQ_BLOCK_DURATION_MS = ${SEQ_BLOCK_DURATION_MS:-null}
 SEQ_BUILD_CHECKPOINT_IF_EMPTY = ${SEQ_BUILD_CHECKPOINT_IF_EMPTY:-null}
+SEQ_SKIP_CHECKPOINT_PUBLISH_PERCENT = ${SEQ_SKIP_CHECKPOINT_PUBLISH_PERCENT}
 PROVER_MNEMONIC = "${LABS_INFRA_MNEMONIC}"
 PROVER_PUBLISHER_MNEMONIC_START_INDEX = ${PROVER_PUBLISHER_MNEMONIC_START_INDEX}
 PROVER_PUBLISHERS_PER_PROVER = ${PUBLISHERS_PER_PROVER}

spartan/terraform/deploy-aztec-infra/main.tf

@@ -220,6 +220,7 @@ locals {
     "validator.node.env.P2P_DROP_TX_CHANCE"                    = var.P2P_DROP_TX_CHANCE
     "validator.node.env.WS_NUM_HISTORIC_BLOCKS"                = var.WS_NUM_HISTORIC_BLOCKS
     "validator.node.env.TX_COLLECTION_FILE_STORE_URLS"         = var.TX_COLLECTION_FILE_STORE_URLS
+    "validator.node.env.SEQ_SKIP_CHECKPOINT_PUBLISH_PERCENT"   = var.SEQ_SKIP_CHECKPOINT_PUBLISH_PERCENT
   }
 
   # Note: nonsensitive() is required here because helm_releases is used in for_each,

spartan/terraform/deploy-aztec-infra/variables.tf

@@ -343,6 +343,12 @@ variable "SEQ_MAX_TX_PER_BLOCK" {
   default     = "8"
 }
 
+variable "SEQ_SKIP_CHECKPOINT_PUBLISH_PERCENT" {
+  description = "Percentage probability of skipping checkpoint publishing"
+  type        = string
+  default     = "0"
+}
+
 variable "SEQ_BLOCK_DURATION_MS" {
   description = "Duration per block in milliseconds when building multiple blocks per slot"
   type        = string

yarn-project/archiver/src/modules/instrumentation.ts

@@ -13,10 +13,13 @@ import {
   createUpDownCounterWithDefault,
 } from '@aztec/telemetry-client';
 
+import type { CheckpointData } from '../store/block_store.js';
+
 export class ArchiverInstrumentation {
   public readonly tracer: Tracer;
 
   private blockHeight: Gauge;
+  private checkpointHeight: Gauge;
   private txCount: UpDownCounter;
   private l1BlockHeight: Gauge;
   private proofsSubmittedDelay: Histogram;
@@ -47,6 +50,8 @@ export class ArchiverInstrumentation {
 
     this.blockHeight = meter.createGauge(Metrics.ARCHIVER_BLOCK_HEIGHT);
 
+    this.checkpointHeight = meter.createGauge(Metrics.ARCHIVER_CHECKPOINT_HEIGHT);
+
     this.l1BlockHeight = meter.createGauge(Metrics.ARCHIVER_L1_BLOCK_HEIGHT);
 
     this.txCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_TOTAL_TXS);
@@ -105,6 +110,7 @@ export class ArchiverInstrumentation {
   public processNewBlocks(syncTimePerBlock: number, blocks: L2Block[]) {
     this.syncDurationPerBlock.record(Math.ceil(syncTimePerBlock));
     this.blockHeight.record(Math.max(...blocks.map(b => b.number)));
+    this.checkpointHeight.record(Math.max(...blocks.map(b => b.checkpointNumber)));
     this.syncBlockCount.add(blocks.length);
 
     for (const block of blocks) {
@@ -127,8 +133,10 @@ export class ArchiverInstrumentation {
     this.pruneDuration.record(Math.ceil(duration));
   }
 
-  public updateLastProvenBlock(blockNumber: number) {
-    this.blockHeight.record(blockNumber, { [Attributes.STATUS]: 'proven' });
+  public updateLastProvenCheckpoint(checkpoint: CheckpointData) {
+    const lastBlockNumberInCheckpoint = checkpoint.startBlock + checkpoint.numBlocks - 1;
+    this.blockHeight.record(lastBlockNumberInCheckpoint, { [Attributes.STATUS]: 'proven' });
+    this.checkpointHeight.record(checkpoint.checkpointNumber, { [Attributes.STATUS]: 'proven' });
   }
 
   public processProofsVerified(logs: { proverId: string; l2BlockNumber: bigint; delay: bigint }[]) {

yarn-project/archiver/src/modules/l1_synchronizer.ts

@@ -599,7 +599,7 @@ export class ArchiverL1Synchronizer implements Traceable {
             slotNumber: provenSlotNumber,
             epochNumber: provenEpochNumber,
           });
-          this.instrumentation.updateLastProvenBlock(lastBlockNumberInCheckpoint);
+          this.instrumentation.updateLastProvenCheckpoint(localCheckpointForDestinationProvenCheckpointNumber);
         } else {
           this.log.trace(`Proven checkpoint ${provenCheckpointNumber} already stored.`);
         }

yarn-project/end-to-end/src/spartan/n_tps.test.ts

@@ -345,7 +345,9 @@ describe('sustained N TPS test', () => {
     let lowValueTxs = 0;
     const lowValueSendTx = async (wallet: TestWallet) => {
       lowValueTxs++;
-      const feeAmount = Number(randomBigInt(10n)) + 1;
+      //const feeAmount = Number(randomBigInt(100n)) + 1;
+      //const feeAmount = 1;
+      const feeAmount = Math.floor(lowValueTxs / 1000) + 1;
       const fee = new GasFees(0, feeAmount);
       logger.info('Sending low value tx ' + lowValueTxs + ' with fee ' + feeAmount);
 
@@ -358,7 +360,7 @@ describe('sustained N TPS test', () => {
     let highValueTxs = 0;
     const highValueSendTx = async (wallet: TestWallet) => {
       highValueTxs++;
-      const feeAmount = Number(randomBigInt(10n)) + 11;
+      const feeAmount = Number(randomBigInt(10n)) + 1000;
       const fee = new GasFees(0, feeAmount);
       logger.info('Sending high value tx ' + highValueTxs + ' with fee ' + feeAmount);
 

yarn-project/foundation/src/config/env_var.ts

@@ -215,6 +215,7 @@ export type EnvVar =
   | 'SEQ_BUILD_CHECKPOINT_IF_EMPTY'
   | 'SEQ_SECONDS_BEFORE_INVALIDATING_BLOCK_AS_COMMITTEE_MEMBER'
   | 'SEQ_SECONDS_BEFORE_INVALIDATING_BLOCK_AS_NON_COMMITTEE_MEMBER'
+  | 'SEQ_SKIP_CHECKPOINT_PUBLISH_PERCENT'
   | 'SLASH_MIN_PENALTY_PERCENTAGE'
   | 'SLASH_MAX_PENALTY_PERCENTAGE'
   | 'SLASH_VALIDATORS_ALWAYS'

yarn-project/p2p/src/client/interface.ts

@@ -107,13 +107,6 @@ export type P2P<T extends P2PClientType = P2PClientType.Full> = P2PApiFull<T> &
    **/
   sendTx(tx: Tx): Promise<void>;
 
-  /**
-   * Adds transactions to the pool. Does not send to peers or validate the tx.
-   * @param txs - The transactions.
-   * @returns The number of txs added to the pool. Note if the transaction already exists, it will not be added again.
-   **/
-  addTxsToPool(txs: Tx[]): Promise<number>;
-
   /**
    * Handles failed transaction execution by removing txs from the pool.
    * @param txHashes - Hashes of the transactions that failed execution.

yarn-project/p2p/src/client/p2p_client.ts

@@ -582,7 +582,7 @@ export class P2PClient<T extends P2PClientType = P2PClientType.Full>
    **/
   public async sendTx(tx: Tx): Promise<void> {
     this.#assertIsReady();
-    const result = await this.txPool.addPendingTxs([tx]);
+    const result = await this.txPool.addPendingTxs([tx], { feeComparisonOnly: true });
     if (result.accepted.length === 1) {
       await this.p2pService.propagate(tx);
     } else {
@@ -592,15 +592,6 @@ export class P2PClient<T extends P2PClientType = P2PClientType.Full>
     }
   }
 
-  /**
-   * Adds transactions to the pool. Does not send to peers or validate the txs.
-   * @param txs - The transactions.
-   **/
-  public async addTxsToPool(txs: Tx[]): Promise<number> {
-    this.#assertIsReady();
-    return (await this.txPool.addPendingTxs(txs)).accepted.length;
-  }
-
   /**
    * Returns whether the given tx hash is flagged as pending or mined.
    * @param txHash - Hash of the tx to query.

yarn-project/p2p/src/mem_pools/tx_pool_v2/deleted_pool.ts

@@ -301,6 +301,17 @@ export class DeletedPool {
     return this.#state.size;
   }
 
+  /** Gets the count of soft-deleted transactions (both prune-based and slot-based). */
+  getSoftDeletedCount(): number {
+    let count = this.#slotDeletedTxs.size;
+    for (const state of this.#state.values()) {
+      if (state.softDeleted) {
+        count++;
+      }
+    }
+    return count;
+  }
+
   /**
    * Gets all transaction hashes from pruned blocks.
    */

yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/eviction_manager.test.ts

@@ -183,6 +183,7 @@ describe('EvictionManager', () => {
       nullifiers: [`0x${txHash.slice(2)}null1`],
       includeByTimestamp: 0n,
       receivedAt: 0,
+      estimatedSizeBytes: 0,
       data: stubTxMetaValidationData(),
     });
 
@@ -204,7 +205,22 @@ describe('EvictionManager', () => {
 
       expect(result.shouldIgnore).toBe(false);
       expect(result.txHashesToEvict).toContain('0x2222');
-      expect(preAddRule.check).toHaveBeenCalledWith(incomingMeta, poolAccess);
+      expect(preAddRule.check).toHaveBeenCalledWith(incomingMeta, poolAccess, undefined);
+    });
+
+    it('forwards PreAddContext to rules', async () => {
+      preAddRule.check.mockResolvedValue({
+        shouldIgnore: false,
+        txHashesToEvict: [],
+      });
+
+      evictionManager.registerPreAddRule(preAddRule);
+      const incomingMeta = createMeta('0x1111', 100n);
+      const context = { feeComparisonOnly: true };
+
+      await evictionManager.runPreAddRules(incomingMeta, poolAccess, context);
+
+      expect(preAddRule.check).toHaveBeenCalledWith(incomingMeta, poolAccess, context);
     });
 
     it('returns ignore result immediately when a rule says to ignore', async () => {
@@ -318,6 +334,7 @@ describe('EvictionManager', () => {
         nullifiers: [`0x${txHash.slice(2)}null1`],
         includeByTimestamp: 0n,
         receivedAt: 0,
+        estimatedSizeBytes: 0,
         data: stubTxMetaValidationData(),
       });
 

yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/eviction_manager.ts

@@ -9,9 +9,11 @@ import {
   EvictionEvent,
   type EvictionRule,
   type PoolOperations,
+  type PreAddContext,
   type PreAddPoolAccess,
   type PreAddResult,
   type PreAddRule,
+  type TaggedEviction,
 } from './interfaces.js';
 
 /**
@@ -47,21 +49,27 @@ export class EvictionManager {
    * Runs all pre-add rules for an incoming transaction.
    * Returns combined result of all rules.
    */
-  async runPreAddRules(incomingMeta: TxMetaData, poolAccess: PreAddPoolAccess): Promise<PreAddResult> {
-    const allTxHashesToEvict: string[] = [];
+  async runPreAddRules(
+    incomingMeta: TxMetaData,
+    poolAccess: PreAddPoolAccess,
+    context?: PreAddContext,
+  ): Promise<PreAddResult> {
+    const evictions: TaggedEviction[] = [];
+    const seen = new Set<string>();
 
     for (const rule of this.preAddRules) {
       try {
-        const result = await rule.check(incomingMeta, poolAccess);
+        const result = await rule.check(incomingMeta, poolAccess, context);
 
         if (result.shouldIgnore) {
           return result;
         }
 
-        // Collect txs to evict from all rules
+        // Collect txs to evict from all rules, tagged with the rule name
         for (const txHash of result.txHashesToEvict) {
-          if (!allTxHashesToEvict.includes(txHash)) {
-            allTxHashesToEvict.push(txHash);
+          if (!seen.has(txHash)) {
+            seen.add(txHash);
+            evictions.push({ txHash, reason: rule.name });
           }
         }
       } catch (err) {
@@ -77,7 +85,8 @@ export class EvictionManager {
 
     return {
       shouldIgnore: false,
-      txHashesToEvict: allTxHashesToEvict,
+      txHashesToEvict: evictions.map(e => e.txHash),
+      evictions,
     };
   }
 

yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.test.ts

@@ -43,6 +43,7 @@ describe('FeePayerBalanceEvictionRule', () => {
     nullifiers: [`0x${txHash.slice(2)}null1`],
     includeByTimestamp: 0n,
     receivedAt: 0,
+    estimatedSizeBytes: 0,
     data: stubTxMetaValidationData(),
   });
 
@@ -144,7 +145,7 @@ describe('FeePayerBalanceEvictionRule', () => {
 
         expect(result.success).toBe(true);
         expect(result.txsEvicted).toEqual(['0x1111']); // Low priority evicted
-        expect(deleteTxsMock).toHaveBeenCalledWith(['0x1111']);
+        expect(deleteTxsMock).toHaveBeenCalledWith(['0x1111'], 'FeePayerBalanceEviction');
       });
 
       it('evicts multiple low-priority txs when balance is insufficient', async () => {
@@ -193,7 +194,7 @@ describe('FeePayerBalanceEvictionRule', () => {
 
         expect(result.success).toBe(true);
         expect(result.txsEvicted).toEqual(['0xaaaa']); // Only lowest priority evicted
-        expect(deleteTxsMock).toHaveBeenCalledWith(['0xaaaa']);
+        expect(deleteTxsMock).toHaveBeenCalledWith(['0xaaaa'], 'FeePayerBalanceEviction');
       });
 
       it('considers claim amount when calculating available balance', async () => {

yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.ts

@@ -67,8 +67,8 @@ export class FeePayerBalanceEvictionRule implements EvictionRule {
     ).flat();
 
     if (txsToEvict.length > 0) {
-      await pool.deleteTxs(txsToEvict);
-      this.log.verbose(`Evicted ${txsToEvict.length} txs due to insufficient fee payer balance`, {
+      await pool.deleteTxs(txsToEvict, this.name);
+      this.log.debug(`Evicted ${txsToEvict.length} txs due to insufficient fee payer balance`, {
         txHashes: txsToEvict,
       });
     }

yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.test.ts

@@ -24,6 +24,7 @@ describe('FeePayerBalancePreAddRule', () => {
     nullifiers: [`0x${txHash.slice(2)}null1`],
     includeByTimestamp: 0n,
     receivedAt: 0,
+    estimatedSizeBytes: 0,
     data: stubTxMetaValidationData(),
   });
 

yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.ts

@@ -1,7 +1,7 @@
 import { createLogger } from '@aztec/foundation/log';
 
 import { type TxMetaData, comparePriority } from '../tx_metadata.js';
-import type { PreAddPoolAccess, PreAddResult, PreAddRule } from './interfaces.js';
+import type { PreAddContext, PreAddPoolAccess, PreAddResult, PreAddRule } from './interfaces.js';
 
 /**
  * Pre-add rule that checks if a fee payer has sufficient balance to cover the incoming transaction.
@@ -19,7 +19,7 @@ export class FeePayerBalancePreAddRule implements PreAddRule {
 
   private log = createLogger('p2p:tx_pool_v2:fee_payer_balance_pre_add_rule');
 
-  async check(incomingMeta: TxMetaData, poolAccess: PreAddPoolAccess): Promise<PreAddResult> {
+  async check(incomingMeta: TxMetaData, poolAccess: PreAddPoolAccess, _context?: PreAddContext): Promise<PreAddResult> {
     // Get fee payer's on-chain balance
     const initialBalance = await poolAccess.getFeePayerBalance(incomingMeta.feePayer);
 

yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/index.ts

@@ -6,9 +6,11 @@ export {
   type EvictionResult,
   type EvictionRule,
   type PoolOperations,
+  type PreAddContext,
   type PreAddPoolAccess,
   type PreAddResult,
   type PreAddRule,
+  type TaggedEviction,
 } from './interfaces.js';
 
 // Pre-add rules