feat: Improved data storage metrics

barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp

@@ -254,4 +254,4 @@ TEST_F(LMDBTreeStoreTest, can_write_and_read_leaves_by_hash)
         success = store.read_leaf_by_hash(VALUES[9], readBack, *transaction);
         EXPECT_FALSE(success);
     }
-}
+}

yarn-project/archiver/src/archiver/archiver.ts

@@ -272,6 +272,9 @@ export class Archiver implements ArchiveSource {
       // the chain locally before we start unwinding stuff. This can be optimized by figuring out
       // up to which point we're pruning, and then requesting L2 blocks up to that point only.
       await this.handleEpochPrune(provenBlockNumber, currentL1BlockNumber);
+
+      const storeSizes = this.store.estimateSize();
+      this.instrumentation.recordDBMetrics(storeSizes);
     }
   }
 
@@ -1040,6 +1043,9 @@ class ArchiverStoreHelper
   getTotalL1ToL2MessageCount(): Promise<bigint> {
     return this.store.getTotalL1ToL2MessageCount();
   }
+  estimateSize(): { mappingSize: number; actualSize: number; numItems: number } {
+    return this.store.estimateSize();
+  }
 }
 
 type L1RollupConstants = {

yarn-project/archiver/src/archiver/archiver_store.ts

@@ -268,4 +268,9 @@ export interface ArchiverDataStore {
 
   addContractArtifact(address: AztecAddress, contract: ContractArtifact): Promise<void>;
   getContractArtifact(address: AztecAddress): Promise<ContractArtifact | undefined>;
+
+  /**
+   * Estimates the size of the store in bytes.
+   */
+  estimateSize(): { mappingSize: number; actualSize: number; numItems: number };
 }

yarn-project/archiver/src/archiver/instrumentation.ts

@@ -4,6 +4,7 @@ import {
   Attributes,
   type Gauge,
   type Histogram,
+  LmdbMetrics,
   Metrics,
   type TelemetryClient,
   type UpDownCounter,
@@ -18,6 +19,7 @@ export class ArchiverInstrumentation {
   private syncDuration: Histogram;
   private proofsSubmittedDelay: Histogram;
   private proofsSubmittedCount: UpDownCounter;
+  private dbMetrics: LmdbMetrics;
 
   private log = createDebugLogger('aztec:archiver:instrumentation');
 
@@ -55,6 +57,26 @@ export class ArchiverInstrumentation {
         explicitBucketBoundaries: millisecondBuckets(1, 80), // 10ms -> ~3hs
       },
     });
+
+    this.dbMetrics = new LmdbMetrics(
+      meter,
+      {
+        name: Metrics.ARCHIVER_DB_MAP_SIZE,
+        description: 'Database map size for the archiver',
+      },
+      {
+        name: Metrics.ARCHIVER_DB_USED_SIZE,
+        description: 'Database used size for the archiver',
+      },
+      {
+        name: Metrics.ARCHIVER_DB_NUM_ITEMS,
+        description: 'Num items in the archiver database',
+      },
+    );
+  }
+
+  public recordDBMetrics(metrics: { mappingSize: number; numItems: number; actualSize: number }) {
+    this.dbMetrics.recordDBMetrics(metrics);
   }
 
   public isEnabled(): boolean {

yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts

@@ -49,7 +49,7 @@ export class KVArchiverDataStore implements ArchiverDataStore {
 
   #log = createDebugLogger('aztec:archiver:data-store');
 
-  constructor(db: AztecKVStore, logsMaxPageSize: number = 1000) {
+  constructor(private db: AztecKVStore, logsMaxPageSize: number = 1000) {
     this.#blockStore = new BlockStore(db);
     this.#logStore = new LogStore(db, this.#blockStore, logsMaxPageSize);
     this.#messageStore = new MessageStore(db);
@@ -344,4 +344,8 @@ export class KVArchiverDataStore implements ArchiverDataStore {
       messagesSynchedTo: this.#messageStore.getSynchedL1BlockNumber(),
     });
   }
+
+  public estimateSize(): { mappingSize: number; actualSize: number; numItems: number } {
+    return this.db.estimateSize();
+  }
 }

yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts

@@ -736,4 +736,8 @@ export class MemoryArchiverStore implements ArchiverDataStore {
   public getContractArtifact(address: AztecAddress): Promise<ContractArtifact | undefined> {
     return Promise.resolve(this.contractArtifacts.get(address.toString()));
   }
+
+  public estimateSize(): { mappingSize: number; actualSize: number; numItems: number } {
+    return { mappingSize: 0, actualSize: 0, numItems: 0 };
+  }
 }

yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts

@@ -146,7 +146,12 @@ describe('L1Publisher integration', () => {
       worldStateProvenBlocksOnly: false,
       worldStateDbMapSizeKb: 10 * 1024 * 1024,
     };
-    worldStateSynchronizer = new ServerWorldStateSynchronizer(builderDb, blockSource, worldStateConfig);
+    worldStateSynchronizer = new ServerWorldStateSynchronizer(
+      builderDb,
+      blockSource,
+      worldStateConfig,
+      new NoopTelemetryClient(),
+    );
     await worldStateSynchronizer.start();
     fork = await worldStateSynchronizer.fork();
     builder = new LightweightBlockBuilder(fork, new NoopTelemetryClient());

yarn-project/kv-store/src/interfaces/store.ts

@@ -72,5 +72,5 @@ export interface AztecKVStore {
   /**
    * Estimates the size of the store in bytes.
    */
-  estimateSize(): { bytes: number };
+  estimateSize(): { mappingSize: number; actualSize: number; numItems: number };
 }

yarn-project/kv-store/src/lmdb/store.ts

@@ -182,15 +182,52 @@ export class AztecLmdbStore implements AztecKVStore {
     }
   }
 
-  estimateSize(): { bytes: number } {
+  estimateSize(): { mappingSize: number; actualSize: number; numItems: number } {
     const stats = this.#rootDb.getStats();
-    // `mapSize` represents to total amount of memory currently being used by the database.
-    // since the database is mmap'd, this is a good estimate of the size of the database for now.
+    // The 'mapSize' is the total amount of virtual address space allocated to the DB (effectively the maximum possible size)
     // http://www.lmdb.tech/doc/group__mdb.html#a4bde3c8b676457342cba2fe27aed5fbd
+    let mapSize = 0;
     if ('mapSize' in stats && typeof stats.mapSize === 'number') {
-      return { bytes: stats.mapSize };
-    } else {
-      return { bytes: 0 };
+      mapSize = stats.mapSize;
     }
+    const dataResult = this.estimateSubDBSize(this.#data);
+    const multiResult = this.estimateSubDBSize(this.#multiMapData);
+    return {
+      mappingSize: mapSize,
+      actualSize: dataResult.actualSize + multiResult.actualSize,
+      numItems: dataResult.numItems + multiResult.numItems,
+    };
+  }
+
+  private estimateSubDBSize(db: Database<unknown, Key>): { actualSize: number; numItems: number } {
+    const stats = db.getStats();
+    let branchPages = 0;
+    let leafPages = 0;
+    let overflowPages = 0;
+    let pageSize = 0;
+    let totalSize = 0;
+    let numItems = 0;
+    // This is the total number of key/value pairs present in the DB
+    if ('entryCount' in stats && typeof stats.entryCount === 'number') {
+      numItems = stats.entryCount;
+    }
+    // The closest value we can get to the actual size of the database is the number of consumed pages * the page size
+    if (
+      'treeBranchPageCount' in stats &&
+      typeof stats.treeBranchPageCount === 'number' &&
+      'treeLeafPageCount' in stats &&
+      typeof stats.treeLeafPageCount === 'number' &&
+      'overflowPages' in stats &&
+      typeof stats.overflowPages === 'number' &&
+      'pageSize' in stats &&
+      typeof stats.pageSize === 'number'
+    ) {
+      branchPages = stats.treeBranchPageCount;
+      leafPages = stats.treeLeafPageCount;
+      overflowPages = stats.overflowPages;
+      pageSize = stats.pageSize;
+      totalSize = (branchPages + leafPages + overflowPages) * pageSize;
+    }
+    return { actualSize: totalSize, numItems };
   }
 }

yarn-project/p2p/src/mem_pools/instrumentation.ts

@@ -1,5 +1,12 @@
 import { type Gossipable } from '@aztec/circuit-types';
-import { Attributes, type Histogram, Metrics, type TelemetryClient, type UpDownCounter } from '@aztec/telemetry-client';
+import {
+  Attributes,
+  type Histogram,
+  LmdbMetrics,
+  Metrics,
+  type TelemetryClient,
+  type UpDownCounter,
+} from '@aztec/telemetry-client';
 
 /**
  * Instrumentation class for the Pools (TxPool, AttestationPool, etc).
@@ -10,6 +17,8 @@ export class PoolInstrumentation<PoolObject extends Gossipable> {
   /** Tracks tx size */
   private objectSize: Histogram;
 
+  private dbMetrics: LmdbMetrics;
+
   private defaultAttributes;
 
   constructor(telemetry: TelemetryClient, name: string) {
@@ -35,6 +44,26 @@ export class PoolInstrumentation<PoolObject extends Gossipable> {
         ],
       },
     });
+
+    this.dbMetrics = new LmdbMetrics(
+      meter,
+      {
+        name: Metrics.MEMPOOL_DB_MAP_SIZE,
+        description: 'Database map size for the Tx mempool',
+      },
+      {
+        name: Metrics.MEMPOOL_DB_USED_SIZE,
+        description: 'Database used size for the Tx mempool',
+      },
+      {
+        name: Metrics.MEMPOOL_DB_NUM_ITEMS,
+        description: 'Num items in database for the Tx mempool',
+      },
+    );
+  }
+
+  public recordDBMetrics(metrics: { mappingSize: number; numItems: number; actualSize: number }) {
+    this.dbMetrics.recordDBMetrics(metrics);
   }
 
   public recordSize(poolObject: PoolObject) {

yarn-project/p2p/src/mem_pools/tx_pool/aztec_kv_tx_pool.ts

@@ -53,6 +53,8 @@ export class AztecKVTxPool implements TxPool {
       }
       this.#metrics.recordRemovedObjects(deleted, 'pending');
       this.#metrics.recordAddedObjects(txHashes.length, 'mined');
+      const storeSizes = this.#store.estimateSize();
+      this.#metrics.recordDBMetrics(storeSizes);
     });
   }
 

yarn-project/telemetry-client/src/index.ts

@@ -2,3 +2,4 @@ export * from './telemetry.js';
 export * from './histogram_utils.js';
 export * from './with_tracer.js';
 export * from './prom_otel_adapter.js';
+export * from './lmdb_metrics.js';

yarn-project/telemetry-client/src/lmdb_metrics.ts

@@ -0,0 +1,38 @@
+import { type Gauge, type Meter, type Metrics, ValueType } from './telemetry.js';
+
+export type LmdbMetricDescriptor = {
+  name: Metrics;
+  description: string;
+};
+
+export class LmdbMetrics {
+  private dbMapSize: Gauge;
+  private dbUsedSize: Gauge;
+  private dbNumItems: Gauge;
+
+  constructor(
+    meter: Meter,
+    dbMapSizeDescriptor: LmdbMetricDescriptor,
+    dbUsedSizeDescriptor: LmdbMetricDescriptor,
+    dbNumItemsDescriptor: LmdbMetricDescriptor,
+  ) {
+    this.dbMapSize = meter.createGauge(dbMapSizeDescriptor.name, {
+      description: dbMapSizeDescriptor.description,
+      valueType: ValueType.INT,
+    });
+    this.dbUsedSize = meter.createGauge(dbUsedSizeDescriptor.name, {
+      description: dbUsedSizeDescriptor.description,
+      valueType: ValueType.INT,
+    });
+    this.dbNumItems = meter.createGauge(dbNumItemsDescriptor.name, {
+      description: dbNumItemsDescriptor.description,
+      valueType: ValueType.INT,
+    });
+  }
+
+  public recordDBMetrics(metrics: { mappingSize: number; numItems: number; actualSize: number }) {
+    this.dbMapSize.record(metrics.mappingSize);
+    this.dbNumItems.record(metrics.actualSize);
+    this.dbUsedSize.record(metrics.actualSize);
+  }
+}