diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp index 717c243fe434..91045ddecda7 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp @@ -103,6 +103,7 @@ const std::string& LMDBTreeStore::get_name() const void LMDBTreeStore::get_stats(TreeDBStats& stats, ReadTransaction& tx) { stats.mapSize = _environment->get_map_size(); + stats.physicalFileSize = _environment->get_data_file_size(); stats.blocksDBStats = _blockDatabase->get_stats(tx); stats.leafPreimagesDBStats = _leafHashToPreImageDatabase->get_stats(tx); stats.leafIndicesDBStats = _leafKeyToIndexDatabase->get_stats(tx); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp index 7a06f2f64a2a..a6f17169b92b 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp @@ -585,3 +585,49 @@ TEST_F(LMDBTreeStoreTest, can_write_and_retrieve_block_numbers_with_duplicate_in EXPECT_EQ(readBack, 5); } } + +TEST_F(LMDBTreeStoreTest, reports_physical_file_size) +{ + LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); + std::string dataDbPath = _directory + "/data.mdb"; + size_t previousFileSize = 0; + + for (size_t i = 0; i < 3; i++) { + { + BlockPayload blockData; + blockData.blockNumber = i; + blockData.root = VALUES[i]; + blockData.size = 45 + (i * 97); + + TreeMeta metaData; + metaData.committedSize = blockData.size; + metaData.size = blockData.size; + metaData.root = blockData.root; + metaData.depth = 32; + metaData.unfinalisedBlockHeight = i; + metaData.name = "NullifierTree"; + + // Write metadata and block data with different values each iteration + LMDBWriteTransaction::Ptr transaction = store.create_write_transaction(); + store.write_meta_data(metaData, *transaction); + store.write_block_data(i, blockData, *transaction); + transaction->commit(); + } + + { + LMDBReadTransaction::Ptr transaction = store.create_read_transaction(); + TreeDBStats stats; + store.get_stats(stats, *transaction); + + EXPECT_TRUE(std::filesystem::exists(dataDbPath)); + + // Verify reported size matches actual file size + EXPECT_EQ(stats.physicalFileSize, std::filesystem::file_size(dataDbPath)); + + // Verify size is increasing due to the new DB writes + EXPECT_GT(stats.physicalFileSize, previousFileSize); + + previousFileSize = stats.physicalFileSize; + } + } +} diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp index ee14993d53d4..a363e1181623 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp @@ -67,6 +67,7 @@ const std::string BLOCK_INDICES_DB = "block indices"; struct TreeDBStats { uint64_t mapSize; + uint64_t physicalFileSize; DBStats blocksDBStats; DBStats nodesDBStats; DBStats leafPreimagesDBStats; @@ -74,16 +75,19 @@ struct TreeDBStats { DBStats blockIndicesDBStats; TreeDBStats() = default; - TreeDBStats(uint64_t mapSize) + TreeDBStats(uint64_t mapSize, uint64_t physicalFileSize) : mapSize(mapSize) + , physicalFileSize(physicalFileSize) {} TreeDBStats(uint64_t mapSize, + uint64_t physicalFileSize, const DBStats& blockStats, const DBStats& nodesStats, const DBStats& leafPreimagesDBStats, const DBStats& leafIndicesStats, const DBStats& blockIndicesStats) : mapSize(mapSize) + , physicalFileSize(physicalFileSize) , blocksDBStats(blockStats) , nodesDBStats(nodesStats) , leafPreimagesDBStats(leafPreimagesDBStats) @@ -95,11 +99,18 @@ struct TreeDBStats { ~TreeDBStats() = default; - MSGPACK_FIELDS(mapSize, blocksDBStats, nodesDBStats, leafPreimagesDBStats, leafIndicesDBStats, blockIndicesDBStats) + MSGPACK_FIELDS(mapSize, + physicalFileSize, + blocksDBStats, + nodesDBStats, + leafPreimagesDBStats, + leafIndicesDBStats, + blockIndicesDBStats) bool operator==(const TreeDBStats& other) const { - return mapSize == other.mapSize && blocksDBStats == other.blocksDBStats && nodesDBStats == other.nodesDBStats && + return mapSize == other.mapSize && physicalFileSize == other.physicalFileSize && + blocksDBStats == other.blocksDBStats && nodesDBStats == other.nodesDBStats && leafPreimagesDBStats == other.leafPreimagesDBStats && leafIndicesDBStats == other.leafIndicesDBStats && blockIndicesDBStats == other.blockIndicesDBStats; } @@ -108,6 +119,7 @@ struct TreeDBStats { { if (this != &other) { mapSize = other.mapSize; + physicalFileSize = other.physicalFileSize; blocksDBStats = std::move(other.blocksDBStats); nodesDBStats = std::move(other.nodesDBStats); leafPreimagesDBStats = std::move(other.leafPreimagesDBStats); @@ -121,9 +133,10 @@ struct TreeDBStats { friend std::ostream& operator<<(std::ostream& os, const TreeDBStats& stats) { - os << "Map Size: " << stats.mapSize << " Blocks DB " << stats.blocksDBStats << ", Nodes DB " - << stats.nodesDBStats << ", Leaf Pre-images DB " << stats.leafPreimagesDBStats << ", Leaf Indices DB " - << stats.leafIndicesDBStats << ", Block Indices DB " << stats.blockIndicesDBStats; + os << "Map Size: " << stats.mapSize << ", Physical File Size: " << stats.physicalFileSize << " Blocks DB " + << stats.blocksDBStats << ", Nodes DB " << stats.nodesDBStats << ", Leaf Pre-images DB " + << stats.leafPreimagesDBStats << ", Leaf Indices DB " << stats.leafIndicesDBStats << ", Block Indices DB " + << stats.blockIndicesDBStats; return os; } }; diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_environment.cpp b/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_environment.cpp index 921dfdfdcc39..99571f661c49 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_environment.cpp +++ b/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_environment.cpp @@ -2,6 +2,7 @@ #include "barretenberg/lmdblib/lmdb_helpers.hpp" #include "lmdb.h" #include +#include #include #include @@ -12,6 +13,7 @@ LMDBEnvironment::LMDBEnvironment(const std::string& directory, uint32_t maxNumDBs, uint32_t maxNumReaders) : _id(0) + , _directory(directory) , _readGuard(maxNumReaders) , _writeGuard(1) // LMDB only permits one write transaction at a time { @@ -71,4 +73,13 @@ uint64_t LMDBEnvironment::get_map_size() const call_lmdb_func(mdb_env_info, _mdbEnv, &info); return info.me_mapsize; } + +uint64_t LMDBEnvironment::get_data_file_size() const +{ + std::string dataPath = (std::filesystem::path(_directory) / "data.mdb").string(); + if (std::filesystem::exists(dataPath)) { + return std::filesystem::file_size(dataPath); + } + return 0; +} } // namespace bb::lmdblib diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_environment.hpp b/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_environment.hpp index 9cea0e5edbb4..c87054028ac4 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_environment.hpp +++ b/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_environment.hpp @@ -48,8 +48,11 @@ class LMDBEnvironment { uint64_t get_map_size() const; + uint64_t get_data_file_size() const; + private: std::atomic_uint64_t _id; + std::string _directory; MDB_env* _mdbEnv; struct ResourceGuard { @@ -82,4 +85,4 @@ class LMDBEnvironment { ResourceGuard _readGuard; ResourceGuard _writeGuard; }; -} // namespace bb::lmdblib \ No newline at end of file +} // namespace bb::lmdblib diff --git a/yarn-project/telemetry-client/src/metrics.ts b/yarn-project/telemetry-client/src/metrics.ts index e830b6645735..d60d1ea476c0 100644 --- a/yarn-project/telemetry-client/src/metrics.ts +++ b/yarn-project/telemetry-client/src/metrics.ts @@ -150,6 +150,7 @@ export const WORLD_STATE_SYNC_DURATION = 'aztec.world_state.sync.duration'; export const WORLD_STATE_MERKLE_TREE_SIZE = 'aztec.world_state.merkle_tree_size'; export const WORLD_STATE_DB_SIZE = 'aztec.world_state.db_size'; export const WORLD_STATE_DB_MAP_SIZE = 'aztec.world_state.db_map_size'; +export const WORLD_STATE_DB_PHYSICAL_SIZE = 'aztec.world_state.db_physical_size'; export const WORLD_STATE_TREE_SIZE = 'aztec.world_state.tree_size'; export const WORLD_STATE_UNFINALISED_HEIGHT = 'aztec.world_state.unfinalised_height'; export const WORLD_STATE_FINALISED_HEIGHT = 'aztec.world_state.finalised_height'; diff --git a/yarn-project/world-state/src/instrumentation/instrumentation.ts b/yarn-project/world-state/src/instrumentation/instrumentation.ts index 8126acc262ce..21252c13d88f 100644 --- a/yarn-project/world-state/src/instrumentation/instrumentation.ts +++ b/yarn-project/world-state/src/instrumentation/instrumentation.ts @@ -31,6 +31,7 @@ const durationTrackDenylist = new Set([ export class WorldStateInstrumentation { private dbMapSize: Gauge; + private dbPhysicalSize: Gauge; private treeSize: Gauge; private unfinalisedHeight: Gauge; private finalisedHeight: Gauge; @@ -50,6 +51,11 @@ export class WorldStateInstrumentation { valueType: ValueType.INT, }); + this.dbPhysicalSize = meter.createGauge(Metrics.WORLD_STATE_DB_PHYSICAL_SIZE, { + description: `The current physical disk space used for each database`, + valueType: ValueType.INT, + }); + this.treeSize = meter.createGauge(Metrics.WORLD_STATE_TREE_SIZE, { description: `The current number of leaves in each merkle tree`, valueType: ValueType.INT, @@ -96,6 +102,9 @@ export class WorldStateInstrumentation { this.dbMapSize.record(Number(treeDbStats.mapSize), { [Attributes.MERKLE_TREE_NAME]: MerkleTreeId[tree], }); + this.dbPhysicalSize.record(Number(treeDbStats.physicalFileSize), { + [Attributes.MERKLE_TREE_NAME]: MerkleTreeId[tree], + }); this.treeSize.record(Number(treeMeta.size), { [Attributes.MERKLE_TREE_NAME]: MerkleTreeId[tree], }); diff --git a/yarn-project/world-state/src/native/message.ts b/yarn-project/world-state/src/native/message.ts index 12b9b1574d0a..eb55a3a93038 100644 --- a/yarn-project/world-state/src/native/message.ts +++ b/yarn-project/world-state/src/native/message.ts @@ -96,6 +96,8 @@ export interface DBStats { export interface TreeDBStats { /** The configured max size of the DB mapping file (effectively the max possible size of the DB) */ mapSize: bigint; + /** The physical file size of the database on disk */ + physicalFileSize: bigint; /** Stats for the 'blocks' DB */ blocksDBStats: DBStats; /** Stats for the 'nodes' DB */ @@ -151,6 +153,7 @@ export function buildEmptyDBStats() { export function buildEmptyTreeDBStats() { return { mapSize: 0n, + physicalFileSize: 0n, blocksDBStats: buildEmptyDBStats(), nodesDBStats: buildEmptyDBStats(), leafIndicesDBStats: buildEmptyDBStats(), @@ -242,6 +245,7 @@ export function sanitiseTreeDBStats(stats: TreeDBStats) { stats.blockIndicesDBStats = sanitiseDBStats(stats.blockIndicesDBStats); stats.nodesDBStats = sanitiseDBStats(stats.nodesDBStats); stats.mapSize = BigInt(stats.mapSize); + stats.physicalFileSize = BigInt(stats.physicalFileSize); return stats; }