diff --git a/yarn-project/bb-prover/src/avm_proving_tests/avm_proving_tester.ts b/yarn-project/bb-prover/src/avm_proving_tests/avm_proving_tester.ts index 6e4b3fb181d5..dfb4457d30a4 100644 --- a/yarn-project/bb-prover/src/avm_proving_tests/avm_proving_tester.ts +++ b/yarn-project/bb-prover/src/avm_proving_tests/avm_proving_tester.ts @@ -1,8 +1,7 @@ import { type MerkleTreeWriteOperations } from '@aztec/circuit-types'; import { type AvmCircuitInputs, AztecAddress, VerificationKeyData } from '@aztec/circuits.js'; -import { openTmpStore } from '@aztec/kv-store/lmdb'; import { PublicTxSimulationTester, type TestEnqueuedCall } from '@aztec/simulator/public/fixtures'; -import { MerkleTrees } from '@aztec/world-state'; +import { NativeWorldStateService } from '@aztec/world-state'; import fs from 'node:fs/promises'; import { tmpdir } from 'node:os'; @@ -37,7 +36,7 @@ export class AvmProvingTester extends PublicTxSimulationTester { const bbWorkingDirectory = await fs.mkdtemp(path.join(tmpdir(), 'bb-')); const contractDataSource = new SimpleContractDataSource(); - const merkleTrees = await (await MerkleTrees.new(openTmpStore())).fork(); + const merkleTrees = await (await NativeWorldStateService.tmp()).fork(); return new AvmProvingTester( bbWorkingDirectory, checkCircuitOnly, @@ -122,7 +121,7 @@ export class AvmProvingTesterV2 extends PublicTxSimulationTester { const bbWorkingDirectory = await fs.mkdtemp(path.join(tmpdir(), 'bb-')); const contractDataSource = new SimpleContractDataSource(); - const merkleTrees = await (await MerkleTrees.new(openTmpStore())).fork(); + const merkleTrees = await (await NativeWorldStateService.tmp()).fork(); return new AvmProvingTesterV2(bbWorkingDirectory, contractDataSource, merkleTrees, skipContractDeployments); } diff --git a/yarn-project/simulator/src/avm/avm_simulator.test.ts b/yarn-project/simulator/src/avm/avm_simulator.test.ts index ea7197eba75a..ab3b1946c344 100644 --- a/yarn-project/simulator/src/avm/avm_simulator.test.ts +++ b/yarn-project/simulator/src/avm/avm_simulator.test.ts @@ -21,9 +21,7 @@ import { AztecAddress } from '@aztec/foundation/aztec-address'; import { keccak256, keccakf1600, pedersenCommit, pedersenHash, poseidon2Hash, sha256 } from '@aztec/foundation/crypto'; import { Fq, Fr, Point } from '@aztec/foundation/fields'; import { type Fieldable } from '@aztec/foundation/serialize'; -import { openTmpStore } from '@aztec/kv-store/lmdb'; -import { getTelemetryClient } from '@aztec/telemetry-client'; -import { MerkleTrees } from '@aztec/world-state'; +import { NativeWorldStateService } from '@aztec/world-state'; import { randomInt } from 'crypto'; import { mock } from 'jest-mock-extended'; @@ -1101,9 +1099,7 @@ describe('AVM simulator: transpiled Noir contracts', () => { trace = mock(); worldStateDB = mock(); - const tmp = openTmpStore(); - const telemetryClient = getTelemetryClient(); - merkleTrees = await (await MerkleTrees.new(tmp, telemetryClient)).fork(); + merkleTrees = await (await NativeWorldStateService.tmp()).fork(); (worldStateDB as jest.Mocked).getMerkleInterface.mockReturnValue(merkleTrees); ephemeralForest = await AvmEphemeralForest.create(worldStateDB.getMerkleInterface()); diff --git a/yarn-project/simulator/src/avm/avm_tree.test.ts b/yarn-project/simulator/src/avm/avm_tree.test.ts index f80de76b8269..18f839027792 100644 --- a/yarn-project/simulator/src/avm/avm_tree.test.ts +++ b/yarn-project/simulator/src/avm/avm_tree.test.ts @@ -16,9 +16,7 @@ import { import { poseidon2Hash } from '@aztec/foundation/crypto'; import { Fr } from '@aztec/foundation/fields'; import { type IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; -import { openTmpStore } from '@aztec/kv-store/lmdb'; -import { getTelemetryClient } from '@aztec/telemetry-client'; -import { MerkleTrees, NativeWorldStateService } from '@aztec/world-state'; +import { NativeWorldStateService } from '@aztec/world-state'; import { AvmEphemeralForest, @@ -531,8 +529,7 @@ describe('Checking forking and merging', () => { */ describe('AVM Ephemeral Tree Sanity Test', () => { it('Should calculate the frontier correctly', async () => { - const store = openTmpStore(true); - const worldStateTrees = await MerkleTrees.new(store, getTelemetryClient()); + const worldStateTrees = await NativeWorldStateService.tmp(); const leaves = []; const numLeaves = 6; for (let i = 0; i < numLeaves; i++) { @@ -557,8 +554,9 @@ describe('AVM Ephemeral Tree Sanity Test', () => { const expectedFrontier = [expectedFrontier0, exepctedFrontier1, expectedFrontier2]; expect(tree.frontier).toEqual(expectedFrontier); // Check root - await worldStateTrees.appendLeaves(MerkleTreeId.NOTE_HASH_TREE, leaves); - const treeInfo = await worldStateTrees.getTreeInfo(MerkleTreeId.NOTE_HASH_TREE, true); + const fork = await worldStateTrees.fork(); + await fork.appendLeaves(MerkleTreeId.NOTE_HASH_TREE, leaves); + const treeInfo = await fork.getTreeInfo(MerkleTreeId.NOTE_HASH_TREE); const localRoot = await tree.getRoot(); expect(localRoot.toBuffer()).toEqual(treeInfo.root); }); diff --git a/yarn-project/simulator/src/avm/fixtures/avm_simulation_tester.ts b/yarn-project/simulator/src/avm/fixtures/avm_simulation_tester.ts index decb5d26e5a0..479c36fce387 100644 --- a/yarn-project/simulator/src/avm/fixtures/avm_simulation_tester.ts +++ b/yarn-project/simulator/src/avm/fixtures/avm_simulation_tester.ts @@ -3,8 +3,7 @@ import { GasFees, GlobalVariables } from '@aztec/circuits.js'; import { encodeArguments } from '@aztec/foundation/abi'; import { type AztecAddress } from '@aztec/foundation/aztec-address'; import { Fr } from '@aztec/foundation/fields'; -import { openTmpStore } from '@aztec/kv-store/lmdb'; -import { MerkleTrees } from '@aztec/world-state'; +import { NativeWorldStateService } from '@aztec/world-state'; import { type AvmContractCallResult } from '../../avm/avm_contract_call_result.js'; import { @@ -40,7 +39,7 @@ export class AvmSimulationTester extends BaseAvmSimulationTester { static async create(skipContractDeployments = false): Promise { const contractDataSource = new SimpleContractDataSource(); - const merkleTrees = await (await MerkleTrees.new(openTmpStore())).fork(); + const merkleTrees = await (await NativeWorldStateService.tmp()).fork(); const worldStateDB = new WorldStateDB(merkleTrees, contractDataSource); const trace = new SideEffectTrace(); const firstNullifier = new Fr(420000); diff --git a/yarn-project/simulator/src/public/fixtures/public_tx_simulation_tester.ts b/yarn-project/simulator/src/public/fixtures/public_tx_simulation_tester.ts index d6c30e8158d4..985a63d771c2 100644 --- a/yarn-project/simulator/src/public/fixtures/public_tx_simulation_tester.ts +++ b/yarn-project/simulator/src/public/fixtures/public_tx_simulation_tester.ts @@ -15,9 +15,8 @@ import { type ContractArtifact, encodeArguments } from '@aztec/foundation/abi'; import { AztecAddress } from '@aztec/foundation/aztec-address'; import { padArrayEnd } from '@aztec/foundation/collection'; import { Fr } from '@aztec/foundation/fields'; -import { openTmpStore } from '@aztec/kv-store/lmdb'; import { AvmTestContractArtifact } from '@aztec/noir-contracts.js/AvmTest'; -import { MerkleTrees } from '@aztec/world-state'; +import { NativeWorldStateService } from '@aztec/world-state'; import { BaseAvmSimulationTester } from '../../avm/fixtures/base_avm_simulation_tester.js'; import { getContractFunctionArtifact, getFunctionSelector } from '../../avm/fixtures/index.js'; @@ -46,7 +45,7 @@ export class PublicTxSimulationTester extends BaseAvmSimulationTester { public static async create(skipContractDeployments = false): Promise { const contractDataSource = new SimpleContractDataSource(); - const merkleTrees = await (await MerkleTrees.new(openTmpStore())).fork(); + const merkleTrees = await (await NativeWorldStateService.tmp()).fork(); return new PublicTxSimulationTester(contractDataSource, merkleTrees, skipContractDeployments); } diff --git a/yarn-project/simulator/src/public/public_tx_simulator.test.ts b/yarn-project/simulator/src/public/public_tx_simulator.test.ts index c422171e0b00..36e0ca3fff63 100644 --- a/yarn-project/simulator/src/public/public_tx_simulator.test.ts +++ b/yarn-project/simulator/src/public/public_tx_simulator.test.ts @@ -36,7 +36,7 @@ import { type AztecKVStore } from '@aztec/kv-store'; import { openTmpStore } from '@aztec/kv-store/lmdb'; import { type AppendOnlyTree, Poseidon, StandardTree, newTree } from '@aztec/merkle-tree'; import { ProtocolContractAddress, REGISTERER_CONTRACT_CLASS_REGISTERED_TAG } from '@aztec/protocol-contracts'; -import { MerkleTrees } from '@aztec/world-state'; +import { NativeWorldStateService } from '@aztec/world-state'; import { jest } from '@jest/globals'; import { mock } from 'jest-mock-extended'; @@ -283,8 +283,7 @@ describe('public_tx_simulator', () => { }; beforeEach(async () => { - const tmp = openTmpStore(); - db = await (await MerkleTrees.new(tmp)).fork(); + db = await (await NativeWorldStateService.tmp()).fork(); worldStateDB = new WorldStateDB(db, mock()); treeStore = openTmpStore(); diff --git a/yarn-project/world-state/package.json b/yarn-project/world-state/package.json index 102155c5af75..7f063c1de493 100644 --- a/yarn-project/world-state/package.json +++ b/yarn-project/world-state/package.json @@ -5,6 +5,7 @@ "exports": { ".": "./dest/index.js", "./native": "./dest/native/index.js", + "./test": "./dest/test/index.js", "./config": "./dest/synchronizer/config.js" }, "typedocOptions": { diff --git a/yarn-project/world-state/src/native/native_world_state_cmp.test.ts b/yarn-project/world-state/src/native/native_world_state_cmp.test.ts deleted file mode 100644 index 676d184fa609..000000000000 --- a/yarn-project/world-state/src/native/native_world_state_cmp.test.ts +++ /dev/null @@ -1,258 +0,0 @@ -import { - type FrTreeId, - type IndexedTreeId, - MerkleTreeId, - type MerkleTreeReadOperations, - type MerkleTreeWriteOperations, -} from '@aztec/circuit-types'; -import { EthAddress, Fr, GENESIS_ARCHIVE_ROOT, NullifierLeaf, PublicDataTreeLeaf } from '@aztec/circuits.js'; -import { type Logger, createLogger } from '@aztec/foundation/log'; -import { elapsed } from '@aztec/foundation/timer'; -import { type AztecKVStore } from '@aztec/kv-store'; -import { AztecLmdbStore } from '@aztec/kv-store/lmdb'; -import { getTelemetryClient } from '@aztec/telemetry-client'; - -import { jest } from '@jest/globals'; -import { mkdtemp, rm } from 'fs/promises'; -import { tmpdir } from 'os'; -import { join } from 'path'; - -import { mockBlock } from '../test/utils.js'; -import { MerkleTrees } from '../world-state-db/merkle_trees.js'; -import { NativeWorldStateService } from './native_world_state.js'; - -jest.setTimeout(60_000); - -describe('NativeWorldState', () => { - let nativeDataDir: string; - let legacyDataDir: string; - - let nativeWS: NativeWorldStateService; - let legacyWS: MerkleTrees; - - let log: Logger; - - let legacyStore: AztecKVStore; - - const allTrees = Object.values(MerkleTreeId) - .filter((x): x is MerkleTreeId => typeof x === 'number') - .map(x => [MerkleTreeId[x], x] as const); - - beforeAll(async () => { - nativeDataDir = await mkdtemp(join(tmpdir(), 'native_world_state_test-')); - legacyDataDir = await mkdtemp(join(tmpdir(), 'js_world_state_test-')); - - log = createLogger('world-state:test:native_world_state_cmp'); - }); - - afterAll(async () => { - await legacyStore.delete(); - await rm(nativeDataDir, { recursive: true }); - }); - - beforeAll(async () => { - legacyStore = AztecLmdbStore.open(legacyDataDir); - nativeWS = await NativeWorldStateService.new(EthAddress.random(), nativeDataDir, 1024 * 1024); - legacyWS = await MerkleTrees.new(legacyStore, getTelemetryClient()); - }); - - it('has to expected genesis archive tree root', async () => { - const archive = await nativeWS.getCommitted().getTreeInfo(MerkleTreeId.ARCHIVE); - expect(new Fr(archive.root)).toEqual(new Fr(GENESIS_ARCHIVE_ROOT)); - }); - - describe('Initial state', () => { - it.each(allTrees)('tree %s is the same', async (_, treeId) => { - await assertSameTree(treeId, nativeWS.getCommitted(), legacyWS.getCommitted()); - }); - - it('initial state is the same', async () => { - await assertSameState(nativeWS.getCommitted(), legacyWS.getCommitted()); - }); - }); - - describe('Uncommitted state', () => { - let nativeFork: MerkleTreeWriteOperations; - let legacyLatest: MerkleTreeWriteOperations; - - beforeEach(async () => { - nativeFork = await nativeWS.fork(); - legacyLatest = await legacyWS.getLatest(); - }); - - afterEach(async () => { - await Promise.all([nativeFork.close(), legacyWS.rollback()]); - }); - - it.each<[string, IndexedTreeId, Buffer[]]>([ - [ - MerkleTreeId[MerkleTreeId.NULLIFIER_TREE], - MerkleTreeId.NULLIFIER_TREE, - Array(64).fill(new NullifierLeaf(Fr.ZERO).toBuffer()), - ], - [ - MerkleTreeId[MerkleTreeId.PUBLIC_DATA_TREE], - MerkleTreeId.PUBLIC_DATA_TREE, - Array(64).fill(new PublicDataTreeLeaf(Fr.ZERO, Fr.ZERO).toBuffer()), - ], - ])('inserting 0 leaves into %s', async (_, treeId, leaves) => { - const height = Math.ceil(Math.log2(leaves.length) | 0); - const [native, js] = await Promise.all([ - nativeFork.batchInsert(treeId, leaves, height), - legacyLatest.batchInsert(treeId, leaves, height), - ]); - - expect(native.sortedNewLeaves.map(Fr.fromBuffer)).toEqual(js.sortedNewLeaves.map(Fr.fromBuffer)); - expect(native.sortedNewLeavesIndexes).toEqual(js.sortedNewLeavesIndexes); - expect(native.newSubtreeSiblingPath.toFields()).toEqual(js.newSubtreeSiblingPath.toFields()); - expect(native.lowLeavesWitnessData).toEqual(js.lowLeavesWitnessData); - - await assertSameTree(treeId, nativeFork, legacyLatest); - }); - - it.each<[string, FrTreeId, Fr[]]>([ - [MerkleTreeId[MerkleTreeId.NOTE_HASH_TREE], MerkleTreeId.NOTE_HASH_TREE, Array(64).fill(Fr.ZERO)], - [MerkleTreeId[MerkleTreeId.L1_TO_L2_MESSAGE_TREE], MerkleTreeId.NOTE_HASH_TREE, Array(64).fill(Fr.ZERO)], - ])('inserting 0 leaves into %s', async (_, treeId, leaves) => { - await Promise.all([nativeFork.appendLeaves(treeId, leaves), legacyLatest.appendLeaves(treeId, leaves)]); - await assertSameTree(treeId, nativeFork, legacyLatest); - }); - - it.each<[string, IndexedTreeId, Buffer[]]>([ - [ - MerkleTreeId[MerkleTreeId.NULLIFIER_TREE], - MerkleTreeId.NULLIFIER_TREE, - Array(64) - .fill(0) - .map(() => new NullifierLeaf(Fr.random()).toBuffer()), - ], - [ - MerkleTreeId[MerkleTreeId.PUBLIC_DATA_TREE], - MerkleTreeId.PUBLIC_DATA_TREE, - Array(64) - .fill(0) - .map(() => new PublicDataTreeLeaf(Fr.random(), Fr.random()).toBuffer()), - ], - ])('inserting real leaves into %s', async (_, treeId, leaves) => { - const height = Math.ceil(Math.log2(leaves.length) | 0); - const [native, js] = await Promise.all([ - nativeFork.batchInsert(treeId, leaves, height), - legacyLatest.batchInsert(treeId, leaves, height), - ]); - - expect(native.sortedNewLeaves.map(Fr.fromBuffer)).toEqual(js.sortedNewLeaves.map(Fr.fromBuffer)); - expect(native.sortedNewLeavesIndexes).toEqual(js.sortedNewLeavesIndexes); - expect(native.newSubtreeSiblingPath.toFields()).toEqual(js.newSubtreeSiblingPath.toFields()); - expect(native.lowLeavesWitnessData).toEqual(js.lowLeavesWitnessData); - - await assertSameTree(treeId, nativeFork, legacyLatest); - }); - - it.each<[string, FrTreeId, Fr[]]>([ - [MerkleTreeId[MerkleTreeId.NOTE_HASH_TREE], MerkleTreeId.NOTE_HASH_TREE, Array(64).fill(0).map(Fr.random)], - [MerkleTreeId[MerkleTreeId.L1_TO_L2_MESSAGE_TREE], MerkleTreeId.NOTE_HASH_TREE, Array(64).fill(0).map(Fr.random)], - ])('inserting real leaves into %s', async (_, treeId, leaves) => { - await Promise.all([nativeFork.appendLeaves(treeId, leaves), legacyLatest.appendLeaves(treeId, leaves)]); - - await assertSameTree(treeId, nativeFork, legacyLatest); - }); - - it.each<[string, MerkleTreeId, number[]]>([ - [MerkleTreeId[MerkleTreeId.NULLIFIER_TREE], MerkleTreeId.NULLIFIER_TREE, [0, 1, 10, 127, 128, 256]], - [MerkleTreeId[MerkleTreeId.NOTE_HASH_TREE], MerkleTreeId.NOTE_HASH_TREE, [0, 1, 10, 127, 128, 256]], - [MerkleTreeId[MerkleTreeId.PUBLIC_DATA_TREE], MerkleTreeId.PUBLIC_DATA_TREE, [0, 1, 10, 127, 128, 256]], - [MerkleTreeId[MerkleTreeId.L1_TO_L2_MESSAGE_TREE], MerkleTreeId.NOTE_HASH_TREE, [0, 1, 10, 127, 128, 256]], - ])('sibling paths to initial leaves match', async (_, treeId, leafIndices) => { - for (const index of leafIndices) { - const [nativeSB, legacySB] = await Promise.all([ - nativeFork.getSiblingPath(treeId, BigInt(index)), - legacyLatest.getSiblingPath(treeId, BigInt(index)), - ]); - expect(nativeSB.toFields()).toEqual(legacySB.toFields()); - } - }); - - it.each<[string, IndexedTreeId, Buffer, (b: Buffer) => bigint]>([ - [ - MerkleTreeId[MerkleTreeId.NULLIFIER_TREE], - MerkleTreeId.NULLIFIER_TREE, - new NullifierLeaf(Fr.random()).toBuffer(), - b => NullifierLeaf.fromBuffer(b).getKey(), - ], - [ - MerkleTreeId[MerkleTreeId.PUBLIC_DATA_TREE], - MerkleTreeId.PUBLIC_DATA_TREE, - new PublicDataTreeLeaf(Fr.random(), Fr.random()).toBuffer(), - b => PublicDataTreeLeaf.fromBuffer(b).getKey(), - ], - ])('inserting real leaves into %s', async (_, treeId, leaf, getKey) => { - await Promise.all([nativeFork.batchInsert(treeId, [leaf], 0), legacyLatest.batchInsert(treeId, [leaf], 0)]); - - const [nativeLeafIndex, legacyLeafIndex] = await Promise.all([ - nativeFork.getPreviousValueIndex(treeId, getKey(leaf)), - legacyLatest.getPreviousValueIndex(treeId, getKey(leaf)), - ]); - expect(nativeLeafIndex).toEqual(legacyLeafIndex); - - const [nativeSB, legacySB] = await Promise.all([ - nativeFork.getSiblingPath(treeId, nativeLeafIndex!.index), - legacyLatest.getSiblingPath(treeId, legacyLeafIndex!.index), - ]); - - expect(nativeSB.toFields()).toEqual(legacySB.toFields()); - }); - }); - - describe('Block synch', () => { - it('syncs a new block from empty state', async () => { - await assertSameState(nativeWS.getCommitted(), legacyWS.getCommitted()); - const tempFork = await nativeWS.fork(); - const numBlocks = 1; - const numTxs = 32; - const blocks = []; - const messagesArray = []; - for (let i = 0; i < numBlocks; i++) { - const [_blockMS, { block, messages }] = await elapsed(mockBlock(1 + i, numTxs, tempFork)); - blocks.push(block); - messagesArray.push(messages); - } - - await tempFork.close(); - - for (let i = 0; i < numBlocks; i++) { - const [_nativeMs] = await elapsed(nativeWS.handleL2BlockAndMessages(blocks[i], messagesArray[i])); - const [_legacyMs] = await elapsed(legacyWS.handleL2BlockAndMessages(blocks[i], messagesArray[i])); - log.info(`Native: ${_nativeMs} ms, Legacy: ${_legacyMs} ms.`); - } - - await assertSameTree(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, nativeWS.getCommitted(), legacyWS.getCommitted()); - await assertSameTree(MerkleTreeId.NOTE_HASH_TREE, nativeWS.getCommitted(), legacyWS.getCommitted()); - await assertSameTree(MerkleTreeId.PUBLIC_DATA_TREE, nativeWS.getCommitted(), legacyWS.getCommitted()); - await assertSameTree(MerkleTreeId.NULLIFIER_TREE, nativeWS.getCommitted(), legacyWS.getCommitted()); - await assertSameTree(MerkleTreeId.ARCHIVE, nativeWS.getCommitted(), legacyWS.getCommitted()); - }, 86400_000); - }); - - async function assertSameTree( - treeId: MerkleTreeId, - forkA: MerkleTreeReadOperations, - forkB: MerkleTreeReadOperations, - ) { - const nativeInfo = await forkA.getTreeInfo(treeId); - const jsInfo = await forkB.getTreeInfo(treeId); - expect(nativeInfo.treeId).toBe(jsInfo.treeId); - expect(nativeInfo.depth).toBe(jsInfo.depth); - expect(nativeInfo.size).toBe(jsInfo.size); - expect(Fr.fromBuffer(nativeInfo.root)).toEqual(Fr.fromBuffer(jsInfo.root)); - } - - async function assertSameState(forkA: MerkleTreeReadOperations, forkB: MerkleTreeReadOperations) { - const nativeStateRef = await forkA.getStateReference(); - const nativeArchive = await forkA.getTreeInfo(MerkleTreeId.ARCHIVE); - const legacyStateRef = await forkB.getStateReference(); - const legacyArchive = await forkB.getTreeInfo(MerkleTreeId.ARCHIVE); - - expect(nativeStateRef).toEqual(legacyStateRef); - expect(nativeArchive).toEqual(legacyArchive); - } -}); diff --git a/yarn-project/world-state/src/test/index.ts b/yarn-project/world-state/src/test/index.ts new file mode 100644 index 000000000000..9be8099fb951 --- /dev/null +++ b/yarn-project/world-state/src/test/index.ts @@ -0,0 +1 @@ +export * from './utils.js'; diff --git a/yarn-project/world-state/src/world-state-db/index.ts b/yarn-project/world-state/src/world-state-db/index.ts index b0d4000267a7..e716161f23b0 100644 --- a/yarn-project/world-state/src/world-state-db/index.ts +++ b/yarn-project/world-state/src/world-state-db/index.ts @@ -1,6 +1,3 @@ -export * from './merkle_trees.js'; export * from './merkle_tree_db.js'; -export * from './merkle_tree_operations_facade.js'; -export * from './merkle_tree_snapshot_operations_facade.js'; export { MerkleTreeReadOperations } from '@aztec/circuit-types/interfaces'; diff --git a/yarn-project/world-state/src/world-state-db/merkle_tree_map.ts b/yarn-project/world-state/src/world-state-db/merkle_tree_map.ts deleted file mode 100644 index 61c6a91c198f..000000000000 --- a/yarn-project/world-state/src/world-state-db/merkle_tree_map.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { type MerkleTreeId } from '@aztec/circuit-types'; -import { type Fr } from '@aztec/circuits.js'; -import { type AppendOnlyTree, type IndexedTree } from '@aztec/merkle-tree'; - -export type MerkleTreeMap = { - [MerkleTreeId.NULLIFIER_TREE]: IndexedTree; - [MerkleTreeId.NOTE_HASH_TREE]: AppendOnlyTree; - [MerkleTreeId.PUBLIC_DATA_TREE]: IndexedTree; - [MerkleTreeId.L1_TO_L2_MESSAGE_TREE]: AppendOnlyTree; - [MerkleTreeId.ARCHIVE]: AppendOnlyTree; -}; diff --git a/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts b/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts deleted file mode 100644 index c4900241a620..000000000000 --- a/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts +++ /dev/null @@ -1,199 +0,0 @@ -import { type BatchInsertionResult, type MerkleTreeId, type SiblingPath } from '@aztec/circuit-types'; -import { - type IndexedTreeId, - type MerkleTreeLeafType, - type MerkleTreeWriteOperations, - type SequentialInsertionResult, - type TreeInfo, -} from '@aztec/circuit-types/interfaces'; -import { type BlockHeader, type StateReference } from '@aztec/circuits.js'; -import { type IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; - -import { type MerkleTrees } from './merkle_trees.js'; - -/** - * Wraps a MerkleTreeDbOperations to call all functions with a preset includeUncommitted flag. - */ -export class MerkleTreeReadOperationsFacade implements MerkleTreeWriteOperations { - constructor(protected trees: MerkleTrees, protected includeUncommitted: boolean) {} - - /** - * Returns the tree info for the specified tree id. - * @param treeId - Id of the tree to get information from. - * @param includeUncommitted - Indicates whether to include uncommitted data. - * @returns The tree info for the specified tree. - */ - getTreeInfo(treeId: MerkleTreeId): Promise { - return this.trees.getTreeInfo(treeId, this.includeUncommitted); - } - - /** - * Get the current state reference. - * @returns The current state reference. - */ - getStateReference(): Promise { - return this.trees.getStateReference(this.includeUncommitted); - } - - /** - * Returns the initial header for the chain before the first block. - * @returns The initial header. - */ - getInitialHeader(): BlockHeader { - return this.trees.getInitialHeader(); - } - - /** - * Appends a set of leaf values to the tree. - * @param treeId - Id of the tree to append leaves to. - * @param leaves - The set of leaves to be appended. - * @returns The tree info of the specified tree. - */ - appendLeaves(treeId: ID, leaves: MerkleTreeLeafType[]): Promise { - return this.trees.appendLeaves(treeId, leaves); - } - - /** - * Returns the sibling path for a requested leaf index. - * @param treeId - Id of the tree to get the sibling path from. - * @param index - The index of the leaf for which a sibling path is required. - * @returns A promise with the sibling path of the specified leaf index. - */ - async getSiblingPath(treeId: MerkleTreeId, index: bigint): Promise> { - const path = await this.trees.getSiblingPath(treeId, index, this.includeUncommitted); - return path as unknown as SiblingPath; - } - - /** - * Finds the index of the largest leaf whose value is less than or equal to the provided value. - * @param treeId - The ID of the tree to search. - * @param value - The value to be inserted into the tree. - * @param includeUncommitted - If true, the uncommitted changes are included in the search. - * @returns The found leaf index and a flag indicating if the corresponding leaf's value is equal to `newValue`. - */ - getPreviousValueIndex( - treeId: ID, - value: bigint, - ): Promise< - | { - /** - * The index of the found leaf. - */ - index: bigint; - /** - * A flag indicating if the corresponding leaf's value is equal to `newValue`. - */ - alreadyPresent: boolean; - } - | undefined - > { - return this.trees.getPreviousValueIndex(treeId, value, this.includeUncommitted); - } - - /** - * Gets the leaf data at a given index and tree. - * @param treeId - The ID of the tree get the leaf from. - * @param index - The index of the leaf to get. - * @returns Leaf preimage. - */ - async getLeafPreimage( - treeId: ID, - index: bigint, - ): Promise { - const preimage = await this.trees.getLeafPreimage(treeId, index, this.includeUncommitted); - return preimage as IndexedTreeLeafPreimage | undefined; - } - - /** - * Returns the index of a leaf given its value, or undefined if no leaf with that value is found. - * @param treeId - The ID of the tree. - * @param value - The leaf value to look for. - * @returns The index of the first leaf found with a given value (undefined if not found). - */ - findLeafIndices( - treeId: ID, - values: MerkleTreeLeafType[], - ): Promise<(bigint | undefined)[]> { - return Promise.all(values.map(leaf => this.trees.findLeafIndex(treeId, leaf, this.includeUncommitted))); - } - - /** - * Returns the first index containing a leaf value after `startIndex`. - * @param treeId - The tree for which the index should be returned. - * @param value - The value to search for in the tree. - * @param startIndex - The index to start searching from (used when skipping nullified messages) - */ - findLeafIndicesAfter( - treeId: ID, - values: MerkleTreeLeafType[], - startIndex: bigint, - ): Promise<(bigint | undefined)[]> { - return Promise.all( - values.map(leaf => this.trees.findLeafIndexAfter(treeId, leaf, startIndex, this.includeUncommitted)), - ); - } - - /** - * Gets the value at the given index. - * @param treeId - The ID of the tree to get the leaf value from. - * @param index - The index of the leaf. - * @param includeUncommitted - Indicates whether to include uncommitted changes. - * @returns Leaf value at the given index (undefined if not found). - */ - getLeafValue( - treeId: ID, - index: bigint, - ): Promise | undefined> { - return this.trees.getLeafValue(treeId, index, this.includeUncommitted) as Promise< - MerkleTreeLeafType | undefined - >; - } - - /** - * Inserts the new block hash into the archive. - * This includes all of the current roots of all of the data trees and the current blocks global vars. - * @param header - The header to insert into the archive. - */ - public updateArchive(header: BlockHeader): Promise { - return this.trees.updateArchive(header); - } - - /** - * Batch insert multiple leaves into the tree. - * @param treeId - The ID of the tree. - * @param leaves - Leaves to insert into the tree. - * @param subtreeHeight - Height of the subtree. - * @returns The data for the leaves to be updated when inserting the new ones. - */ - public batchInsert( - treeId: IndexedTreeId, - leaves: Buffer[], - subtreeHeight: number, - ): Promise> { - return this.trees.batchInsert(treeId, leaves, subtreeHeight); - } - - /** - * Sequentially inserts multiple leaves into the tree. - * @param treeId - The ID of the tree. - * @param leaves - Leaves to insert into the tree. - * @returns Witnesses for the operations performed. - */ - public sequentialInsert( - _treeId: IndexedTreeId, - _leaves: Buffer[], - ): Promise> { - throw new Error('Method not implemented in legacy merkle tree'); - } - - getBlockNumbersForLeafIndices( - _treeId: ID, - _leafIndices: bigint[], - ): Promise<(bigint | undefined)[]> { - throw new Error('Method not implemented in legacy merkle tree'); - } - - close(): Promise { - return Promise.resolve(); - } -} diff --git a/yarn-project/world-state/src/world-state-db/merkle_tree_snapshot_operations_facade.ts b/yarn-project/world-state/src/world-state-db/merkle_tree_snapshot_operations_facade.ts deleted file mode 100644 index 498375a64ea0..000000000000 --- a/yarn-project/world-state/src/world-state-db/merkle_tree_snapshot_operations_facade.ts +++ /dev/null @@ -1,151 +0,0 @@ -import { MerkleTreeId, type SiblingPath } from '@aztec/circuit-types'; -import { - type IndexedTreeId, - type MerkleTreeLeafType, - type MerkleTreeReadOperations, - type TreeInfo, -} from '@aztec/circuit-types/interfaces'; -import { - AppendOnlyTreeSnapshot, - type BlockHeader, - Fr, - PartialStateReference, - StateReference, -} from '@aztec/circuits.js'; -import { type IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; -import { type IndexedTreeSnapshot } from '@aztec/merkle-tree'; - -import { type TreeSnapshots } from './merkle_tree_db.js'; -import { type MerkleTrees } from './merkle_trees.js'; - -/** - * Merkle tree operations on readonly tree snapshots. - */ -export class MerkleTreeSnapshotOperationsFacade implements MerkleTreeReadOperations { - #treesDb: MerkleTrees; - #blockNumber: number; - #treeSnapshots: TreeSnapshots = {} as any; - - constructor(trees: MerkleTrees, blockNumber: number) { - this.#treesDb = trees; - this.#blockNumber = blockNumber; - } - - async #getTreeSnapshot(treeId: MerkleTreeId): Promise { - if (this.#treeSnapshots[treeId]) { - return this.#treeSnapshots[treeId]; - } - - this.#treeSnapshots = await this.#treesDb.getTreeSnapshots(this.#blockNumber); - return this.#treeSnapshots[treeId]!; - } - - async findLeafIndices( - treeId: ID, - values: MerkleTreeLeafType[], - ): Promise<(bigint | undefined)[]> { - const tree = await this.#getTreeSnapshot(treeId); - // TODO #5448 fix "as any" - return values.map(leaf => tree.findLeafIndex(leaf as any)); - } - - async findLeafIndicesAfter( - treeId: MerkleTreeId, - values: MerkleTreeLeafType[], - startIndex: bigint, - ): Promise<(bigint | undefined)[]> { - const tree = await this.#getTreeSnapshot(treeId); - // TODO #5448 fix "as any" - return values.map(leaf => tree.findLeafIndexAfter(leaf as any, startIndex)); - } - - async getLeafPreimage( - treeId: ID, - index: bigint, - ): Promise { - const snapshot = (await this.#getTreeSnapshot(treeId)) as IndexedTreeSnapshot; - return snapshot.getLatestLeafPreimageCopy(BigInt(index)); - } - - async getLeafValue( - treeId: ID, - index: bigint, - ): Promise | undefined> { - const snapshot = await this.#getTreeSnapshot(treeId); - return snapshot.getLeafValue(BigInt(index)) as MerkleTreeLeafType | undefined; - } - - async getPreviousValueIndex( - treeId: IndexedTreeId, - value: bigint, - ): Promise< - | { - /** - * The index of the found leaf. - */ - index: bigint; - /** - * A flag indicating if the corresponding leaf's value is equal to `newValue`. - */ - alreadyPresent: boolean; - } - | undefined - > { - const snapshot = (await this.#getTreeSnapshot(treeId)) as IndexedTreeSnapshot; - return snapshot.findIndexOfPreviousKey(value); - } - - async getSiblingPath(treeId: MerkleTreeId, index: bigint): Promise> { - const snapshot = await this.#getTreeSnapshot(treeId); - return snapshot.getSiblingPath(index); - } - - async getTreeInfo(treeId: MerkleTreeId): Promise { - const snapshot = await this.#getTreeSnapshot(treeId); - return { - depth: snapshot.getDepth(), - root: snapshot.getRoot(), - size: snapshot.getNumLeaves(), - treeId, - }; - } - - getBlockNumbersForLeafIndices(_a: ID, _b: bigint[]): Promise<(bigint | undefined)[]> { - throw new Error('Not implemented'); - } - - async getStateReference(): Promise { - const snapshots = await Promise.all([ - this.#getTreeSnapshot(MerkleTreeId.NULLIFIER_TREE), - this.#getTreeSnapshot(MerkleTreeId.NOTE_HASH_TREE), - this.#getTreeSnapshot(MerkleTreeId.PUBLIC_DATA_TREE), - this.#getTreeSnapshot(MerkleTreeId.L1_TO_L2_MESSAGE_TREE), - this.#getTreeSnapshot(MerkleTreeId.ARCHIVE), - ]); - - return new StateReference( - new AppendOnlyTreeSnapshot( - Fr.fromBuffer(snapshots[MerkleTreeId.L1_TO_L2_MESSAGE_TREE].getRoot()), - Number(snapshots[MerkleTreeId.L1_TO_L2_MESSAGE_TREE].getNumLeaves()), - ), - new PartialStateReference( - new AppendOnlyTreeSnapshot( - Fr.fromBuffer(snapshots[MerkleTreeId.NOTE_HASH_TREE].getRoot()), - Number(snapshots[MerkleTreeId.NOTE_HASH_TREE].getNumLeaves()), - ), - new AppendOnlyTreeSnapshot( - Fr.fromBuffer(snapshots[MerkleTreeId.NULLIFIER_TREE].getRoot()), - Number(snapshots[MerkleTreeId.NULLIFIER_TREE].getNumLeaves()), - ), - new AppendOnlyTreeSnapshot( - Fr.fromBuffer(snapshots[MerkleTreeId.PUBLIC_DATA_TREE].getRoot()), - Number(snapshots[MerkleTreeId.PUBLIC_DATA_TREE].getNumLeaves()), - ), - ), - ); - } - - getInitialHeader(): BlockHeader { - throw new Error('Getting initial header not supported on snapshot.'); - } -} diff --git a/yarn-project/world-state/src/world-state-db/merkle_trees.ts b/yarn-project/world-state/src/world-state-db/merkle_trees.ts deleted file mode 100644 index 2da852c6a259..000000000000 --- a/yarn-project/world-state/src/world-state-db/merkle_trees.ts +++ /dev/null @@ -1,727 +0,0 @@ -import { type L2Block, MerkleTreeId, type SiblingPath } from '@aztec/circuit-types'; -import { - type BatchInsertionResult, - type IndexedTreeId, - type MerkleTreeLeafType, - type MerkleTreeReadOperations, - type MerkleTreeWriteOperations, - type TreeInfo, -} from '@aztec/circuit-types/interfaces'; -import { - ARCHIVE_HEIGHT, - AppendOnlyTreeSnapshot, - BlockHeader, - Fr, - L1_TO_L2_MSG_TREE_HEIGHT, - MAX_NOTE_HASHES_PER_TX, - MAX_NULLIFIERS_PER_TX, - MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, - NOTE_HASH_TREE_HEIGHT, - NULLIFIER_SUBTREE_HEIGHT, - NULLIFIER_TREE_HEIGHT, - NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, - NullifierLeaf, - NullifierLeafPreimage, - PUBLIC_DATA_SUBTREE_HEIGHT, - PUBLIC_DATA_TREE_HEIGHT, - PartialStateReference, - PublicDataTreeLeaf, - PublicDataTreeLeafPreimage, - PublicDataWrite, - StateReference, -} from '@aztec/circuits.js'; -import { padArrayEnd } from '@aztec/foundation/collection'; -import { type Logger, createLogger } from '@aztec/foundation/log'; -import { SerialQueue } from '@aztec/foundation/queue'; -import { Timer, elapsed } from '@aztec/foundation/timer'; -import { type IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; -import { type AztecKVStore, type AztecSingleton } from '@aztec/kv-store'; -import { openTmpStore } from '@aztec/kv-store/lmdb'; -import { - type AppendOnlyTree, - type IndexedTree, - Poseidon, - StandardIndexedTree, - StandardTree, - getTreeMeta, - loadTree, - newTree, -} from '@aztec/merkle-tree'; -import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client'; -import { type Hasher } from '@aztec/types/interfaces'; - -import { - type WorldStateStatusFull, - type WorldStateStatusSummary, - buildEmptyWorldStateStatusFull, -} from '../native/message.js'; -import { - INITIAL_NULLIFIER_TREE_SIZE, - INITIAL_PUBLIC_DATA_TREE_SIZE, - type MerkleTreeAdminDatabase, - type TreeSnapshots, -} from './merkle_tree_db.js'; -import { type MerkleTreeMap } from './merkle_tree_map.js'; -import { MerkleTreeReadOperationsFacade } from './merkle_tree_operations_facade.js'; -import { MerkleTreeSnapshotOperationsFacade } from './merkle_tree_snapshot_operations_facade.js'; -import { WorldStateMetrics } from './metrics.js'; - -/** - * The nullifier tree is an indexed tree. - */ -class NullifierTree extends StandardIndexedTree { - constructor( - store: AztecKVStore, - hasher: Hasher, - name: string, - depth: number, - size: bigint = 0n, - _noop: any, - root?: Buffer, - ) { - super(store, hasher, name, depth, size, NullifierLeafPreimage, NullifierLeaf, root); - } -} - -/** - * The public data tree is an indexed tree. - */ -class PublicDataTree extends StandardIndexedTree { - constructor( - store: AztecKVStore, - hasher: Hasher, - name: string, - depth: number, - size: bigint = 0n, - _noop: any, - root?: Buffer, - ) { - super(store, hasher, name, depth, size, PublicDataTreeLeafPreimage, PublicDataTreeLeaf, root); - } -} - -/** - * A convenience class for managing multiple merkle trees. - */ -export class MerkleTrees implements MerkleTreeAdminDatabase { - // gets initialized in #init - private trees: MerkleTreeMap = null as any; - private jobQueue = new SerialQueue(); - private initialStateReference: AztecSingleton; - private metrics: WorldStateMetrics; - - private constructor(private store: AztecKVStore, private telemetryClient: TelemetryClient, private log: Logger) { - this.initialStateReference = store.openSingleton('merkle_trees_initial_state_reference'); - this.metrics = new WorldStateMetrics(telemetryClient); - } - - /** - * Method to asynchronously create and initialize a MerkleTrees instance. - * @param store - The db instance to use for data persistance. - * @returns - A fully initialized MerkleTrees instance. - */ - public static async new( - store: AztecKVStore, - client: TelemetryClient = getTelemetryClient(), - log = createLogger('world-state:merkle_trees'), - ) { - const merkleTrees = new MerkleTrees(store, client, log); - await merkleTrees.#init(); - return merkleTrees; - } - - /** - * Creates a temporary store. Useful for testing. - */ - public static tmp() { - const store = openTmpStore(); - return MerkleTrees.new(store, getTelemetryClient()); - } - - /** - * Initializes the collection of Merkle Trees. - */ - async #init(loadFromDb?: boolean) { - const fromDb = loadFromDb === undefined ? this.#isDbPopulated() : loadFromDb; - const initializeTree = fromDb ? loadTree : newTree; - - const hasher = new Poseidon(); - - const nullifierTree = await initializeTree( - NullifierTree, - this.store, - hasher, - `${MerkleTreeId[MerkleTreeId.NULLIFIER_TREE]}`, - {}, - NULLIFIER_TREE_HEIGHT, - INITIAL_NULLIFIER_TREE_SIZE, - ); - const noteHashTree: AppendOnlyTree = await initializeTree( - StandardTree, - this.store, - hasher, - `${MerkleTreeId[MerkleTreeId.NOTE_HASH_TREE]}`, - Fr, - NOTE_HASH_TREE_HEIGHT, - ); - const publicDataTree = await initializeTree( - PublicDataTree, - this.store, - hasher, - `${MerkleTreeId[MerkleTreeId.PUBLIC_DATA_TREE]}`, - {}, - PUBLIC_DATA_TREE_HEIGHT, - INITIAL_PUBLIC_DATA_TREE_SIZE, - ); - const l1Tol2MessageTree: AppendOnlyTree = await initializeTree( - StandardTree, - this.store, - hasher, - `${MerkleTreeId[MerkleTreeId.L1_TO_L2_MESSAGE_TREE]}`, - Fr, - L1_TO_L2_MSG_TREE_HEIGHT, - ); - const archive: AppendOnlyTree = await initializeTree( - StandardTree, - this.store, - hasher, - `${MerkleTreeId[MerkleTreeId.ARCHIVE]}`, - Fr, - ARCHIVE_HEIGHT, - ); - this.trees = [nullifierTree, noteHashTree, publicDataTree, l1Tol2MessageTree, archive]; - - this.jobQueue.start(); - - if (!fromDb) { - // We are not initializing from db so we need to populate the first leaf of the archive tree which is a hash of the initial header, - // and persist the initial header state reference so we can later load it when requested. - const initialState = await this.getStateReference(true); - await this.#saveInitialStateReference(initialState); - await this.#updateArchive(this.getInitialHeader()); - - // And commit anything we did to initialize this set of trees - await this.#commit(); - } - } - - public removeHistoricalBlocks(_toBlockNumber: bigint): Promise { - throw new Error('Method not implemented.'); - } - - public unwindBlocks(_toBlockNumber: bigint): Promise { - throw new Error('Method not implemented.'); - } - - public setFinalised(_toBlockNumber: bigint): Promise { - throw new Error('Method not implemented.'); - } - - public getStatusSummary(): Promise { - throw new Error('Method not implemented.'); - } - - public async fork(blockNumber?: number): Promise { - if (blockNumber) { - throw new Error('Block number forking is not supported in js world state'); - } - const [ms, db] = await elapsed(async () => { - const forked = await this.store.fork(); - return MerkleTrees.new(forked, this.telemetryClient, this.log); - }); - - this.metrics.recordForkDuration(ms); - return new MerkleTreeReadOperationsFacade(db, true); - } - - // REFACTOR: We're hiding the `commit` operations in the tree behind a type check only, but - // we should make sure it's not accidentally called elsewhere by splitting this class into one - // that can work on a read-only store and one that actually writes to the store. This implies - // having read-only versions of the kv-stores, all kv-containers, and all trees. - public async ephemeralFork(): Promise { - const forked = new MerkleTrees( - this.store, - this.telemetryClient, - createLogger('world-state:merkle_trees:ephemeral_fork'), - ); - await forked.#init(true); - return new MerkleTreeReadOperationsFacade(forked, true); - } - - public async delete() { - await this.store.delete(); - } - - public getInitialHeader(): BlockHeader { - return BlockHeader.empty({ state: this.#loadInitialStateReference() }); - } - - /** - * Stops the job queue (waits for all jobs to finish). - */ - public async close() { - await this.jobQueue.end(); - } - - /** - * Gets a view of this db that returns uncommitted data. - * @returns - A facade for this instance. - */ - public getLatest(): Promise { - return Promise.resolve(new MerkleTreeReadOperationsFacade(this, true)); - } - - /** - * Gets a view of this db that returns committed data only. - * @returns - A facade for this instance. - */ - public getCommitted(): MerkleTreeReadOperations { - return new MerkleTreeReadOperationsFacade(this, false); - } - - public getSnapshot(blockNumber: number): MerkleTreeReadOperations { - return new MerkleTreeSnapshotOperationsFacade(this, blockNumber); - } - - /** - * Updates the archive with the new block/header hash. - * @param header - The header whose hash to insert into the archive. - * @param includeUncommitted - Indicates whether to include uncommitted data. - */ - public async updateArchive(header: BlockHeader) { - await this.synchronize(() => this.#updateArchive(header)); - } - - /** - * Gets the tree info for the specified tree. - * @param treeId - Id of the tree to get information from. - * @param includeUncommitted - Indicates whether to include uncommitted data. - * @returns The tree info for the specified tree. - */ - public async getTreeInfo(treeId: MerkleTreeId, includeUncommitted: boolean): Promise { - return await this.synchronize(() => this.#getTreeInfo(treeId, includeUncommitted)); - } - - /** - * Get the current state reference - * @param includeUncommitted - Indicates whether to include uncommitted data. - * @returns The current state reference - */ - public getStateReference(includeUncommitted: boolean): Promise { - const getAppendOnlyTreeSnapshot = (treeId: MerkleTreeId) => { - const tree = this.trees[treeId] as AppendOnlyTree; - return new AppendOnlyTreeSnapshot( - Fr.fromBuffer(tree.getRoot(includeUncommitted)), - Number(tree.getNumLeaves(includeUncommitted)), - ); - }; - - const state = new StateReference( - getAppendOnlyTreeSnapshot(MerkleTreeId.L1_TO_L2_MESSAGE_TREE), - new PartialStateReference( - getAppendOnlyTreeSnapshot(MerkleTreeId.NOTE_HASH_TREE), - getAppendOnlyTreeSnapshot(MerkleTreeId.NULLIFIER_TREE), - getAppendOnlyTreeSnapshot(MerkleTreeId.PUBLIC_DATA_TREE), - ), - ); - return Promise.resolve(state); - } - - /** - * Gets the value at the given index. - * @param treeId - The ID of the tree to get the leaf value from. - * @param index - The index of the leaf. - * @param includeUncommitted - Indicates whether to include uncommitted changes. - * @returns Leaf value at the given index (undefined if not found). - */ - public async getLeafValue( - treeId: MerkleTreeId, - index: bigint, - includeUncommitted: boolean, - ): Promise | undefined> { - return await this.synchronize(() => Promise.resolve(this.trees[treeId].getLeafValue(index, includeUncommitted))); - } - - /** - * Gets the sibling path for a leaf in a tree. - * @param treeId - The ID of the tree. - * @param index - The index of the leaf. - * @param includeUncommitted - Indicates whether the sibling path should include uncommitted data. - * @returns The sibling path for the leaf. - */ - public async getSiblingPath( - treeId: MerkleTreeId, - index: bigint, - includeUncommitted: boolean, - ): Promise> { - return await this.synchronize(() => this.trees[treeId].getSiblingPath(index, includeUncommitted)); - } - - /** - * Appends leaves to a tree. - * @param treeId - The ID of the tree. - * @param leaves - The leaves to append. - * @returns Empty promise. - */ - public async appendLeaves(treeId: ID, leaves: MerkleTreeLeafType[]): Promise { - return await this.synchronize(() => this.#appendLeaves(treeId, leaves)); - } - - /** - * Commits all pending updates. - * @returns Empty promise. - */ - public async commit(): Promise { - return await this.synchronize(() => this.#commit()); - } - - /** - * Rolls back all pending updates. - * @returns Empty promise. - */ - public async rollback(): Promise { - return await this.synchronize(() => this.#rollback()); - } - - /** - * Finds the index of the largest leaf whose value is less than or equal to the provided value. - * @param treeId - The ID of the tree to search. - * @param value - The value to be inserted into the tree. - * @param includeUncommitted - If true, the uncommitted changes are included in the search. - * @returns The found leaf index and a flag indicating if the corresponding leaf's value is equal to `newValue`. - */ - public async getPreviousValueIndex( - treeId: IndexedTreeId, - value: bigint, - includeUncommitted: boolean, - ): Promise< - | { - /** - * The index of the found leaf. - */ - index: bigint; - /** - * A flag indicating if the corresponding leaf's value is equal to `newValue`. - */ - alreadyPresent: boolean; - } - | undefined - > { - return await this.synchronize(() => - Promise.resolve(this.#getIndexedTree(treeId).findIndexOfPreviousKey(value, includeUncommitted)), - ); - } - - /** - * Gets the leaf data at a given index and tree. - * @param treeId - The ID of the tree get the leaf from. - * @param index - The index of the leaf to get. - * @param includeUncommitted - Indicates whether to include uncommitted data. - * @returns Leaf preimage. - */ - public async getLeafPreimage( - treeId: IndexedTreeId, - index: bigint, - includeUncommitted: boolean, - ): Promise { - return await this.synchronize(() => - Promise.resolve(this.#getIndexedTree(treeId).getLatestLeafPreimageCopy(index, includeUncommitted)), - ); - } - - /** - * Returns the index of a leaf given its value, or undefined if no leaf with that value is found. - * @param treeId - The ID of the tree. - * @param value - The leaf value to look for. - * @param includeUncommitted - Indicates whether to include uncommitted data. - * @returns The index of the first leaf found with a given value (undefined if not found). - */ - public async findLeafIndex( - treeId: ID, - value: MerkleTreeLeafType, - includeUncommitted: boolean, - ): Promise { - return await this.synchronize(() => { - const tree = this.trees[treeId]; - // TODO #5448 fix "as any" - return Promise.resolve(tree.findLeafIndex(value as any, includeUncommitted)); - }); - } - - /** - * Returns the first index containing a leaf value after `startIndex`. - * @param treeId - The tree for which the index should be returned. - * @param value - The value to search for in the tree. - * @param startIndex - The index to start searching from (used when skipping nullified messages) - * @param includeUncommitted - Indicates whether to include uncommitted data. - */ - public async findLeafIndexAfter( - treeId: ID, - value: MerkleTreeLeafType, - startIndex: bigint, - includeUncommitted: boolean, - ): Promise { - return await this.synchronize(() => { - const tree = this.trees[treeId]; - // TODO #5448 fix "as any" - return Promise.resolve(tree.findLeafIndexAfter(value as any, startIndex, includeUncommitted)); - }); - } - - /** - * Handles a single L2 block (i.e. Inserts the new note hashes into the merkle tree). - * @param block - The L2 block to handle. - * @param l1ToL2Messages - The L1 to L2 messages for the block. - * @returns Whether the block handled was produced by this same node. - */ - public async handleL2BlockAndMessages(block: L2Block, l1ToL2Messages: Fr[]): Promise { - return await this.synchronize(() => this.#handleL2BlockAndMessages(block, l1ToL2Messages)); - } - - /** - * Batch insert multiple leaves into the tree. - * @param treeId - The ID of the tree. - * @param leaves - Leaves to insert into the tree. - * @param subtreeHeight - Height of the subtree. - * @returns The data for the leaves to be updated when inserting the new ones. - */ - public async batchInsert< - TreeHeight extends number, - SubtreeHeight extends number, - SubtreeSiblingPathHeight extends number, - >( - treeId: IndexedTreeId, - leaves: Buffer[], - subtreeHeight: SubtreeHeight, - ): Promise> { - const tree = this.trees[treeId] as StandardIndexedTree; - if (!('batchInsert' in tree)) { - throw new Error('Tree does not support `batchInsert` method'); - } - return await this.synchronize(() => tree.batchInsert(leaves, subtreeHeight)); - } - - /** - * Waits for all jobs to finish before executing the given function. - * @param fn - The function to execute. - * @returns Promise containing the result of the function. - */ - private async synchronize(fn: () => Promise): Promise { - return await this.jobQueue.put(fn); - } - - #saveInitialStateReference(state: StateReference) { - return this.initialStateReference.set(state.toBuffer()); - } - - #loadInitialStateReference(): StateReference { - const serialized = this.initialStateReference.get(); - if (!serialized) { - throw new Error('Initial state reference not found'); - } - return StateReference.fromBuffer(serialized); - } - - async #updateArchive(header: BlockHeader) { - const state = await this.getStateReference(true); - - // This method should be called only when the block builder already updated the state so we sanity check that it's - // the case here. - if (!state.toBuffer().equals(header.state.toBuffer())) { - throw new Error('State in header does not match current state'); - } - - const blockHash = await header.hash(); - await this.#appendLeaves(MerkleTreeId.ARCHIVE, [blockHash]); - } - - /** - * Returns the tree info for the specified tree id. - * @param treeId - Id of the tree to get information from. - * @param includeUncommitted - Indicates whether to include uncommitted data. - * @returns The tree info for the specified tree. - */ - #getTreeInfo(treeId: MerkleTreeId, includeUncommitted: boolean): Promise { - const treeInfo = { - treeId, - root: this.trees[treeId].getRoot(includeUncommitted), - size: this.trees[treeId].getNumLeaves(includeUncommitted), - depth: this.trees[treeId].getDepth(), - } as TreeInfo; - return Promise.resolve(treeInfo); - } - - /** - * Returns an instance of an indexed tree. - * @param treeId - Id of the tree to get an instance of. - * @returns The indexed tree for the specified tree id. - */ - #getIndexedTree(treeId: IndexedTreeId): IndexedTree { - return this.trees[treeId] as IndexedTree; - } - - /** - * Appends leaves to a tree. - * @param treeId - Id of the tree to append leaves to. - * @param leaves - Leaves to append. - * @returns Empty promise. - */ - async #appendLeaves(treeId: ID, leaves: MerkleTreeLeafType[]): Promise { - const tree = this.trees[treeId]; - if (!('appendLeaves' in tree)) { - throw new Error('Tree does not support `appendLeaves` method'); - } - // TODO #5448 fix "as any" - return await tree.appendLeaves(leaves as any[]); - } - - /** - * Commits all pending updates. - * @returns Empty promise. - */ - async #commit(): Promise { - for (const tree of Object.values(this.trees)) { - await tree.commit(); - } - } - - /** - * Rolls back all pending updates. - * @returns Empty promise. - */ - async #rollback(): Promise { - for (const tree of Object.values(this.trees)) { - await tree.rollback(); - } - } - - public async getTreeSnapshots(blockNumber: number): Promise { - const snapshots = await Promise.all([ - this.trees[MerkleTreeId.NULLIFIER_TREE].getSnapshot(blockNumber), - this.trees[MerkleTreeId.NOTE_HASH_TREE].getSnapshot(blockNumber), - this.trees[MerkleTreeId.PUBLIC_DATA_TREE].getSnapshot(blockNumber), - this.trees[MerkleTreeId.L1_TO_L2_MESSAGE_TREE].getSnapshot(blockNumber), - this.trees[MerkleTreeId.ARCHIVE].getSnapshot(blockNumber), - ]); - - return { - [MerkleTreeId.NULLIFIER_TREE]: snapshots[0], - [MerkleTreeId.NOTE_HASH_TREE]: snapshots[1], - [MerkleTreeId.PUBLIC_DATA_TREE]: snapshots[2], - [MerkleTreeId.L1_TO_L2_MESSAGE_TREE]: snapshots[3], - [MerkleTreeId.ARCHIVE]: snapshots[4], - }; - } - - async #snapshot(blockNumber: number): Promise { - for (const tree of Object.values(this.trees)) { - await tree.snapshot(blockNumber); - } - } - - /** - * Handles a single L2 block (i.e. Inserts the new note hashes into the merkle tree). - * @param l2Block - The L2 block to handle. - * @param l1ToL2Messages - The L1 to L2 messages for the block. - */ - async #handleL2BlockAndMessages(l2Block: L2Block, l1ToL2Messages: Fr[]): Promise { - const timer = new Timer(); - - const treeRootWithIdPairs = [ - [l2Block.header.state.partial.nullifierTree.root, MerkleTreeId.NULLIFIER_TREE], - [l2Block.header.state.partial.noteHashTree.root, MerkleTreeId.NOTE_HASH_TREE], - [l2Block.header.state.partial.publicDataTree.root, MerkleTreeId.PUBLIC_DATA_TREE], - [l2Block.header.state.l1ToL2MessageTree.root, MerkleTreeId.L1_TO_L2_MESSAGE_TREE], - [l2Block.archive.root, MerkleTreeId.ARCHIVE], - ] as const; - const compareRoot = (root: Fr, treeId: MerkleTreeId) => { - const treeRoot = this.trees[treeId].getRoot(true); - return treeRoot.equals(root.toBuffer()); - }; - const ourBlock = treeRootWithIdPairs.every(([root, id]) => compareRoot(root, id)); - if (ourBlock) { - this.log.verbose(`Block ${l2Block.number} is ours, committing world state`); - await this.#commit(); - } else { - this.log.verbose(`Block ${l2Block.number} is not ours, rolling back world state and committing state from chain`); - await this.#rollback(); - - // We have to pad the values within tx effects because that's how the trees are built by circuits. - - // Sync the append only trees - { - const noteHashesPadded = l2Block.body.txEffects.flatMap(txEffect => - padArrayEnd(txEffect.noteHashes, Fr.ZERO, MAX_NOTE_HASHES_PER_TX), - ); - await this.#appendLeaves(MerkleTreeId.NOTE_HASH_TREE, noteHashesPadded); - - const l1ToL2MessagesPadded = padArrayEnd(l1ToL2Messages, Fr.ZERO, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP); - await this.#appendLeaves(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, l1ToL2MessagesPadded); - } - - // Sync the indexed trees - { - const nullifiersPadded = l2Block.body.txEffects.flatMap(txEffect => - padArrayEnd(txEffect.nullifiers, Fr.ZERO, MAX_NULLIFIERS_PER_TX), - ); - await (this.trees[MerkleTreeId.NULLIFIER_TREE] as StandardIndexedTree).batchInsert( - nullifiersPadded.map(nullifier => nullifier.toBuffer()), - NULLIFIER_SUBTREE_HEIGHT, - ); - - const publicDataTree = this.trees[MerkleTreeId.PUBLIC_DATA_TREE] as StandardIndexedTree; - - // We insert the public data tree leaves with one batch per tx to avoid updating the same key twice - for (const txEffect of l2Block.body.txEffects) { - const publicDataWrites = padArrayEnd( - txEffect.publicDataWrites, - PublicDataWrite.empty(), - MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, - ); - - await publicDataTree.batchInsert( - publicDataWrites.map(write => write.toBuffer()), - PUBLIC_DATA_SUBTREE_HEIGHT, - ); - } - } - - // The last thing remaining is to update the archive - await this.#updateArchive(l2Block.header); - - await this.#commit(); - } - - for (const [root, treeId] of treeRootWithIdPairs) { - const treeName = MerkleTreeId[treeId]; - const info = await this.#getTreeInfo(treeId, false); - const syncedStr = '0x' + info.root.toString('hex'); - const rootStr = root.toString(); - // Sanity check that the rebuilt trees match the roots published by the L2 block - if (!info.root.equals(root.toBuffer())) { - throw new Error( - `Synced tree root ${treeName} does not match published L2 block root: ${syncedStr} != ${rootStr}`, - ); - } else { - this.log.debug(`Tree ${treeName} synched with size ${info.size} root ${rootStr}`); - this.metrics.recordTreeSize(treeName, info.size); - } - } - await this.#snapshot(l2Block.number); - - this.metrics.recordDbSize((await this.store.estimateSize()).actualSize); - this.metrics.recordSyncDuration('commit', timer); - return buildEmptyWorldStateStatusFull(); - } - - #isDbPopulated(): boolean { - try { - getTreeMeta(this.store, MerkleTreeId[MerkleTreeId.NULLIFIER_TREE]); - // Tree meta was found --> db is populated - return true; - } catch (e) { - // Tree meta was not found --> db is not populated - return false; - } - } -} diff --git a/yarn-project/world-state/src/world-state-db/metrics.ts b/yarn-project/world-state/src/world-state-db/metrics.ts deleted file mode 100644 index d7965590e1f5..000000000000 --- a/yarn-project/world-state/src/world-state-db/metrics.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { type Timer } from '@aztec/foundation/timer'; -import { - Attributes, - type Gauge, - type Histogram, - Metrics, - type TelemetryClient, - ValueType, -} from '@aztec/telemetry-client'; - -export class WorldStateMetrics { - private treeSize: Gauge; - private dbSize: Gauge; - private forkDuration: Histogram; - private syncDuration: Histogram; - - constructor(client: TelemetryClient, name = 'MerkleTreesDb') { - const meter = client.getMeter(name); - this.treeSize = meter.createGauge(Metrics.WORLD_STATE_MERKLE_TREE_SIZE, { - description: 'The size of Merkle trees', - valueType: ValueType.INT, - }); - - this.dbSize = meter.createGauge(Metrics.WORLD_STATE_DB_SIZE, { - description: 'The size of the World State DB', - valueType: ValueType.INT, - unit: 'By', - }); - - this.forkDuration = meter.createHistogram(Metrics.WORLD_STATE_FORK_DURATION, { - description: 'The duration of a fork operation', - unit: 'ms', - valueType: ValueType.INT, - }); - - this.syncDuration = meter.createHistogram(Metrics.WORLD_STATE_SYNC_DURATION, { - description: 'The duration of a sync operation', - unit: 'ms', - valueType: ValueType.INT, - }); - } - - recordTreeSize(treeName: string, treeSize: bigint) { - this.treeSize.record(Number(treeSize), { - [Attributes.MERKLE_TREE_NAME]: treeName, - }); - } - - recordDbSize(dbSizeInBytes: number) { - this.dbSize.record(dbSizeInBytes); - } - - recordForkDuration(timerOrMs: Timer | number) { - const ms = Math.ceil(typeof timerOrMs === 'number' ? timerOrMs : timerOrMs.ms()); - this.forkDuration.record(ms); - } - - recordSyncDuration(syncType: 'commit' | 'rollback_and_update', timerOrMs: Timer | number) { - const ms = Math.ceil(typeof timerOrMs === 'number' ? timerOrMs : timerOrMs.ms()); - this.syncDuration.record(ms, { - [Attributes.STATUS]: syncType, - }); - } -}