diff --git a/boxes/package.json b/boxes/package.json index 03e4c87b41d1..f2b7cc8acc0f 100644 --- a/boxes/package.json +++ b/boxes/package.json @@ -41,7 +41,6 @@ "@aztec/blob-lib": "link:../yarn-project/blob-lib", "@aztec/native": "link:../yarn-project/native", "@aztec/builder": "link:../yarn-project/builder", - "@aztec/merkle-tree": "link:../yarn-project/merkle-tree", "@aztec/types": "link:../yarn-project/types", "@aztec/utils": "link:../yarn-project/utils", "@aztec/testing-utils": "link:../yarn-project/testing-utils", diff --git a/docs/examples/ts/tsconfig.template.json b/docs/examples/ts/tsconfig.template.json index cce5959a0c03..70cc01a73034 100644 --- a/docs/examples/ts/tsconfig.template.json +++ b/docs/examples/ts/tsconfig.template.json @@ -61,9 +61,6 @@ { "path": "../../../../yarn-project/l1-artifacts" }, - { - "path": "../../../../yarn-project/merkle-tree" - }, { "path": "../../../../yarn-project/node-keystore" }, diff --git a/yarn-project/aztec-node/package.json b/yarn-project/aztec-node/package.json index be268d1753df..ef0347475f3d 100644 --- a/yarn-project/aztec-node/package.json +++ b/yarn-project/aztec-node/package.json @@ -75,7 +75,6 @@ "@aztec/foundation": "workspace:^", "@aztec/kv-store": "workspace:^", "@aztec/l1-artifacts": "workspace:^", - "@aztec/merkle-tree": "workspace:^", "@aztec/node-keystore": "workspace:^", "@aztec/node-lib": "workspace:^", "@aztec/noir-protocol-circuits-types": "workspace:^", diff --git a/yarn-project/aztec-node/tsconfig.json b/yarn-project/aztec-node/tsconfig.json index 272c00696903..86bc75e353ac 100644 --- a/yarn-project/aztec-node/tsconfig.json +++ b/yarn-project/aztec-node/tsconfig.json @@ -36,9 +36,6 @@ { "path": "../l1-artifacts" }, - { - "path": "../merkle-tree" - }, { "path": "../node-keystore" }, diff --git a/yarn-project/end-to-end/package.json b/yarn-project/end-to-end/package.json index dab6f573d094..c3b0cb00854c 100644 --- a/yarn-project/end-to-end/package.json +++ b/yarn-project/end-to-end/package.json @@ -44,7 +44,6 @@ "@aztec/foundation": "workspace:^", "@aztec/kv-store": "workspace:^", "@aztec/l1-artifacts": "workspace:^", - "@aztec/merkle-tree": "workspace:^", "@aztec/node-keystore": "workspace:^", "@aztec/noir-contracts.js": "workspace:^", "@aztec/noir-noirc_abi": "portal:../../noir/packages/noirc_abi", diff --git a/yarn-project/end-to-end/tsconfig.json b/yarn-project/end-to-end/tsconfig.json index 86aa22718cda..648e526c1c45 100644 --- a/yarn-project/end-to-end/tsconfig.json +++ b/yarn-project/end-to-end/tsconfig.json @@ -58,9 +58,6 @@ { "path": "../l1-artifacts" }, - { - "path": "../merkle-tree" - }, { "path": "../node-keystore" }, diff --git a/yarn-project/merkle-tree/README.md b/yarn-project/merkle-tree/README.md deleted file mode 100644 index 68d7716d4b2d..000000000000 --- a/yarn-project/merkle-tree/README.md +++ /dev/null @@ -1,41 +0,0 @@ -# Merkle Tree - -## Overview - -The Merkle Trees package contains typescript implementations of the various Merkle Trees required by the system. - -- Append only 'Standard' merkle trees. New values are inserted into the next available leaf index. Values are never updated. -- Indexed trees are also append only in nature but retain the ability to update leaves. The reason for this is that the Indexed Tree leaves not only store the value but the index of the next highest leaf. New insertions can require prior leaves to be updated. -- Sparse trees that can be updated at any index. The 'size' of the tree is defined by the number of non-empty leaves, not by the highest populated leaf index as is the case with a Standard Tree. - -## Implementations - -All implementations require the following arguments: - -- An instance of level db for storing information at a tree node level as well as some tree level metadata. -- An instance of a Hasher object. Used for hashing the nodes of the Merkle Tree. -- A name, for namespacing the tree's nodes within the db. -- The depth of the tree, this is exclusive of the root. - -Implementations have commit/rollback semantics with modifications stored only in memory until being committed. Rolling back returns the tree discards the cached modifications and returns the tree to it's previous committed state. - -## Tree Operations - -The primary operations available on the various implementations are: - -- Create a new, empty tree -- Restore a tree from the provided DB -- Append new leaves (Standard and Indexed trees only) -- Update a leaf (Sparse tree and Indexed trees only) -- Retrieve the number of leaves. This is the number of non empty leaves for a Sparse tree and the highest populated index for Standard and Indexed trees. -- Commit modifications -- Rollback modifications -- Retrieve a Sibling Path for a given index. For performing Merkle proofs it is necessary to derive the set of nodes from a leaf index to the root of the tree, known as the 'hash path'. Given a leaf, it is therefore required to have the 'sibling' node at each tree level in order for the hash path to be computed. - -Note: Tree operations are not 'thread' safe. Operations should be queued or otherwise executed serially to ensure consistency of the data structures and any data retrieved from them. - -## Building/Testing - -To build the package, execute `yarn build` in the root. - -To run the tests, execute `yarn test`. diff --git a/yarn-project/merkle-tree/eslint.config.js b/yarn-project/merkle-tree/eslint.config.js deleted file mode 100644 index 0331d0552f62..000000000000 --- a/yarn-project/merkle-tree/eslint.config.js +++ /dev/null @@ -1,3 +0,0 @@ -import config from '@aztec/foundation/eslint'; - -export default config; diff --git a/yarn-project/merkle-tree/package.json b/yarn-project/merkle-tree/package.json deleted file mode 100644 index d96b46c43a0e..000000000000 --- a/yarn-project/merkle-tree/package.json +++ /dev/null @@ -1,88 +0,0 @@ -{ - "name": "@aztec/merkle-tree", - "version": "0.1.0", - "type": "module", - "exports": "./dest/index.js", - "typedocOptions": { - "entryPoints": [ - "./src/index.ts" - ], - "name": "Merkle Tree", - "tsconfig": "./tsconfig.json" - }, - "scripts": { - "build": "yarn clean && ../scripts/tsc.sh", - "build:dev": "../scripts/tsc.sh --watch", - "clean": "rm -rf ./dest .tsbuildinfo", - "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}" - }, - "inherits": [ - "../package.common.json", - "./package.local.json" - ], - "jest": { - "moduleNameMapper": { - "^(\\.{1,2}/.*)\\.[cm]?js$": "$1" - }, - "testRegex": "./src/.*\\.test\\.(js|mjs|ts)$", - "rootDir": "./src", - "transform": { - "^.+\\.tsx?$": [ - "@swc/jest", - { - "jsc": { - "parser": { - "syntax": "typescript", - "decorators": true - }, - "transform": { - "decoratorVersion": "2022-03" - } - } - } - ] - }, - "extensionsToTreatAsEsm": [ - ".ts" - ], - "reporters": [ - "default" - ], - "testTimeout": 120000, - "setupFiles": [ - "../../foundation/src/jest/setup.mjs" - ], - "testEnvironment": "../../foundation/src/jest/env.mjs", - "setupFilesAfterEnv": [ - "../../foundation/src/jest/setupAfterEnv.mjs" - ] - }, - "dependencies": { - "@aztec/foundation": "workspace:^", - "@aztec/kv-store": "workspace:^", - "@aztec/stdlib": "workspace:^", - "sha256": "^0.2.0", - "tslib": "^2.4.0" - }, - "devDependencies": { - "@jest/globals": "^30.0.0", - "@types/jest": "^30.0.0", - "@types/node": "^22.15.17", - "@types/sha256": "^0.2.0", - "@typescript/native-preview": "7.0.0-dev.20260113.1", - "jest": "^30.0.0", - "ts-node": "^10.9.1", - "typescript": "^5.3.3" - }, - "files": [ - "dest", - "src", - "!*.test.*", - "!dest/test", - "!src/test" - ], - "types": "./dest/index.d.ts", - "engines": { - "node": ">=20.10" - } -} diff --git a/yarn-project/merkle-tree/package.local.json b/yarn-project/merkle-tree/package.local.json deleted file mode 100644 index 4294a7b630f1..000000000000 --- a/yarn-project/merkle-tree/package.local.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "files": ["dest", "src", "!*.test.*", "!dest/test", "!src/test"] -} diff --git a/yarn-project/merkle-tree/src/hasher_with_stats.ts b/yarn-project/merkle-tree/src/hasher_with_stats.ts deleted file mode 100644 index 87e8f171aacf..000000000000 --- a/yarn-project/merkle-tree/src/hasher_with_stats.ts +++ /dev/null @@ -1,51 +0,0 @@ -import type { Hasher } from '@aztec/foundation/trees'; - -import { createHistogram, performance } from 'perf_hooks'; - -/** - * A helper class to track stats for a Hasher - */ -export class HasherWithStats implements Hasher { - hashCount = 0; - hashInputsCount = 0; - hashHistogram = createHistogram(); - hashInputsHistogram = createHistogram(); - - hash: Hasher['hash']; - hashInputs: Hasher['hashInputs']; - - constructor(hasher: Hasher) { - this.hash = performance.timerify( - (lhs, rhs) => { - this.hashCount++; - return hasher.hash(lhs, rhs); - }, - { histogram: this.hashHistogram }, - ); - this.hashInputs = performance.timerify( - (inputs: Buffer[]) => { - this.hashInputsCount++; - return hasher.hashInputs(inputs); - }, - { histogram: this.hashInputsHistogram }, - ); - } - - stats() { - return { - hashCount: this.hashCount, - // timerify records in ns, convert to ms - hashDuration: this.hashHistogram.mean / 1e6, - hashInputsCount: this.hashInputsCount, - hashInputsDuration: this.hashInputsHistogram.mean / 1e6, - }; - } - - reset() { - this.hashCount = 0; - this.hashHistogram.reset(); - - this.hashInputsCount = 0; - this.hashInputsHistogram.reset(); - } -} diff --git a/yarn-project/merkle-tree/src/index.ts b/yarn-project/merkle-tree/src/index.ts deleted file mode 100644 index af115fbb1c4d..000000000000 --- a/yarn-project/merkle-tree/src/index.ts +++ /dev/null @@ -1,17 +0,0 @@ -export * from './interfaces/append_only_tree.js'; -export * from './interfaces/indexed_tree.js'; -export * from './interfaces/merkle_tree.js'; -export * from './interfaces/update_only_tree.js'; -export * from './poseidon.js'; -export * from './sparse_tree/sparse_tree.js'; -export { StandardIndexedTree } from './standard_indexed_tree/standard_indexed_tree.js'; -export { StandardIndexedTreeWithAppend } from './standard_indexed_tree/test/standard_indexed_tree_with_append.js'; -export * from './standard_tree/standard_tree.js'; -export * from './unbalanced_tree.js'; -export { INITIAL_LEAF, getTreeMeta } from './tree_base.js'; -export { newTree } from './new_tree.js'; -export { loadTree } from './load_tree.js'; -export * from './snapshots/snapshot_builder.js'; -export * from './snapshots/full_snapshot.js'; -export * from './snapshots/append_only_snapshot.js'; -export * from './snapshots/indexed_tree_snapshot.js'; diff --git a/yarn-project/merkle-tree/src/interfaces/append_only_tree.ts b/yarn-project/merkle-tree/src/interfaces/append_only_tree.ts deleted file mode 100644 index 4427dac83f89..000000000000 --- a/yarn-project/merkle-tree/src/interfaces/append_only_tree.ts +++ /dev/null @@ -1,17 +0,0 @@ -import type { Bufferable } from '@aztec/foundation/serialize'; - -import type { TreeSnapshot, TreeSnapshotBuilder } from '../snapshots/snapshot_builder.js'; -import type { MerkleTree } from './merkle_tree.js'; - -/** - * A Merkle tree that supports only appending leaves and not updating existing leaves. - */ -export interface AppendOnlyTree - extends MerkleTree, - TreeSnapshotBuilder> { - /** - * Appends a set of leaf values to the tree. - * @param leaves - The set of leaves to be appended. - */ - appendLeaves(leaves: T[]): void; -} diff --git a/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts b/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts deleted file mode 100644 index c0d11ec16e21..000000000000 --- a/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts +++ /dev/null @@ -1,83 +0,0 @@ -import type { IndexedTreeLeaf, IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; -import type { BatchInsertionResult } from '@aztec/stdlib/trees'; - -import type { IndexedTreeSnapshot, TreeSnapshot, TreeSnapshotBuilder } from '../snapshots/snapshot_builder.js'; -import type { AppendOnlyTree } from './append_only_tree.js'; -import type { MerkleTree } from './merkle_tree.js'; - -/** - * Factory for creating leaf preimages. - */ -export interface PreimageFactory { - /** - * Creates a new preimage from a leaf. - * @param leaf - Leaf to create a preimage from. - * @param nextKey - Next key of the leaf. - * @param nextIndex - Next index of the leaf. - */ - fromLeaf(leaf: IndexedTreeLeaf, nextKey: bigint, nextIndex: bigint): IndexedTreeLeafPreimage; - /** - * Creates a new preimage from a buffer. - * @param buffer - Buffer to create a preimage from. - */ - fromBuffer(buffer: Buffer): IndexedTreeLeafPreimage; - /** - * Creates an empty preimage. - */ - empty(): IndexedTreeLeafPreimage; - /** - * Creates a copy of a preimage. - * @param preimage - Preimage to be cloned. - */ - clone(preimage: IndexedTreeLeafPreimage): IndexedTreeLeafPreimage; -} - -/** - * Indexed merkle tree. - */ -export interface IndexedTree - extends MerkleTree, - TreeSnapshotBuilder, - Omit, keyof TreeSnapshotBuilder>> { - /** - * Finds the index of the largest leaf whose value is less than or equal to the provided value. - * @param newValue - The new value to be inserted into the tree. - * @param includeUncommitted - If true, the uncommitted changes are included in the search. - * @returns The found leaf index and a flag indicating if the corresponding leaf's value is equal to `newValue`. - */ - findIndexOfPreviousKey( - newValue: bigint, - includeUncommitted: boolean, - ): - | { - /** - * The index of the found leaf. - */ - index: bigint; - /** - * A flag indicating if the corresponding leaf's value is equal to `newValue`. - */ - alreadyPresent: boolean; - } - | undefined; - - /** - * Gets the latest LeafPreimage copy. - * @param index - Index of the leaf of which to obtain the LeafPreimage copy. - * @param includeUncommitted - If true, the uncommitted changes are included in the search. - * @returns A copy of the leaf preimage at the given index or undefined if the leaf was not found. - */ - getLatestLeafPreimageCopy(index: bigint, includeUncommitted: boolean): IndexedTreeLeafPreimage | undefined; - - /** - * Batch insert multiple leaves into the tree. - * @param leaves - Leaves to insert into the tree. - * @param subtreeHeight - Height of the subtree. - * @param includeUncommitted - If true, the uncommitted changes are included in the search. - */ - batchInsert( - leaves: Buffer[], - subtreeHeight: SubtreeHeight, - includeUncommitted: boolean, - ): Promise>; -} diff --git a/yarn-project/merkle-tree/src/interfaces/merkle_tree.ts b/yarn-project/merkle-tree/src/interfaces/merkle_tree.ts deleted file mode 100644 index ae2ccd11de75..000000000000 --- a/yarn-project/merkle-tree/src/interfaces/merkle_tree.ts +++ /dev/null @@ -1,70 +0,0 @@ -import type { Bufferable } from '@aztec/foundation/serialize'; -import type { SiblingPath } from '@aztec/foundation/trees'; - -/** - * Defines the interface for a source of sibling paths. - */ -export interface SiblingPathSource { - /** - * Returns the sibling path for a requested leaf index. - * @param index - The index of the leaf for which a sibling path is required. - * @param includeUncommitted - Set to true to include uncommitted updates in the sibling path. - */ - getSiblingPath(index: bigint, includeUncommitted: boolean): Promise>; -} - -/** - * Defines the interface for a Merkle tree. - */ -export interface MerkleTree extends SiblingPathSource { - /** - * Returns the current root of the tree. - * @param includeUncommitted - Set to true to include uncommitted updates in the calculated root. - */ - getRoot(includeUncommitted: boolean): Buffer; - - /** - * Returns the number of leaves in the tree. - * @param includeUncommitted - Set to true to include uncommitted updates in the returned value. - */ - getNumLeaves(includeUncommitted: boolean): bigint; - - /** - * Commit pending updates to the tree. - */ - commit(): Promise; - - /** - * Returns the depth of the tree. - */ - getDepth(): number; - - /** - * Rollback pending update to the tree. - */ - rollback(): Promise; - - /** - * Returns the value of a leaf at the specified index. - * @param index - The index of the leaf value to be returned. - * @param includeUncommitted - Set to true to include uncommitted updates in the data set. - */ - getLeafValue(index: bigint, includeUncommitted: boolean): T | undefined; - - /** - * Returns the index of a leaf given its value, or undefined if no leaf with that value is found. - * @param leaf - The leaf value to look for. - * @param includeUncommitted - Indicates whether to include uncommitted data. - * @returns The index of the first leaf found with a given value (undefined if not found). - */ - findLeafIndex(leaf: T, includeUncommitted: boolean): bigint | undefined; - - /** - * Returns the first index containing a leaf value after `startIndex`. - * @param leaf - The leaf value to look for. - * @param startIndex - The index to start searching from (used when skipping nullified messages) - * @param includeUncommitted - Indicates whether to include uncommitted data. - * @returns The index of the first leaf found with a given value (undefined if not found). - */ - findLeafIndexAfter(leaf: T, startIndex: bigint, includeUncommitted: boolean): bigint | undefined; -} diff --git a/yarn-project/merkle-tree/src/interfaces/update_only_tree.ts b/yarn-project/merkle-tree/src/interfaces/update_only_tree.ts deleted file mode 100644 index 676e449b5aeb..000000000000 --- a/yarn-project/merkle-tree/src/interfaces/update_only_tree.ts +++ /dev/null @@ -1,18 +0,0 @@ -import type { Bufferable } from '@aztec/foundation/serialize'; - -import type { TreeSnapshot, TreeSnapshotBuilder } from '../snapshots/snapshot_builder.js'; -import type { MerkleTree } from './merkle_tree.js'; - -/** - * A Merkle tree that supports updates at arbitrary indices but not appending. - */ -export interface UpdateOnlyTree - extends MerkleTree, - TreeSnapshotBuilder> { - /** - * Updates a leaf at a given index in the tree. - * @param leaf - The leaf value to be updated. - * @param index - The leaf to be updated. - */ - updateLeaf(leaf: T, index: bigint): Promise; -} diff --git a/yarn-project/merkle-tree/src/load_tree.ts b/yarn-project/merkle-tree/src/load_tree.ts deleted file mode 100644 index 751d0ed96c45..000000000000 --- a/yarn-project/merkle-tree/src/load_tree.ts +++ /dev/null @@ -1,33 +0,0 @@ -import type { Bufferable, FromBuffer } from '@aztec/foundation/serialize'; -import type { Hasher } from '@aztec/foundation/trees'; -import type { AztecKVStore } from '@aztec/kv-store'; - -import { type TreeBase, getTreeMeta } from './tree_base.js'; - -/** - * Creates a new tree and sets its root, depth and size based on the meta data which are associated with the name. - * @param c - The class of the tree to be instantiated. - * @param db - A database used to store the Merkle tree data. - * @param hasher - A hasher used to compute hash paths. - * @param name - Name of the tree. - * @returns The newly created tree. - */ -export function loadTree, D extends FromBuffer>( - c: new ( - store: AztecKVStore, - hasher: Hasher, - name: string, - depth: number, - size: bigint, - deserializer: D, - root: Buffer, - ) => T, - store: AztecKVStore, - hasher: Hasher, - name: string, - deserializer: D, -): Promise { - const { root, depth, size } = getTreeMeta(store, name); - const tree = new c(store, hasher, name, depth, size, deserializer, root); - return Promise.resolve(tree); -} diff --git a/yarn-project/merkle-tree/src/new_tree.ts b/yarn-project/merkle-tree/src/new_tree.ts deleted file mode 100644 index 76e376c3c337..000000000000 --- a/yarn-project/merkle-tree/src/new_tree.ts +++ /dev/null @@ -1,29 +0,0 @@ -import type { Bufferable, FromBuffer } from '@aztec/foundation/serialize'; -import type { Hasher } from '@aztec/foundation/trees'; -import type { AztecKVStore } from '@aztec/kv-store'; - -import type { TreeBase } from './tree_base.js'; - -/** - * Creates a new tree. - * @param c - The class of the tree to be instantiated. - * @param db - A database used to store the Merkle tree data. - * @param hasher - A hasher used to compute hash paths. - * @param name - Name of the tree. - * @param depth - Depth of the tree. - * @param prefilledSize - A number of leaves that are prefilled with values. - * @returns The newly created tree. - */ -export async function newTree, D extends FromBuffer>( - c: new (store: AztecKVStore, hasher: Hasher, name: string, depth: number, size: bigint, deserializer: D) => T, - store: AztecKVStore, - hasher: Hasher, - name: string, - deserializer: D, - depth: number, - prefilledSize = 1, -): Promise { - const tree = new c(store, hasher, name, depth, 0n, deserializer); - await tree.init(prefilledSize); - return tree; -} diff --git a/yarn-project/merkle-tree/src/poseidon.ts b/yarn-project/merkle-tree/src/poseidon.ts deleted file mode 100644 index 3dbfa2c6fcb4..000000000000 --- a/yarn-project/merkle-tree/src/poseidon.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { poseidon2Hash } from '@aztec/foundation/crypto/sync'; -import { Fr } from '@aztec/foundation/curves/bn254'; -import type { Hasher } from '@aztec/foundation/trees'; - -/** - * A helper class encapsulating poseidon2 hash functionality. - * @deprecated Don't call poseidon2 directly in production code. Instead, create suitably-named functions for specific - * purposes. - */ -export class Poseidon implements Hasher { - /* - * @deprecated Don't call poseidon2 directly in production code. Instead, create suitably-named functions for specific - * purposes. - */ - public hash(lhs: Uint8Array, rhs: Uint8Array) { - return poseidon2Hash([ - Fr.fromBuffer(Buffer.from(lhs)), - Fr.fromBuffer(Buffer.from(rhs)), - ]).toBuffer() as Buffer; - } - - /* - * @deprecated Don't call poseidon2 directly in production code. Instead, create suitably-named functions for specific - * purposes. - */ - public hashInputs(inputs: Buffer[]) { - const inputFields = inputs.map(i => Fr.fromBuffer(i)); - return poseidon2Hash(inputFields).toBuffer() as Buffer; - } -} diff --git a/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.test.ts b/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.test.ts deleted file mode 100644 index 9712657f6295..000000000000 --- a/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.test.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { Fr } from '@aztec/foundation/curves/bn254'; -import type { FromBuffer } from '@aztec/foundation/serialize'; -import type { AztecKVStore } from '@aztec/kv-store'; -import { openTmpStore } from '@aztec/kv-store/lmdb'; - -import { Poseidon, StandardTree, newTree } from '../index.js'; -import { AppendOnlySnapshotBuilder } from './append_only_snapshot.js'; -import { describeSnapshotBuilderTestSuite } from './snapshot_builder_test_suite.js'; - -describe('AppendOnlySnapshot', () => { - let tree: StandardTree; - let snapshotBuilder: AppendOnlySnapshotBuilder; - let db: AztecKVStore; - - beforeEach(async () => { - db = openTmpStore(); - const hasher = new Poseidon(); - const deserializer: FromBuffer = { fromBuffer: b => b }; - tree = await newTree(StandardTree, db, hasher, 'test', deserializer, 4); - snapshotBuilder = new AppendOnlySnapshotBuilder(db, tree, hasher, deserializer); - }); - - describeSnapshotBuilderTestSuite( - () => tree, - () => snapshotBuilder, - tree => { - const newLeaves = Array.from({ length: 2 }).map(() => Fr.random().toBuffer()); - tree.appendLeaves(newLeaves); - return Promise.resolve(); - }, - ); -}); diff --git a/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.ts b/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.ts deleted file mode 100644 index 7c878e775c64..000000000000 --- a/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.ts +++ /dev/null @@ -1,279 +0,0 @@ -import type { BlockNumber } from '@aztec/foundation/branded-types'; -import { type Bufferable, type FromBuffer, serializeToBuffer } from '@aztec/foundation/serialize'; -import { SiblingPath } from '@aztec/foundation/trees'; -import type { Hasher } from '@aztec/foundation/trees'; -import type { AztecKVStore, AztecMap } from '@aztec/kv-store'; - -import type { AppendOnlyTree } from '../interfaces/append_only_tree.js'; -import type { TreeBase } from '../tree_base.js'; -import type { TreeSnapshot, TreeSnapshotBuilder } from './snapshot_builder.js'; - -// stores the last block that modified this node -const nodeModifiedAtBlockKey = (level: number, index: bigint) => `node:${level}:${index}:modifiedAtBlock`; - -// stores the value of the node at the above block -const historicalNodeKey = (level: number, index: bigint) => `node:${level}:${index}:value`; - -/** - * Metadata for a snapshot, per block - */ -type SnapshotMetadata = { - /** The tree root at the time */ - root: Buffer; - /** The number of filled leaves */ - numLeaves: bigint; -}; - -/** - * A more space-efficient way of storing snapshots of AppendOnlyTrees that trades space need for slower - * sibling path reads. - * - * Complexity: - * - * N - count of non-zero nodes in tree - * M - count of snapshots - * H - tree height - * - * Space complexity: O(N + M) (N nodes - stores the last snapshot for each node and M - ints, for each snapshot stores up to which leaf its written to) - * Sibling path access: - * Best case: O(H) database reads + O(1) hashes - * Worst case: O(H) database reads + O(H) hashes - */ -export class AppendOnlySnapshotBuilder implements TreeSnapshotBuilder> { - #nodeValue: AztecMap, Buffer>; - #nodeLastModifiedByBlock: AztecMap, BlockNumber>; - #snapshotMetadata: AztecMap; - - constructor( - private db: AztecKVStore, - private tree: TreeBase & AppendOnlyTree, - private hasher: Hasher, - private deserializer: FromBuffer, - ) { - const treeName = tree.getName(); - this.#nodeValue = db.openMap(`append_only_snapshot:${treeName}:node`); - this.#nodeLastModifiedByBlock = db.openMap(`append_ony_snapshot:${treeName}:block`); - this.#snapshotMetadata = db.openMap(`append_only_snapshot:${treeName}:snapshot_metadata`); - } - - getSnapshot(block: BlockNumber): Promise> { - const meta = this.#getSnapshotMeta(block); - - if (typeof meta === 'undefined') { - return Promise.reject(new Error(`Snapshot for tree ${this.tree.getName()} at block ${block} does not exist`)); - } - - return Promise.resolve( - new AppendOnlySnapshot( - this.#nodeValue, - this.#nodeLastModifiedByBlock, - block, - meta.numLeaves, - meta.root, - this.tree, - this.hasher, - this.deserializer, - ), - ); - } - - snapshot(block: BlockNumber): Promise> { - return this.db.transaction(() => { - const meta = this.#getSnapshotMeta(block); - if (typeof meta !== 'undefined') { - // no-op, we already have a snapshot - return new AppendOnlySnapshot( - this.#nodeValue, - this.#nodeLastModifiedByBlock, - block, - meta.numLeaves, - meta.root, - this.tree, - this.hasher, - this.deserializer, - ); - } - - const root = this.tree.getRoot(false); - const depth = this.tree.getDepth(); - const queue: [Buffer, number, bigint][] = [[root, 0, 0n]]; - - // walk the tree in BF and store latest nodes - while (queue.length > 0) { - const [node, level, index] = queue.shift()!; - - const historicalValue = this.#nodeValue.get(historicalNodeKey(level, index)); - if (!historicalValue || !node.equals(historicalValue)) { - // we've never seen this node before or it's different than before - // update the historical tree and tag it with the block that modified it - void this.#nodeLastModifiedByBlock.set(nodeModifiedAtBlockKey(level, index), block); - void this.#nodeValue.set(historicalNodeKey(level, index), node); - } else { - // if this node hasn't changed, that means, nothing below it has changed either - continue; - } - - if (level + 1 > depth) { - // short circuit if we've reached the leaf level - // otherwise getNode might throw if we ask for the children of a leaf - continue; - } - - // these could be undefined because zero hashes aren't stored in the tree - const [lhs, rhs] = [this.tree.getNode(level + 1, 2n * index), this.tree.getNode(level + 1, 2n * index + 1n)]; - - if (lhs) { - queue.push([lhs, level + 1, 2n * index]); - } - - if (rhs) { - queue.push([rhs, level + 1, 2n * index + 1n]); - } - } - - const numLeaves = this.tree.getNumLeaves(false); - void this.#snapshotMetadata.set(block, { - numLeaves, - root, - }); - - return new AppendOnlySnapshot( - this.#nodeValue, - this.#nodeLastModifiedByBlock, - block, - numLeaves, - root, - this.tree, - this.hasher, - this.deserializer, - ); - }); - } - - #getSnapshotMeta(block: BlockNumber): SnapshotMetadata | undefined { - return this.#snapshotMetadata.get(block); - } -} - -/** - * a - */ -class AppendOnlySnapshot implements TreeSnapshot { - constructor( - private nodes: AztecMap, - private nodeHistory: AztecMap, - private block: BlockNumber, - private leafCount: bigint, - private historicalRoot: Buffer, - private tree: TreeBase & AppendOnlyTree, - private hasher: Hasher, - private deserializer: FromBuffer, - ) {} - - public getSiblingPath(index: bigint): SiblingPath { - const path: Buffer[] = []; - const depth = this.tree.getDepth(); - let level = depth; - - while (level > 0) { - const isRight = index & 0x01n; - const siblingIndex = isRight ? index - 1n : index + 1n; - - const sibling = this.#getHistoricalNodeValue(level, siblingIndex); - path.push(sibling); - - level -= 1; - index >>= 1n; - } - - return new SiblingPath(depth as N, path); - } - - getDepth(): number { - return this.tree.getDepth(); - } - - getNumLeaves(): bigint { - return this.leafCount; - } - - getRoot(): Buffer { - // we could recompute it, but it's way cheaper to just store the root - return this.historicalRoot; - } - - getLeafValue(index: bigint): T | undefined { - const leafLevel = this.getDepth(); - const blockNumber = this.#getBlockNumberThatModifiedNode(leafLevel, index); - - // leaf hasn't been set yet - if (typeof blockNumber === 'undefined') { - return undefined; - } - - // leaf was set some time in the past - if (blockNumber <= this.block) { - const val = this.nodes.get(historicalNodeKey(leafLevel, index)); - return val ? this.deserializer.fromBuffer(val) : undefined; - } - - // leaf has been set but in a block in the future - return undefined; - } - - #getHistoricalNodeValue(level: number, index: bigint): Buffer { - const blockNumber = this.#getBlockNumberThatModifiedNode(level, index); - - // node has never been set - if (typeof blockNumber === 'undefined') { - return this.tree.getZeroHash(level); - } - - // node was set some time in the past - if (blockNumber <= this.block) { - return this.nodes.get(historicalNodeKey(level, index))!; - } - - // the node has been modified since this snapshot was taken - // because we're working with an AppendOnly tree, historical leaves never change - // so what we do instead is rebuild this Merkle path up using zero hashes as needed - // worst case this will do O(H) hashes - // - // we first check if this subtree was touched by the block - // compare how many leaves this block added to the leaf interval of this subtree - // if they don't intersect then the whole subtree was a hash of zero - // if they do then we need to rebuild the merkle tree - const depth = this.tree.getDepth(); - const leafStart = index * 2n ** BigInt(depth - level); - if (leafStart >= this.leafCount) { - return this.tree.getZeroHash(level); - } - - const [lhs, rhs] = [ - this.#getHistoricalNodeValue(level + 1, 2n * index), - this.#getHistoricalNodeValue(level + 1, 2n * index + 1n), - ]; - - return this.hasher.hash(lhs, rhs); - } - - #getBlockNumberThatModifiedNode(level: number, index: bigint): BlockNumber | undefined { - return this.nodeHistory.get(nodeModifiedAtBlockKey(level, index)); - } - - findLeafIndex(value: T): bigint | undefined { - return this.findLeafIndexAfter(value, 0n); - } - - findLeafIndexAfter(value: T, startIndex: bigint): bigint | undefined { - const valueBuffer = serializeToBuffer(value); - const numLeaves = this.getNumLeaves(); - for (let i = startIndex; i < numLeaves; i++) { - const currentValue = this.getLeafValue(i); - if (currentValue && serializeToBuffer(currentValue).equals(valueBuffer)) { - return i; - } - } - return undefined; - } -} diff --git a/yarn-project/merkle-tree/src/snapshots/base_full_snapshot.ts b/yarn-project/merkle-tree/src/snapshots/base_full_snapshot.ts deleted file mode 100644 index a70358e01419..000000000000 --- a/yarn-project/merkle-tree/src/snapshots/base_full_snapshot.ts +++ /dev/null @@ -1,226 +0,0 @@ -import type { BlockNumber } from '@aztec/foundation/branded-types'; -import { type Bufferable, type FromBuffer, serializeToBuffer } from '@aztec/foundation/serialize'; -import { SiblingPath } from '@aztec/foundation/trees'; -import type { AztecKVStore, AztecMap } from '@aztec/kv-store'; - -import type { TreeBase } from '../tree_base.js'; -import type { TreeSnapshot, TreeSnapshotBuilder } from './snapshot_builder.js'; - -/** - * Metadata for a snapshot, per block - */ -type SnapshotMetadata = { - /** The tree root at the time */ - root: Buffer; - /** The number of filled leaves */ - numLeaves: bigint; -}; - -/** - * Builds a full snapshot of a tree. This implementation works for any Merkle tree and stores - * it in a database in a similar way to how a tree is stored in memory, using pointers. - * - * Sharing the same database between versions and trees is recommended as the trees would share - * structure. - * - * Implement the protected method `handleLeaf` to store any additional data you need for each leaf. - * - * Complexity: - * N - count of non-zero nodes in tree - * M - count of snapshots - * H - tree height - * Worst case space complexity: O(N * M) - * Sibling path access: O(H) database reads - */ -export abstract class BaseFullTreeSnapshotBuilder, S extends TreeSnapshot> - implements TreeSnapshotBuilder -{ - protected nodes: AztecMap; - protected snapshotMetadata: AztecMap; - - constructor( - protected db: AztecKVStore, - protected tree: T, - ) { - this.nodes = db.openMap(`full_snapshot:${tree.getName()}:node`); - this.snapshotMetadata = db.openMap(`full_snapshot:${tree.getName()}:metadata`); - } - - snapshot(block: BlockNumber): Promise { - return this.db.transaction(() => { - const snapshotMetadata = this.#getSnapshotMeta(block); - - if (snapshotMetadata) { - return this.openSnapshot(snapshotMetadata.root, snapshotMetadata.numLeaves); - } - - const root = this.tree.getRoot(false); - const numLeaves = this.tree.getNumLeaves(false); - const depth = this.tree.getDepth(); - const queue: [Buffer, number, bigint][] = [[root, 0, 0n]]; - - // walk the tree breadth-first and store each of its nodes in the database - // for each node we save two keys - // :0 -> - // :1 -> - while (queue.length > 0) { - const [node, level, i] = queue.shift()!; - const nodeKey = node.toString('hex'); - // check if the database already has a child for this tree - // if it does, then we know we've seen the whole subtree below it before - // and we don't have to traverse it anymore - // we use the left child here, but it could be anything that shows we've stored the node before - if (this.nodes.has(nodeKey)) { - continue; - } - - if (level + 1 > depth) { - // short circuit if we've reached the leaf level - // otherwise getNode might throw if we ask for the children of a leaf - this.handleLeaf(i, node); - continue; - } - - const [lhs, rhs] = [this.tree.getNode(level + 1, 2n * i), this.tree.getNode(level + 1, 2n * i + 1n)]; - - // we want the zero hash at the children's level, not the node's level - const zeroHash = this.tree.getZeroHash(level + 1); - - void this.nodes.set(nodeKey, [lhs ?? zeroHash, rhs ?? zeroHash]); - // enqueue the children only if they're not zero hashes - if (lhs) { - queue.push([lhs, level + 1, 2n * i]); - } - - if (rhs) { - queue.push([rhs, level + 1, 2n * i + 1n]); - } - } - - void this.snapshotMetadata.set(block, { root, numLeaves }); - return this.openSnapshot(root, numLeaves); - }); - } - - protected handleLeaf(_index: bigint, _node: Buffer): void {} - - getSnapshot(version: BlockNumber): Promise { - const snapshotMetadata = this.#getSnapshotMeta(version); - - if (!snapshotMetadata) { - return Promise.reject(new Error(`Version ${version} does not exist for tree ${this.tree.getName()}`)); - } - - return Promise.resolve(this.openSnapshot(snapshotMetadata.root, snapshotMetadata.numLeaves)); - } - - protected abstract openSnapshot(root: Buffer, numLeaves: bigint): S; - - #getSnapshotMeta(block: BlockNumber): SnapshotMetadata | undefined { - return this.snapshotMetadata.get(block); - } -} - -/** - * A source of sibling paths from a snapshot tree - */ -export class BaseFullTreeSnapshot implements TreeSnapshot { - constructor( - protected db: AztecMap, - protected historicRoot: Buffer, - protected numLeaves: bigint, - protected tree: TreeBase, - protected deserializer: FromBuffer, - ) {} - - getSiblingPath(index: bigint): SiblingPath { - const siblings: Buffer[] = []; - - for (const [_node, sibling] of this.pathFromRootToLeaf(index)) { - siblings.push(sibling); - } - - // we got the siblings we were looking for, but they are in root-leaf order - // reverse them here so we have leaf-root (what SiblingPath expects) - siblings.reverse(); - - return new SiblingPath(this.tree.getDepth() as N, siblings); - } - - getLeafValue(index: bigint): T | undefined { - let leafNode: Buffer | undefined = undefined; - for (const [node, _sibling] of this.pathFromRootToLeaf(index)) { - leafNode = node; - } - - return leafNode ? this.deserializer.fromBuffer(leafNode) : undefined; - } - - getDepth(): number { - return this.tree.getDepth(); - } - - getRoot(): Buffer { - return this.historicRoot; - } - - getNumLeaves(): bigint { - return this.numLeaves; - } - - protected *pathFromRootToLeaf(leafIndex: bigint) { - const root = this.historicRoot; - const pathFromRoot = this.#getPathFromRoot(leafIndex); - - let node: Buffer = root; - for (let i = 0; i < pathFromRoot.length; i++) { - // get both children. We'll need both anyway (one to keep track of, the other to walk down to) - const children: [Buffer, Buffer] = this.db.get(node.toString('hex')) ?? [ - this.tree.getZeroHash(i + 1), - this.tree.getZeroHash(i + 1), - ]; - const next = children[pathFromRoot[i]]; - const sibling = children[(pathFromRoot[i] + 1) % 2]; - - yield [next, sibling]; - - node = next; - } - } - - /** - * Calculates the path from the root to the target leaf. Returns an array of 0s and 1s, - * each 0 represents walking down a left child and each 1 walking down to the child on the right. - * - * @param leafIndex - The target leaf - * @returns An array of 0s and 1s - */ - #getPathFromRoot(leafIndex: bigint): ReadonlyArray<0 | 1> { - const path: Array<0 | 1> = []; - let level = this.tree.getDepth(); - while (level > 0) { - path.push(leafIndex & 0x01n ? 1 : 0); - leafIndex >>= 1n; - level--; - } - - path.reverse(); - return path; - } - - findLeafIndex(value: T): bigint | undefined { - return this.findLeafIndexAfter(value, 0n); - } - - public findLeafIndexAfter(value: T, startIndex: bigint): bigint | undefined { - const numLeaves = this.getNumLeaves(); - const buffer = serializeToBuffer(value); - for (let i = startIndex; i < numLeaves; i++) { - const currentValue = this.getLeafValue(i); - if (currentValue && serializeToBuffer(currentValue).equals(buffer)) { - return i; - } - } - return undefined; - } -} diff --git a/yarn-project/merkle-tree/src/snapshots/full_snapshot.test.ts b/yarn-project/merkle-tree/src/snapshots/full_snapshot.test.ts deleted file mode 100644 index f943b9bbdb66..000000000000 --- a/yarn-project/merkle-tree/src/snapshots/full_snapshot.test.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { Fr } from '@aztec/foundation/curves/bn254'; -import type { FromBuffer } from '@aztec/foundation/serialize'; -import type { AztecKVStore } from '@aztec/kv-store'; -import { openTmpStore } from '@aztec/kv-store/lmdb'; - -import { Poseidon, StandardTree, newTree } from '../index.js'; -import { FullTreeSnapshotBuilder } from './full_snapshot.js'; -import { describeSnapshotBuilderTestSuite } from './snapshot_builder_test_suite.js'; - -describe('FullSnapshotBuilder', () => { - let tree: StandardTree; - let snapshotBuilder: FullTreeSnapshotBuilder; - let db: AztecKVStore; - - beforeEach(async () => { - db = openTmpStore(); - const deserializer: FromBuffer = { fromBuffer: b => b }; - tree = await newTree(StandardTree, db, new Poseidon(), 'test', deserializer, 4); - snapshotBuilder = new FullTreeSnapshotBuilder(db, tree, deserializer); - }); - - describeSnapshotBuilderTestSuite( - () => tree, - () => snapshotBuilder, - () => { - const newLeaves = Array.from({ length: 2 }).map(() => Fr.random().toBuffer()); - tree.appendLeaves(newLeaves); - return Promise.resolve(); - }, - ); -}); diff --git a/yarn-project/merkle-tree/src/snapshots/full_snapshot.ts b/yarn-project/merkle-tree/src/snapshots/full_snapshot.ts deleted file mode 100644 index 166b64af41cd..000000000000 --- a/yarn-project/merkle-tree/src/snapshots/full_snapshot.ts +++ /dev/null @@ -1,37 +0,0 @@ -import type { Bufferable, FromBuffer } from '@aztec/foundation/serialize'; -import type { AztecKVStore } from '@aztec/kv-store'; - -import type { TreeBase } from '../tree_base.js'; -import { BaseFullTreeSnapshot, BaseFullTreeSnapshotBuilder } from './base_full_snapshot.js'; -import type { TreeSnapshot, TreeSnapshotBuilder } from './snapshot_builder.js'; - -/** - * Builds a full snapshot of a tree. This implementation works for any Merkle tree and stores - * it in a database in a similar way to how a tree is stored in memory, using pointers. - * - * Sharing the same database between versions and trees is recommended as the trees would share - * structure. - * - * Complexity: - * N - count of non-zero nodes in tree - * M - count of snapshots - * H - tree height - * Worst case space complexity: O(N * M) - * Sibling path access: O(H) database reads - */ -export class FullTreeSnapshotBuilder - extends BaseFullTreeSnapshotBuilder, TreeSnapshot> - implements TreeSnapshotBuilder> -{ - constructor( - db: AztecKVStore, - tree: TreeBase, - private deserializer: FromBuffer, - ) { - super(db, tree); - } - - protected openSnapshot(root: Buffer, numLeaves: bigint): TreeSnapshot { - return new BaseFullTreeSnapshot(this.nodes, root, numLeaves, this.tree, this.deserializer); - } -} diff --git a/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.test.ts b/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.test.ts deleted file mode 100644 index 7d558a5f6df2..000000000000 --- a/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.test.ts +++ /dev/null @@ -1,115 +0,0 @@ -import { BlockNumber } from '@aztec/foundation/branded-types'; -import { Fr } from '@aztec/foundation/curves/bn254'; -import type { Hasher } from '@aztec/foundation/trees'; -import type { AztecKVStore } from '@aztec/kv-store'; -import { openTmpStore } from '@aztec/kv-store/lmdb'; -import { NullifierLeaf, NullifierLeafPreimage } from '@aztec/stdlib/trees'; - -import { Poseidon, newTree } from '../index.js'; -import { StandardIndexedTreeWithAppend } from '../standard_indexed_tree/test/standard_indexed_tree_with_append.js'; -import { IndexedTreeSnapshotBuilder } from './indexed_tree_snapshot.js'; -import { describeSnapshotBuilderTestSuite } from './snapshot_builder_test_suite.js'; - -class NullifierTree extends StandardIndexedTreeWithAppend { - constructor( - db: AztecKVStore, - hasher: Hasher, - name: string, - depth: number, - size: bigint = 0n, - _noop: any, - root?: Buffer, - ) { - super(db, hasher, name, depth, size, NullifierLeafPreimage, NullifierLeaf, root); - } -} - -describe('IndexedTreeSnapshotBuilder', () => { - let db: AztecKVStore; - let tree: StandardIndexedTreeWithAppend; - let snapshotBuilder: IndexedTreeSnapshotBuilder; - - beforeEach(async () => { - db = openTmpStore(); - tree = await newTree(NullifierTree, db, new Poseidon(), 'test', { fromBuffer: (b: Buffer) => b }, 4); - snapshotBuilder = new IndexedTreeSnapshotBuilder(db, tree, NullifierLeafPreimage); - }); - - describeSnapshotBuilderTestSuite( - () => tree, - () => snapshotBuilder, - () => { - const newLeaves = Array.from({ length: 2 }).map(() => new NullifierLeaf(Fr.random()).toBuffer()); - tree.appendLeaves(newLeaves); - return Promise.resolve(); - }, - ); - - describe('getSnapshot', () => { - it('returns historical leaf data', async () => { - tree.appendLeaves([Fr.random().toBuffer(), Fr.random().toBuffer(), Fr.random().toBuffer()]); - await tree.commit(); - const expectedLeavesAtBlock1 = await Promise.all([ - tree.getLatestLeafPreimageCopy(0n, false), - tree.getLatestLeafPreimageCopy(1n, false), - tree.getLatestLeafPreimageCopy(2n, false), - // id'expect these to be undefined, but leaf 3 isn't? - // must be some indexed-tree quirk I don't quite understand yet - tree.getLatestLeafPreimageCopy(3n, false), - tree.getLatestLeafPreimageCopy(4n, false), - tree.getLatestLeafPreimageCopy(5n, false), - ]); - - await snapshotBuilder.snapshot(BlockNumber(1)); - - tree.appendLeaves([Fr.random().toBuffer(), Fr.random().toBuffer(), Fr.random().toBuffer()]); - await tree.commit(); - const expectedLeavesAtBlock2 = [ - tree.getLatestLeafPreimageCopy(0n, false), - tree.getLatestLeafPreimageCopy(1n, false), - tree.getLatestLeafPreimageCopy(2n, false), - tree.getLatestLeafPreimageCopy(3n, false), - tree.getLatestLeafPreimageCopy(4n, false), - tree.getLatestLeafPreimageCopy(5n, false), - ]; - - await snapshotBuilder.snapshot(BlockNumber(2)); - - const snapshot1 = await snapshotBuilder.getSnapshot(BlockNumber(1)); - const actualLeavesAtBlock1 = [ - snapshot1.getLatestLeafPreimageCopy(0n), - snapshot1.getLatestLeafPreimageCopy(1n), - snapshot1.getLatestLeafPreimageCopy(2n), - snapshot1.getLatestLeafPreimageCopy(3n), - snapshot1.getLatestLeafPreimageCopy(4n), - snapshot1.getLatestLeafPreimageCopy(5n), - ]; - expect(actualLeavesAtBlock1).toEqual(expectedLeavesAtBlock1); - - const snapshot2 = await snapshotBuilder.getSnapshot(BlockNumber(2)); - const actualLeavesAtBlock2 = await Promise.all([ - snapshot2.getLatestLeafPreimageCopy(0n), - snapshot2.getLatestLeafPreimageCopy(1n), - snapshot2.getLatestLeafPreimageCopy(2n), - snapshot2.getLatestLeafPreimageCopy(3n), - snapshot2.getLatestLeafPreimageCopy(4n), - snapshot2.getLatestLeafPreimageCopy(5n), - ]); - expect(actualLeavesAtBlock2).toEqual(expectedLeavesAtBlock2); - }); - }); - - describe('findIndexOfPreviousValue', () => { - it('returns the index of the leaf with the closest value to the given value', async () => { - tree.appendLeaves([Fr.random().toBuffer(), Fr.random().toBuffer(), Fr.random().toBuffer()]); - await tree.commit(); - const snapshot = await snapshotBuilder.snapshot(BlockNumber(1)); - const historicalPrevValue = tree.findIndexOfPreviousKey(2n, false); - - tree.appendLeaves([Fr.random().toBuffer(), Fr.random().toBuffer(), Fr.random().toBuffer()]); - await tree.commit(); - - expect(snapshot.findIndexOfPreviousKey(2n)).toEqual(historicalPrevValue); - }); - }); -}); diff --git a/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.ts b/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.ts deleted file mode 100644 index f94b40f9e126..000000000000 --- a/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.ts +++ /dev/null @@ -1,112 +0,0 @@ -import type { IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; -import type { AztecKVStore, AztecMap } from '@aztec/kv-store'; - -import type { IndexedTree, PreimageFactory } from '../interfaces/indexed_tree.js'; -import type { TreeBase } from '../tree_base.js'; -import { BaseFullTreeSnapshot, BaseFullTreeSnapshotBuilder } from './base_full_snapshot.js'; -import type { IndexedTreeSnapshot, TreeSnapshotBuilder } from './snapshot_builder.js'; - -const snapshotLeafValue = (node: Buffer, index: bigint) => 'snapshot:leaf:' + node.toString('hex') + ':' + index; - -/** a */ -export class IndexedTreeSnapshotBuilder - extends BaseFullTreeSnapshotBuilder, IndexedTreeSnapshot> - implements TreeSnapshotBuilder -{ - leaves: AztecMap; - constructor( - store: AztecKVStore, - tree: IndexedTree & TreeBase, - private leafPreimageBuilder: PreimageFactory, - ) { - super(store, tree); - this.leaves = store.openMap('indexed_tree_snapshot:' + tree.getName()); - } - - protected openSnapshot(root: Buffer, numLeaves: bigint): IndexedTreeSnapshot { - return new IndexedTreeSnapshotImpl(this.nodes, this.leaves, root, numLeaves, this.tree, this.leafPreimageBuilder); - } - - protected override handleLeaf(index: bigint, node: Buffer) { - const leafPreimage = this.tree.getLatestLeafPreimageCopy(index, false); - if (leafPreimage) { - void this.leaves.set(snapshotLeafValue(node, index), leafPreimage.toBuffer()); - } - } -} - -/** A snapshot of an indexed tree at a particular point in time */ -class IndexedTreeSnapshotImpl extends BaseFullTreeSnapshot implements IndexedTreeSnapshot { - constructor( - db: AztecMap, - private leaves: AztecMap, - historicRoot: Buffer, - numLeaves: bigint, - tree: IndexedTree & TreeBase, - private leafPreimageBuilder: PreimageFactory, - ) { - super(db, historicRoot, numLeaves, tree, { fromBuffer: buf => buf }); - } - - override getLeafValue(index: bigint): Buffer | undefined { - const leafPreimage = this.getLatestLeafPreimageCopy(index); - return leafPreimage?.toBuffer(); - } - - getLatestLeafPreimageCopy(index: bigint): IndexedTreeLeafPreimage | undefined { - const leafNode = super.getLeafValue(index); - const leafValue = this.leaves.get(snapshotLeafValue(leafNode!, index)); - if (leafValue) { - return this.leafPreimageBuilder.fromBuffer(leafValue); - } else { - return undefined; - } - } - - findIndexOfPreviousKey(newValue: bigint): { - /** - * The index of the found leaf. - */ - index: bigint; - /** - * A flag indicating if the corresponding leaf's value is equal to `newValue`. - */ - alreadyPresent: boolean; - } { - const numLeaves = this.getNumLeaves(); - const diff: bigint[] = []; - - for (let i = 0; i < numLeaves; i++) { - // this is very inefficient - const storedLeaf = this.getLatestLeafPreimageCopy(BigInt(i))!; - - // The stored leaf can be undefined if it addresses an empty leaf - // If the leaf is empty we do the same as if the leaf was larger - if (storedLeaf === undefined) { - diff.push(newValue); - } else if (storedLeaf.getKey() > newValue) { - diff.push(newValue); - } else if (storedLeaf.getKey() === newValue) { - return { index: BigInt(i), alreadyPresent: true }; - } else { - diff.push(newValue - storedLeaf.getKey()); - } - } - - let minIndex = 0; - for (let i = 1; i < diff.length; i++) { - if (diff[i] < diff[minIndex]) { - minIndex = i; - } - } - - return { index: BigInt(minIndex), alreadyPresent: false }; - } - - override findLeafIndex(value: Buffer): bigint | undefined { - const index = this.tree.findLeafIndex(value, false); - if (index !== undefined && index < this.getNumLeaves()) { - return index; - } - } -} diff --git a/yarn-project/merkle-tree/src/snapshots/snapshot_builder.ts b/yarn-project/merkle-tree/src/snapshots/snapshot_builder.ts deleted file mode 100644 index 242937fe5c1f..000000000000 --- a/yarn-project/merkle-tree/src/snapshots/snapshot_builder.ts +++ /dev/null @@ -1,93 +0,0 @@ -import type { BlockNumber } from '@aztec/foundation/branded-types'; -import type { Bufferable } from '@aztec/foundation/serialize'; -import type { IndexedTreeLeafPreimage, SiblingPath } from '@aztec/foundation/trees'; - -/** - * An interface for a tree that can record snapshots of its contents. - */ -export interface TreeSnapshotBuilder> { - /** - * Creates a snapshot of the tree at the given version. - * @param block - The version to snapshot the tree at. - */ - snapshot(block: BlockNumber): Promise; - - /** - * Returns a snapshot of the tree at the given version. - * @param block - The version of the snapshot to return. - */ - getSnapshot(block: BlockNumber): Promise; -} - -/** - * A tree snapshot - */ -export interface TreeSnapshot { - /** - * Returns the current root of the tree. - */ - getRoot(): Buffer; - - /** - * Returns the number of leaves in the tree. - */ - getDepth(): number; - - /** - * Returns the number of leaves in the tree. - */ - getNumLeaves(): bigint; - - /** - * Returns the value of a leaf at the specified index. - * @param index - The index of the leaf value to be returned. - */ - getLeafValue(index: bigint): T | undefined; - - /** - * Returns the sibling path for a requested leaf index. - * @param index - The index of the leaf for which a sibling path is required. - */ - getSiblingPath(index: bigint): SiblingPath; - - /** - * Returns the index of a leaf given its value, or undefined if no leaf with that value is found. - * @param treeId - The ID of the tree. - * @param value - The leaf value to look for. - * @returns The index of the first leaf found with a given value (undefined if not found). - */ - findLeafIndex(value: T): bigint | undefined; - - /** - * Returns the first index containing a leaf value after `startIndex`. - * @param leaf - The leaf value to look for. - * @param startIndex - The index to start searching from (used when skipping nullified messages) - * @returns The index of the first leaf found with a given value (undefined if not found). - */ - findLeafIndexAfter(leaf: T, startIndex: bigint): bigint | undefined; -} - -/** A snapshot of an indexed tree */ -export interface IndexedTreeSnapshot extends TreeSnapshot { - /** - * Gets the historical data for a leaf - * @param index - The index of the leaf to get the data for - */ - getLatestLeafPreimageCopy(index: bigint): IndexedTreeLeafPreimage | undefined; - - /** - * Finds the index of the largest leaf whose value is less than or equal to the provided value. - * @param newValue - The new value to be inserted into the tree. - * @returns The found leaf index and a flag indicating if the corresponding leaf's value is equal to `newValue`. - */ - findIndexOfPreviousKey(newValue: bigint): { - /** - * The index of the found leaf. - */ - index: bigint; - /** - * A flag indicating if the corresponding leaf's value is equal to `newValue`. - */ - alreadyPresent: boolean; - }; -} diff --git a/yarn-project/merkle-tree/src/snapshots/snapshot_builder_test_suite.ts b/yarn-project/merkle-tree/src/snapshots/snapshot_builder_test_suite.ts deleted file mode 100644 index a7fbdf183661..000000000000 --- a/yarn-project/merkle-tree/src/snapshots/snapshot_builder_test_suite.ts +++ /dev/null @@ -1,227 +0,0 @@ -import { BlockNumber } from '@aztec/foundation/branded-types'; -import { randomBigInt } from '@aztec/foundation/crypto/random'; -import type { Bufferable } from '@aztec/foundation/serialize'; - -import { jest } from '@jest/globals'; - -import type { TreeBase } from '../tree_base.js'; -import type { TreeSnapshot, TreeSnapshotBuilder } from './snapshot_builder.js'; - -jest.setTimeout(50_000); - -/** Creates a test suit for snapshots */ -export function describeSnapshotBuilderTestSuite< - T extends TreeBase, - S extends TreeSnapshotBuilder>, ->(getTree: () => T, getSnapshotBuilder: () => S, modifyTree: (tree: T) => Promise) { - describe('SnapshotBuilder', () => { - let tree: T; - let snapshotBuilder: S; - let leaves: bigint[]; - - beforeEach(() => { - tree = getTree(); - snapshotBuilder = getSnapshotBuilder(); - - leaves = Array.from({ length: 4 }).map(() => randomBigInt(BigInt(2 ** tree.getDepth()))); - }); - - describe('snapshot', () => { - it('takes snapshots', async () => { - await modifyTree(tree); - await tree.commit(); - await expect(snapshotBuilder.snapshot(BlockNumber(1))).resolves.toBeDefined(); - }); - - it('is idempotent', async () => { - await modifyTree(tree); - await tree.commit(); - - const block = BlockNumber(1); - const snapshot = await snapshotBuilder.snapshot(block); - const newSnapshot = await snapshotBuilder.snapshot(block); - - expect(newSnapshot.getRoot()).toEqual(snapshot.getRoot()); - }); - - it('returns the same path if tree has not diverged', async () => { - await modifyTree(tree); - await tree.commit(); - const snapshot = await snapshotBuilder.snapshot(BlockNumber(1)); - - const historicPaths = await Promise.all(leaves.map(leaf => snapshot.getSiblingPath(leaf))); - const expectedPaths = await Promise.all(leaves.map(leaf => tree.getSiblingPath(leaf, false))); - - for (const [index, path] of historicPaths.entries()) { - expect(path).toEqual(expectedPaths[index]); - } - }); - - it('returns historic paths if tree has diverged and no new snapshots have been taken', async () => { - await modifyTree(tree); - await tree.commit(); - const snapshot = await snapshotBuilder.snapshot(BlockNumber(1)); - - const expectedPaths = await Promise.all(leaves.map(leaf => tree.getSiblingPath(leaf, false))); - - await modifyTree(tree); - await tree.commit(); - - const historicPaths = await Promise.all(leaves.map(leaf => snapshot.getSiblingPath(leaf))); - - for (const [index, path] of historicPaths.entries()) { - expect(path).toEqual(expectedPaths[index]); - } - }); - - it('retains old snapshots even if new one are created', async () => { - await modifyTree(tree); - await tree.commit(); - - const expectedPaths = await Promise.all(leaves.map(leaf => tree.getSiblingPath(leaf, false))); - - const snapshot = await snapshotBuilder.snapshot(BlockNumber(1)); - - await modifyTree(tree); - await tree.commit(); - - await snapshotBuilder.snapshot(BlockNumber(2)); - - // check that snapshot 2 has not influenced snapshot(1) at all - const historicPaths = await Promise.all(leaves.map(leaf => snapshot.getSiblingPath(leaf))); - - for (const [index, path] of historicPaths.entries()) { - expect(path).toEqual(expectedPaths[index]); - } - }); - - it('retains old snapshots even if new one are created and the tree diverges', async () => { - await modifyTree(tree); - await tree.commit(); - - const expectedPaths = await Promise.all(leaves.map(leaf => tree.getSiblingPath(leaf, false))); - - const snapshot = await snapshotBuilder.snapshot(BlockNumber(1)); - - await modifyTree(tree); - await tree.commit(); - - await snapshotBuilder.snapshot(BlockNumber(2)); - - await modifyTree(tree); - await tree.commit(); - - // check that snapshot 2 has not influenced snapshot(1) at all - // and that the diverging tree does not influence the old snapshot - const historicPaths = await Promise.all(leaves.map(leaf => snapshot.getSiblingPath(leaf))); - - for (const [index, path] of historicPaths.entries()) { - expect(path).toEqual(expectedPaths[index]); - } - }); - }); - - describe('getSnapshot', () => { - it('returns old snapshots', async () => { - await modifyTree(tree); - await tree.commit(); - const expectedPaths = await Promise.all(leaves.map(leaf => tree.getSiblingPath(leaf, false))); - await snapshotBuilder.snapshot(BlockNumber(1)); - - for (let i = 2; i < 5; i++) { - await modifyTree(tree); - await tree.commit(); - await snapshotBuilder.snapshot(BlockNumber(i)); - } - - const firstSnapshot = await snapshotBuilder.getSnapshot(BlockNumber(1)); - const historicPaths = await Promise.all(leaves.map(leaf => firstSnapshot.getSiblingPath(leaf))); - - for (const [index, path] of historicPaths.entries()) { - expect(path).toEqual(expectedPaths[index]); - } - }); - - it('throws if an unknown snapshot is requested', async () => { - await modifyTree(tree); - await tree.commit(); - await snapshotBuilder.snapshot(BlockNumber(1)); - - await expect(snapshotBuilder.getSnapshot(BlockNumber(2))).rejects.toThrow(); - }); - }); - - describe('getRoot', () => { - it('returns the historical root of the tree when the snapshot was taken', async () => { - await modifyTree(tree); - await tree.commit(); - const snapshot = await snapshotBuilder.snapshot(BlockNumber(1)); - const historicalRoot = tree.getRoot(false); - - await modifyTree(tree); - await tree.commit(); - - expect(snapshot.getRoot()).toEqual(historicalRoot); - expect(snapshot.getRoot()).not.toEqual(tree.getRoot(false)); - }); - }); - - describe('getDepth', () => { - it('returns the same depth as the tree', async () => { - await modifyTree(tree); - await tree.commit(); - const snapshot = await snapshotBuilder.snapshot(BlockNumber(1)); - expect(snapshot.getDepth()).toEqual(tree.getDepth()); - }); - }); - - describe('getNumLeaves', () => { - it('returns the historical leaves count when the snapshot was taken', async () => { - await modifyTree(tree); - await tree.commit(); - const snapshot = await snapshotBuilder.snapshot(BlockNumber(1)); - const historicalNumLeaves = tree.getNumLeaves(false); - - await modifyTree(tree); - await tree.commit(); - - expect(snapshot.getNumLeaves()).toEqual(historicalNumLeaves); - }); - }); - - describe('getLeafValue', () => { - it('returns the historical leaf value when the snapshot was taken', async () => { - await modifyTree(tree); - await tree.commit(); - const snapshot = await snapshotBuilder.snapshot(BlockNumber(1)); - const historicalLeafValue = tree.getLeafValue(0n, false); - expect(snapshot.getLeafValue(0n)).toEqual(historicalLeafValue); - - await modifyTree(tree); - await tree.commit(); - - expect(snapshot.getLeafValue(0n)).toEqual(historicalLeafValue); - }); - }); - - describe('findLeafIndex', () => { - it('returns the historical leaf index when the snapshot was taken', async () => { - await modifyTree(tree); - await tree.commit(); - const snapshot = await snapshotBuilder.snapshot(BlockNumber(1)); - - const initialLastLeafIndex = tree.getNumLeaves(false) - 1n; - let lastLeaf = tree.getLeafValue(initialLastLeafIndex, false); - expect(snapshot.findLeafIndex(lastLeaf!)).toBe(initialLastLeafIndex); - - await modifyTree(tree); - await tree.commit(); - - const newLastLeafIndex = tree.getNumLeaves(false) - 1n; - lastLeaf = tree.getLeafValue(newLastLeafIndex, false); - - expect(snapshot.findLeafIndex(lastLeaf!)).toBe(undefined); - }); - }); - }); -} diff --git a/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.test.ts b/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.test.ts deleted file mode 100644 index 1f70bf5bacee..000000000000 --- a/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.test.ts +++ /dev/null @@ -1,187 +0,0 @@ -import { randomBigInt } from '@aztec/foundation/crypto/random'; -import { Fr } from '@aztec/foundation/curves/bn254'; -import { createLogger } from '@aztec/foundation/log'; -import { SiblingPath } from '@aztec/foundation/trees'; -import type { Hasher } from '@aztec/foundation/trees'; -import type { AztecKVStore } from '@aztec/kv-store'; -import { openTmpStore } from '@aztec/kv-store/lmdb'; - -import { INITIAL_LEAF, newTree } from '../index.js'; -import type { UpdateOnlyTree } from '../interfaces/update_only_tree.js'; -import { loadTree } from '../load_tree.js'; -import { Poseidon } from '../poseidon.js'; -import { standardBasedTreeTestSuite } from '../test/standard_based_test_suite.js'; -import { treeTestSuite } from '../test/test_suite.js'; -import { SparseTree } from './sparse_tree.js'; - -const log = createLogger('merkle-tree:test:sparse_tree'); - -const createDb = async ( - db: AztecKVStore, - hasher: Hasher, - name: string, - depth: number, -): Promise> => { - return await newTree( - SparseTree, - db, - hasher, - name, - { - fromBuffer: (buffer: Buffer): Buffer => buffer, - }, - depth, - ); -}; - -const createFromName = async (db: AztecKVStore, hasher: Hasher, name: string): Promise> => { - return await loadTree(SparseTree, db, hasher, name, { - fromBuffer: (buffer: Buffer): Buffer => buffer, - }); -}; - -const TEST_TREE_DEPTH = 3; - -treeTestSuite('SparseTree', createDb, createFromName); -standardBasedTreeTestSuite('SparseTree', createDb); - -describe('SparseTreeSpecific', () => { - let poseidon: Poseidon; - - beforeEach(() => { - poseidon = new Poseidon(); - }); - - it('throws when index is bigger than (2^DEPTH - 1) ', async () => { - const db = openTmpStore(); - const depth = 32; - const tree = await createDb(db, poseidon, 'test', depth); - - const index = 2n ** BigInt(depth); - expect(() => tree.updateLeaf(Buffer.alloc(32), index)).toThrow(); - }); - - it('updating non-empty leaf does not change tree size', async () => { - const depth = 32; - const maxIndex = 2 ** depth - 1; - - const db = openTmpStore(); - const tree = await createDb(db, poseidon, 'test', depth); - - const randomIndex = randomBigInt(BigInt(maxIndex)); - expect(tree.getNumLeaves(false)).toEqual(0n); - - // Insert a leaf - await tree.updateLeaf(Fr.random().toBuffer(), randomIndex); - expect(tree.getNumLeaves(true)).toEqual(1n); - - // Update a leaf - await tree.updateLeaf(Fr.random().toBuffer(), randomIndex); - expect(tree.getNumLeaves(true)).toEqual(1n); - }); - - it('deleting leaf decrements tree size', async () => { - const depth = 254; - const maxIndex = 2 ** depth - 1; - - const db = openTmpStore(); - const tree = await createDb(db, poseidon, 'test', depth); - - const randomIndex = randomBigInt(BigInt(maxIndex)); - expect(tree.getNumLeaves(false)).toEqual(0n); - - // Insert a leaf - await tree.updateLeaf(Fr.random().toBuffer(), randomIndex); - expect(tree.getNumLeaves(true)).toEqual(1n); - - // Delete a leaf - await tree.updateLeaf(INITIAL_LEAF, randomIndex); - expect(tree.getNumLeaves(true)).toEqual(0n); - }); - - it('should have correct root and sibling path after in a "non-append-only" way', async () => { - const db = openTmpStore(); - const tree = await createDb(db, poseidon, 'test', 3); - - const level2ZeroHash = poseidon.hash(INITIAL_LEAF, INITIAL_LEAF); - const level1ZeroHash = poseidon.hash(level2ZeroHash, level2ZeroHash); - - expect(tree.getNumLeaves(false)).toEqual(0n); - expect(tree.getRoot(false)).toEqual(poseidon.hash(level1ZeroHash, level1ZeroHash)); - - // Insert leaf at index 3 - let level1LeftHash: Buffer; - const leafAtIndex3 = Fr.random().toBuffer(); - { - await tree.updateLeaf(leafAtIndex3, 3n); - expect(tree.getNumLeaves(true)).toEqual(1n); - const level2Hash = poseidon.hash(INITIAL_LEAF, leafAtIndex3); - level1LeftHash = poseidon.hash(level2ZeroHash, level2Hash); - const root = poseidon.hash(level1LeftHash, level1ZeroHash); - expect(tree.getRoot(true)).toEqual(root); - expect(await tree.getSiblingPath(3n, true)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, level2ZeroHash, level1ZeroHash]), - ); - } - - // Insert leaf at index 6 - let level1RightHash: Buffer; - { - const leafAtIndex6 = Fr.random().toBuffer(); - await tree.updateLeaf(leafAtIndex6, 6n); - expect(tree.getNumLeaves(true)).toEqual(2n); - const level2Hash = poseidon.hash(leafAtIndex6, INITIAL_LEAF); - level1RightHash = poseidon.hash(level2ZeroHash, level2Hash); - const root = poseidon.hash(level1LeftHash, level1RightHash); - expect(tree.getRoot(true)).toEqual(root); - expect(await tree.getSiblingPath(6n, true)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, level2ZeroHash, level1LeftHash]), - ); - } - - // Insert leaf at index 2 - const leafAtIndex2 = Fr.random().toBuffer(); - { - await tree.updateLeaf(leafAtIndex2, 2n); - expect(tree.getNumLeaves(true)).toEqual(3n); - const level2Hash = poseidon.hash(leafAtIndex2, leafAtIndex3); - level1LeftHash = poseidon.hash(level2ZeroHash, level2Hash); - const root = poseidon.hash(level1LeftHash, level1RightHash); - expect(tree.getRoot(true)).toEqual(root); - expect(await tree.getSiblingPath(2n, true)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [leafAtIndex3, level2ZeroHash, level1RightHash]), - ); - } - - // Updating leaf at index 3 - { - const updatedLeafAtIndex3 = Fr.random().toBuffer(); - await tree.updateLeaf(updatedLeafAtIndex3, 3n); - expect(tree.getNumLeaves(true)).toEqual(3n); - const level2Hash = poseidon.hash(leafAtIndex2, updatedLeafAtIndex3); - level1LeftHash = poseidon.hash(level2ZeroHash, level2Hash); - const root = poseidon.hash(level1LeftHash, level1RightHash); - expect(tree.getRoot(true)).toEqual(root); - expect(await tree.getSiblingPath(3n, true)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [leafAtIndex2, level2ZeroHash, level1RightHash]), - ); - } - }); - - // This one is a performance measurement and is enabled only to check regression in performance. - it.skip('measures time of inserting 1000 leaves at random positions for depth 254', async () => { - const depth = 254; - const maxIndex = 2 ** depth - 1; - - const db = openTmpStore(); - const tree = await createDb(db, poseidon, 'test', depth); - - const leaves = Array.from({ length: 1000 }).map(() => Fr.random().toBuffer()); - const indices = Array.from({ length: 1000 }).map(() => randomBigInt(BigInt(maxIndex))); - - const start = Date.now(); - await Promise.all(leaves.map((leaf, i) => tree.updateLeaf(leaf, indices[i]))); - const end = Date.now(); - log.info(`Inserting 1000 leaves at random positions for depth 254 took ${end - start}ms`); - }, 300_000); -}); diff --git a/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.ts b/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.ts deleted file mode 100644 index bcfb8265b1a0..000000000000 --- a/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.ts +++ /dev/null @@ -1,57 +0,0 @@ -import type { BlockNumber } from '@aztec/foundation/branded-types'; -import { type Bufferable, serializeToBuffer } from '@aztec/foundation/serialize'; - -import type { UpdateOnlyTree } from '../interfaces/update_only_tree.js'; -import { FullTreeSnapshotBuilder } from '../snapshots/full_snapshot.js'; -import type { TreeSnapshot } from '../snapshots/snapshot_builder.js'; -import { INITIAL_LEAF, TreeBase } from '../tree_base.js'; - -/** - * A Merkle tree implementation that uses a LevelDB database to store the tree. - */ -export class SparseTree extends TreeBase implements UpdateOnlyTree { - #snapshotBuilder = new FullTreeSnapshotBuilder(this.store, this, this.deserializer); - /** - * Updates a leaf in the tree. - * @param leaf - New contents of the leaf. - * @param index - Index of the leaf to be updated. - */ - public updateLeaf(value: T, index: bigint): Promise { - if (index > this.maxIndex) { - throw Error(`Index out of bounds. Index ${index}, max index: ${this.maxIndex}.`); - } - - const leaf = serializeToBuffer(value); - const insertingZeroElement = leaf.equals(INITIAL_LEAF); - const originallyZeroElement = this.getLeafBuffer(index, true)?.equals(INITIAL_LEAF); - if (insertingZeroElement && originallyZeroElement) { - return Promise.resolve(); - } - this.addLeafToCacheAndHashToRoot(leaf, index); - if (insertingZeroElement) { - // Deleting element (originally non-zero and new value is zero) - this.cachedSize = (this.cachedSize ?? this.size) - 1n; - } else if (originallyZeroElement) { - // Inserting new element (originally zero and new value is non-zero) - this.cachedSize = (this.cachedSize ?? this.size) + 1n; - } - - return Promise.resolve(); - } - - public snapshot(block: BlockNumber): Promise> { - return this.#snapshotBuilder.snapshot(block); - } - - public getSnapshot(block: BlockNumber): Promise> { - return this.#snapshotBuilder.getSnapshot(block); - } - - public findLeafIndex(_value: T, _includeUncommitted: boolean): bigint | undefined { - throw new Error('Finding leaf index is not supported for sparse trees'); - } - - public findLeafIndexAfter(_value: T, _startIndex: bigint, _includeUncommitted: boolean): bigint | undefined { - throw new Error('Finding leaf index is not supported for sparse trees'); - } -} diff --git a/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts b/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts deleted file mode 100644 index 7da62012052b..000000000000 --- a/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts +++ /dev/null @@ -1,641 +0,0 @@ -import { toBufferBE } from '@aztec/foundation/bigint-buffer'; -import type { BlockNumber } from '@aztec/foundation/branded-types'; -import type { FromBuffer } from '@aztec/foundation/serialize'; -import { Timer } from '@aztec/foundation/timer'; -import { SiblingPath } from '@aztec/foundation/trees'; -import type { Hasher, IndexedTreeLeaf, IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; -import type { AztecKVStore, AztecMap } from '@aztec/kv-store'; -import type { TreeInsertionStats } from '@aztec/stdlib/stats'; -import type { BatchInsertionResult, LeafUpdateWitnessData } from '@aztec/stdlib/trees'; - -import type { IndexedTree, PreimageFactory } from '../interfaces/indexed_tree.js'; -import { IndexedTreeSnapshotBuilder } from '../snapshots/indexed_tree_snapshot.js'; -import type { IndexedTreeSnapshot } from '../snapshots/snapshot_builder.js'; -import { TreeBase } from '../tree_base.js'; - -export const buildDbKeyForPreimage = (name: string, index: bigint) => { - return `${name}:leaf_by_index:${toBufferBE(index, 32).toString('hex')}` as const; -}; - -export const buildDbKeyForLeafIndex = (name: string, key: bigint) => { - return `${name}:leaf_index_by_leaf_key:${toBufferBE(key, 32).toString('hex')}` as const; -}; - -/** - * Factory for creating leaves. - */ -export interface LeafFactory { - /** - * Creates a new leaf from a buffer. - * @param key - Key of the leaf. - */ - buildDummy(key: bigint): IndexedTreeLeaf; - /** - * Creates a new leaf from a buffer. - * @param buffer - Buffer to create a leaf from. - */ - fromBuffer(buffer: Buffer): IndexedTreeLeaf; -} - -/** - * Pre-compute empty witness. - * @param treeHeight - Height of tree for sibling path. - * @returns An empty witness. - */ -function getEmptyLowLeafWitness( - treeHeight: N, - leafPreimageFactory: PreimageFactory, -): LeafUpdateWitnessData { - return { - leafPreimage: leafPreimageFactory.empty(), - index: 0n, - siblingPath: new SiblingPath(treeHeight, Array(treeHeight).fill(toBufferBE(0n, 32))), - }; -} - -export const noopDeserializer: FromBuffer = { - fromBuffer: (buf: Buffer) => buf, -}; - -/** - * Standard implementation of an indexed tree. - */ -export class StandardIndexedTree extends TreeBase implements IndexedTree { - #snapshotBuilder: IndexedTreeSnapshotBuilder; - - protected cachedLeafPreimages: { [key: string]: IndexedTreeLeafPreimage } = {}; - protected leaves: AztecMap, Buffer>; - protected leafIndex: AztecMap, bigint>; - - public constructor( - store: AztecKVStore, - hasher: Hasher, - name: string, - depth: number, - size: bigint = 0n, - protected leafPreimageFactory: PreimageFactory, - protected leafFactory: LeafFactory, - root?: Buffer, - ) { - super(store, hasher, name, depth, size, noopDeserializer, root); - this.leaves = store.openMap(`tree_${name}_leaves`); - this.leafIndex = store.openMap(`tree_${name}_leaf_index`); - this.#snapshotBuilder = new IndexedTreeSnapshotBuilder(this.store, this, this.leafPreimageFactory); - } - - /** - * Appends the given leaves to the tree. - * @param _leaves - The leaves to append. - * @returns Empty promise. - * @remarks Use batchInsert method instead. - */ - override appendLeaves(_leaves: Buffer[]): void { - throw new Error('Not implemented'); - } - - /** - * Commits the changes to the database. - * @returns Empty promise. - */ - public override async commit(): Promise { - await super.commit(); - await this.commitLeaves(); - } - - /** - * Rolls back the not-yet-committed changes. - * @returns Empty promise. - */ - public override async rollback(): Promise { - await super.rollback(); - this.clearCachedLeaves(); - } - - /** - * Gets the value of the leaf at the given index. - * @param index - Index of the leaf of which to obtain the value. - * @param includeUncommitted - Indicates whether to include uncommitted leaves in the computation. - * @returns The value of the leaf at the given index or undefined if the leaf is empty. - */ - public override getLeafValue(index: bigint, includeUncommitted: boolean): Buffer | undefined { - const preimage = this.getLatestLeafPreimageCopy(index, includeUncommitted); - return preimage && preimage.toBuffer(); - } - - /** - * Finds the index of the largest leaf whose value is less than or equal to the provided value. - * @param newKey - The new key to be inserted into the tree. - * @param includeUncommitted - If true, the uncommitted changes are included in the search. - * @returns The found leaf index and a flag indicating if the corresponding leaf's value is equal to `newValue`. - */ - findIndexOfPreviousKey( - newKey: bigint, - includeUncommitted: boolean, - ): - | { - /** - * The index of the found leaf. - */ - index: bigint; - /** - * A flag indicating if the corresponding leaf's value is equal to `newValue`. - */ - alreadyPresent: boolean; - } - | undefined { - let lowLeafIndex = this.getDbLowLeafIndex(newKey); - let lowLeafPreimage = lowLeafIndex !== undefined ? this.getDbPreimage(lowLeafIndex) : undefined; - - if (includeUncommitted) { - const cachedLowLeafIndex = this.getCachedLowLeafIndex(newKey); - if (cachedLowLeafIndex !== undefined) { - const cachedLowLeafPreimage = this.getCachedPreimage(cachedLowLeafIndex)!; - if (!lowLeafPreimage || cachedLowLeafPreimage.getKey() > lowLeafPreimage.getKey()) { - lowLeafIndex = cachedLowLeafIndex; - lowLeafPreimage = cachedLowLeafPreimage; - } - } - } - - if (lowLeafIndex === undefined || !lowLeafPreimage) { - return undefined; - } - - return { - index: lowLeafIndex, - alreadyPresent: lowLeafPreimage.getKey() === newKey, - }; - } - - private getCachedLowLeafIndex(key: bigint): bigint | undefined { - const indexes = Object.getOwnPropertyNames(this.cachedLeafPreimages); - const lowLeafIndexes = indexes - .map(index => ({ - index: BigInt(index), - key: this.cachedLeafPreimages[index].getKey(), - })) - .filter(({ key: candidateKey }) => candidateKey <= key) - .sort((a, b) => Number(b.key - a.key)); - return lowLeafIndexes[0]?.index; - } - - private getCachedLeafIndex(key: bigint): bigint | undefined { - const index = Object.keys(this.cachedLeafPreimages).find(index => { - return this.cachedLeafPreimages[index].getKey() === key; - }); - if (index) { - return BigInt(index); - } - return undefined; - } - - private getDbLowLeafIndex(key: bigint): bigint | undefined { - const values = Array.from( - this.leafIndex.values({ - end: buildDbKeyForLeafIndex(this.getName(), key), - limit: 1, - reverse: true, - }), - ); - - return values[0]; - } - - private getDbPreimage(index: bigint): IndexedTreeLeafPreimage | undefined { - const value = this.leaves.get(buildDbKeyForPreimage(this.getName(), index)); - return value ? this.leafPreimageFactory.fromBuffer(value) : undefined; - } - - private getCachedPreimage(index: bigint): IndexedTreeLeafPreimage | undefined { - return this.cachedLeafPreimages[index.toString()]; - } - - /** - * Gets the latest LeafPreimage copy. - * @param index - Index of the leaf of which to obtain the LeafPreimage copy. - * @param includeUncommitted - If true, the uncommitted changes are included in the search. - * @returns A copy of the leaf preimage at the given index or undefined if the leaf was not found. - */ - public getLatestLeafPreimageCopy(index: bigint, includeUncommitted: boolean): IndexedTreeLeafPreimage | undefined { - const preimage = !includeUncommitted - ? this.getDbPreimage(index) - : (this.getCachedPreimage(index) ?? this.getDbPreimage(index)); - return preimage && this.leafPreimageFactory.clone(preimage); - } - - /** - * Returns the index of a leaf given its value, or undefined if no leaf with that value is found. - * @param value - The leaf value to look for. - * @param includeUncommitted - Indicates whether to include uncommitted data. - * @returns The index of the first leaf found with a given value (undefined if not found). - */ - public findLeafIndex(value: Buffer, includeUncommitted: boolean): bigint | undefined { - const leaf = this.leafFactory.fromBuffer(value); - let index = this.leafIndex.get(buildDbKeyForLeafIndex(this.getName(), leaf.getKey())); - - if (includeUncommitted && index === undefined) { - const cachedIndex = this.getCachedLeafIndex(leaf.getKey()); - index = cachedIndex; - } - - return index; - } - - public findLeafIndexAfter(_leaf: Buffer, _startIndex: bigint, _includeUncommitted: boolean): bigint | undefined { - throw new Error('Method not implemented for indexed trees'); - } - - /** - * Initializes the tree. - * @param prefilledSize - A number of leaves that are prefilled with values. - * @returns Empty promise. - * - * @remarks Explanation of pre-filling: - * There needs to be an initial (0,0,0) leaf in the tree, so that when we insert the first 'proper' leaf, we can - * prove that any value greater than 0 doesn't exist in the tree yet. We prefill/pad the tree with "the number of - * leaves that are added by one block" so that the first 'proper' block can insert a full subtree. - * - * Without this padding, there would be a leaf (0,0,0) at leaf index 0, making it really difficult to insert e.g. - * 1024 leaves for the first block, because there's only neat space for 1023 leaves after 0. By padding with 1023 - * more leaves, we can then insert the first block of 1024 leaves into indices 1024:2047. - */ - public override async init(prefilledSize: number): Promise { - if (prefilledSize < 1) { - throw new Error(`Prefilled size must be at least 1!`); - } - - const leaves: IndexedTreeLeafPreimage[] = []; - for (let i = 0n; i < prefilledSize; i++) { - const newLeaf = this.leafFactory.buildDummy(i); - const newLeafPreimage = this.leafPreimageFactory.fromLeaf(newLeaf, i + 1n, i + 1n); - leaves.push(newLeafPreimage); - } - - // Make the last leaf point to the first leaf - leaves[prefilledSize - 1] = this.leafPreimageFactory.fromLeaf(leaves[prefilledSize - 1].asLeaf(), 0n, 0n); - - this.encodeAndAppendLeaves(leaves, true); - await this.commit(); - } - - /** - * Commits all the leaves to the database and removes them from a cache. - */ - private commitLeaves(): Promise { - return this.store.transaction(() => { - const keys = Object.getOwnPropertyNames(this.cachedLeafPreimages); - for (const key of keys) { - const leaf = this.cachedLeafPreimages[key]; - const index = BigInt(key); - void this.leaves.set(buildDbKeyForPreimage(this.getName(), index), leaf.toBuffer()); - void this.leafIndex.set(buildDbKeyForLeafIndex(this.getName(), leaf.getKey()), index); - } - this.clearCachedLeaves(); - }); - } - - /** - * Clears the cache. - */ - private clearCachedLeaves() { - this.cachedLeafPreimages = {}; - } - - /** - * Updates a leaf in the tree. - * @param preimage - New contents of the leaf. - * @param index - Index of the leaf to be updated. - */ - protected updateLeaf(preimage: IndexedTreeLeafPreimage, index: bigint) { - if (index > this.maxIndex) { - throw Error(`Index out of bounds. Index ${index}, max index: ${this.maxIndex}.`); - } - - this.cachedLeafPreimages[index.toString()] = preimage; - const encodedLeaf = this.encodeLeaf(preimage, true); - this.addLeafToCacheAndHashToRoot(encodedLeaf, index); - const numLeaves = this.getNumLeaves(true); - if (index >= numLeaves) { - this.cachedSize = index + 1n; - } - } - - /* The following doc block messes up with complete-sentence, so we just disable it */ - - /** - * - * Each base rollup needs to provide non membership / inclusion proofs for each of the nullifier. - * This method will return membership proofs and perform partial node updates that will - * allow the circuit to incrementally update the tree and perform a batch insertion. - * - * This offers massive circuit performance savings over doing incremental insertions. - * - * WARNING: This function has side effects, it will insert values into the tree. - * - * Assumptions: - * 1. There are 8 nullifiers provided and they are either unique or empty. (denoted as 0) - * 2. If kc 0 has 1 nullifier, and kc 1 has 3 nullifiers the layout will assume to be the sparse - * nullifier layout: [kc0-0, 0, 0, 0, kc1-0, kc1-1, kc1-2, 0] - * - * Algorithm overview - * - * In general, if we want to batch insert items, we first need to update their low nullifier to point to them, - * then batch insert all of the values at once in the final step. - * To update a low nullifier, we provide an insertion proof that the low nullifier currently exists to the - * circuit, then update the low nullifier. - * Updating this low nullifier will in turn change the root of the tree. Therefore future low nullifier insertion proofs - * must be given against this new root. - * As a result, each low nullifier membership proof will be provided against an intermediate tree state, each with differing - * roots. - * - * This become tricky when two items that are being batch inserted need to update the same low nullifier, or need to use - * a value that is part of the same batch insertion as their low nullifier. What we do to avoid this case is to - * update the existing leaves in the tree with the nullifiers in high to low order, ensuring that this case never occurs. - * The circuit has to sort the nullifiers (or take a hint of the sorted nullifiers and prove that it's a valid permutation). - * Then we just batch insert the new nullifiers in the original order. - * - * The following example will illustrate attempting to insert 2,3,20,19 into a tree already containing 0,5,10,15 - * - * The example will explore two cases. In each case the values low nullifier will exist within the batch insertion, - * One where the low nullifier comes before the item in the set (2,3), and one where it comes after (20,19). - * - * First, we sort the nullifiers high to low, that's 20,19,3,2 - * - * The original tree: Pending insertion subtree - * - * index 0 1 2 3 - - - - - * ------------------------------------- ---------------------------- - * val 0 5 10 15 - - - - - * nextIdx 1 2 3 0 - - - - - * nextVal 5 10 15 0 - - - - - * - * - * Inserting 20: - * 1. Find the low nullifier (3) - provide inclusion proof - * 2. Update its pointers - * 3. Insert 20 into the pending subtree - * - * index 0 1 2 3 - - 6 - - * ------------------------------------- ---------------------------- - * val 0 5 10 15 - - 20 - - * nextIdx 1 2 3 6 - - 0 - - * nextVal 5 10 15 20 - - 0 - - * - * Inserting 19: - * 1. Find the low nullifier (3) - provide inclusion proof - * 2. Update its pointers - * 3. Insert 19 into the pending subtree - * - * index 0 1 2 3 - - 6 7 - * ------------------------------------- ---------------------------- - * val 0 5 10 15 - - 20 19 - * nextIdx 1 2 3 7 - - 0 6 - * nextVal 5 10 15 19 - - 0 20 - * - * Inserting 3: - * 1. Find the low nullifier (0) - provide inclusion proof - * 2. Update its pointers - * 3. Insert 3 into the pending subtree - * - * index 0 1 2 3 - 5 6 7 - * ------------------------------------- ---------------------------- - * val 0 5 10 15 - 3 20 19 - * nextIdx 5 2 3 7 - 1 0 6 - * nextVal 3 10 15 19 - 5 0 20 - * - * Inserting 2: - * 1. Find the low nullifier (0) - provide inclusion proof - * 2. Update its pointers - * 3. Insert 2 into the pending subtree - * - * index 0 1 2 3 4 5 6 7 - * ------------------------------------- ---------------------------- - * val 0 5 10 15 2 3 20 19 - * nextIdx 4 2 3 7 5 1 0 6 - * nextVal 2 10 15 19 3 5 0 20 - * - * Perform subtree insertion - * - * index 0 1 2 3 4 5 6 7 - * --------------------------------------------------------------------- - * val 0 5 10 15 2 3 20 19 - * nextIdx 4 2 3 7 5 1 0 6 - * nextVal 2 10 15 19 3 5 0 20 - * - * For leaves that allow updating the process is exactly the same. When a leaf is inserted that is already present, - * the low leaf will be the leaf that is being updated, and it'll get updated and an empty leaf will be inserted instead. - * For example: - * - * Initial state: - * - * index 0 1 2 3 4 5 6 7 - * --------------------------------------------------------------------- - * slot 0 0 0 0 0 0 0 0 - * value 0 0 0 0 0 0 0 0 - * nextIdx 0 0 0 0 0 0 0 0 - * nextSlot 0 0 0 0 0 0 0 0. - * - * - * Add new value 30:5: - * - * index 0 1 2 3 4 5 6 7 - * --------------------------------------------------------------------- - * slot 0 30 0 0 0 0 0 0 - * value 0 5 0 0 0 0 0 0 - * nextIdx 1 0 0 0 0 0 0 0 - * nextSlot 30 0 0 0 0 0 0 0. - * - * - * Update the value of 30 to 10 (insert 30:10): - * - * index 0 1 2 3 4 5 6 7 - * --------------------------------------------------------------------- - * slot 0 30 0 0 0 0 0 0 - * value 0 10 0 0 0 0 0 0 - * nextIdx 1 0 0 0 0 0 0 0 - * nextSlot 30 0 0 0 0 0 0 0. - * - * The low leaf is 30, so we update it to 10, and insert an empty leaf at index 2. - * - * @param leaves - Values to insert into the tree. - * @param subtreeHeight - Height of the subtree. - * @returns The data for the leaves to be updated when inserting the new ones. - */ - public async batchInsert< - TreeHeight extends number, - SubtreeHeight extends number, - SubtreeSiblingPathHeight extends number, - >( - leaves: Buffer[], - subtreeHeight: SubtreeHeight, - ): Promise> { - this.hasher.reset(); - const timer = new Timer(); - const insertedKeys = new Map(); - const emptyLowLeafWitness = getEmptyLowLeafWitness(this.getDepth() as TreeHeight, this.leafPreimageFactory); - // Accumulators - const lowLeavesWitnesses: LeafUpdateWitnessData[] = leaves.map(() => emptyLowLeafWitness); - const pendingInsertionSubtree: IndexedTreeLeafPreimage[] = leaves.map(() => this.leafPreimageFactory.empty()); - - // Start info - const startInsertionIndex = this.getNumLeaves(true); - - const leavesToInsert = leaves.map(leaf => this.leafFactory.fromBuffer(leaf)); - const sortedDescendingLeafTuples = leavesToInsert - .map((leaf, index) => ({ leaf, index })) - .sort((a, b) => Number(b.leaf.getKey() - a.leaf.getKey())); - const sortedDescendingLeaves = sortedDescendingLeafTuples.map(leafTuple => leafTuple.leaf); - - // Get insertion path for each leaf - for (let i = 0; i < leavesToInsert.length; i++) { - const newLeaf = sortedDescendingLeaves[i]; - const originalIndex = leavesToInsert.indexOf(newLeaf); - - if (newLeaf.isEmpty()) { - continue; - } - - if (insertedKeys.has(newLeaf.getKey())) { - throw new Error('Cannot insert duplicated keys in the same batch'); - } else { - insertedKeys.set(newLeaf.getKey(), true); - } - - const indexOfPrevious = this.findIndexOfPreviousKey(newLeaf.getKey(), true); - if (indexOfPrevious === undefined) { - return { - lowLeavesWitnessData: undefined, - sortedNewLeaves: sortedDescendingLeafTuples.map(leafTuple => leafTuple.leaf.toBuffer()), - sortedNewLeavesIndexes: sortedDescendingLeafTuples.map(leafTuple => leafTuple.index), - newSubtreeSiblingPath: await this.getSubtreeSiblingPath(subtreeHeight, true), - }; - } - - const isUpdate = indexOfPrevious.alreadyPresent; - - // get the low leaf (existence checked in getting index) - const lowLeafPreimage = this.getLatestLeafPreimageCopy(indexOfPrevious.index, true)!; - const siblingPath = await this.getSiblingPath(BigInt(indexOfPrevious.index), true); - - const witness: LeafUpdateWitnessData = { - leafPreimage: lowLeafPreimage, - index: BigInt(indexOfPrevious.index), - siblingPath, - }; - - // Update the running paths - lowLeavesWitnesses[i] = witness; - - if (isUpdate) { - const newLowLeaf = lowLeafPreimage.asLeaf().updateTo(newLeaf); - - const newLowLeafPreimage = this.leafPreimageFactory.fromLeaf( - newLowLeaf, - lowLeafPreimage.getNextKey(), - lowLeafPreimage.getNextIndex(), - ); - - this.updateLeaf(newLowLeafPreimage, indexOfPrevious.index); - - pendingInsertionSubtree[originalIndex] = this.leafPreimageFactory.empty(); - } else { - const newLowLeafPreimage = this.leafPreimageFactory.fromLeaf( - lowLeafPreimage.asLeaf(), - newLeaf.getKey(), - startInsertionIndex + BigInt(originalIndex), - ); - - this.updateLeaf(newLowLeafPreimage, indexOfPrevious.index); - - const currentPendingPreimageLeaf = this.leafPreimageFactory.fromLeaf( - newLeaf, - lowLeafPreimage.getNextKey(), - lowLeafPreimage.getNextIndex(), - ); - - pendingInsertionSubtree[originalIndex] = currentPendingPreimageLeaf; - } - } - - const newSubtreeSiblingPath = await this.getSubtreeSiblingPath( - subtreeHeight, - true, - ); - - // Perform batch insertion of new pending values - // Note: In this case we set `hash0Leaf` param to false because batch insertion algorithm use forced null leaf - // inclusion. See {@link encodeLeaf} for a more through param explanation. - this.encodeAndAppendLeaves(pendingInsertionSubtree, false); - - this.log.debug(`Inserted ${leaves.length} leaves into ${this.getName()} tree`, { - eventName: 'tree-insertion', - duration: timer.ms(), - batchSize: leaves.length, - treeName: this.getName(), - treeDepth: this.getDepth(), - treeType: 'indexed', - ...this.hasher.stats(), - } satisfies TreeInsertionStats); - - return { - lowLeavesWitnessData: lowLeavesWitnesses, - sortedNewLeaves: sortedDescendingLeafTuples.map(leafTuple => leafTuple.leaf.toBuffer()), - sortedNewLeavesIndexes: sortedDescendingLeafTuples.map(leafTuple => leafTuple.index), - newSubtreeSiblingPath, - }; - } - - async getSubtreeSiblingPath( - subtreeHeight: SubtreeHeight, - includeUncommitted: boolean, - ): Promise> { - const nextAvailableLeafIndex = this.getNumLeaves(includeUncommitted); - const fullSiblingPath = await this.getSiblingPath(nextAvailableLeafIndex, includeUncommitted); - - // Drop the first subtreeHeight items since we only care about the path to the subtree root - return fullSiblingPath.getSubtreeSiblingPath(subtreeHeight); - } - - snapshot(blockNumber: BlockNumber): Promise { - return this.#snapshotBuilder.snapshot(blockNumber); - } - - getSnapshot(blockNumber: BlockNumber): Promise { - return this.#snapshotBuilder.getSnapshot(blockNumber); - } - - /** - * Encodes leaves and appends them to a tree. - * @param preimages - Leaves to encode. - * @param hash0Leaf - Indicates whether 0 value leaf should be hashed. See {@link encodeLeaf}. - * @returns Empty promise - */ - private encodeAndAppendLeaves(preimages: IndexedTreeLeafPreimage[], hash0Leaf: boolean): void { - const startInsertionIndex = this.getNumLeaves(true); - - const hashedLeaves = preimages.map((preimage, i) => { - this.cachedLeafPreimages[(startInsertionIndex + BigInt(i)).toString()] = preimage; - return this.encodeLeaf(preimage, hash0Leaf); - }); - - super.appendLeaves(hashedLeaves); - } - - /** - * Encode a leaf into a buffer. - * @param leaf - Leaf to encode. - * @param hash0Leaf - Indicates whether 0 value leaf should be hashed. Not hashing 0 value can represent a forced - * null leaf insertion. Detecting this case by checking for 0 value is safe as in the case of - * nullifier it is improbable that a valid nullifier would be 0. - * @returns Leaf encoded in a buffer. - */ - private encodeLeaf(leaf: IndexedTreeLeafPreimage, hash0Leaf: boolean): Buffer { - let encodedLeaf; - if (!hash0Leaf && leaf.getKey() == 0n) { - encodedLeaf = toBufferBE(0n, 32); - } else { - encodedLeaf = this.hasher.hashInputs(leaf.toHashInputs()); - } - return encodedLeaf; - } -} diff --git a/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree.test.ts b/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree.test.ts deleted file mode 100644 index 5cb10fc46bb2..000000000000 --- a/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree.test.ts +++ /dev/null @@ -1,675 +0,0 @@ -import { toBufferBE } from '@aztec/foundation/bigint-buffer'; -import { Fr } from '@aztec/foundation/curves/bn254'; -import type { FromBuffer } from '@aztec/foundation/serialize'; -import { SiblingPath } from '@aztec/foundation/trees'; -import type { Hasher } from '@aztec/foundation/trees'; -import type { AztecKVStore } from '@aztec/kv-store'; -import { openTmpStore } from '@aztec/kv-store/lmdb'; -import { - NullifierLeaf, - NullifierLeafPreimage, - PublicDataTreeLeaf, - PublicDataTreeLeafPreimage, -} from '@aztec/stdlib/trees'; - -import { INITIAL_LEAF, type MerkleTree, Poseidon, loadTree, newTree } from '../../index.js'; -import { treeTestSuite } from '../../test/test_suite.js'; -import { StandardIndexedTreeWithAppend } from './standard_indexed_tree_with_append.js'; - -class NullifierTree extends StandardIndexedTreeWithAppend { - constructor( - store: AztecKVStore, - hasher: Hasher, - name: string, - depth: number, - size: bigint = 0n, - _noop: any, - root?: Buffer, - ) { - super(store, hasher, name, depth, size, NullifierLeafPreimage, NullifierLeaf, root); - } -} - -class PublicDataTree extends StandardIndexedTreeWithAppend { - constructor( - store: AztecKVStore, - hasher: Hasher, - name: string, - depth: number, - size: bigint = 0n, - _noop: any, - root?: Buffer, - ) { - super(store, hasher, name, depth, size, PublicDataTreeLeafPreimage, PublicDataTreeLeaf, root); - } -} - -const noopDeserializer: FromBuffer = { - fromBuffer: (buffer: Buffer) => buffer, -}; - -const createDb = async (store: AztecKVStore, hasher: Hasher, name: string, depth: number, prefilledSize = 1) => { - return await newTree(NullifierTree, store, hasher, name, noopDeserializer, depth, prefilledSize); -}; - -const createFromName = async (store: AztecKVStore, hasher: Hasher, name: string) => { - return await loadTree(NullifierTree, store, hasher, name, noopDeserializer); -}; - -const createNullifierTreeLeafHashInputs = (value: number, nextIndex: number, nextValue: number) => { - return new NullifierLeafPreimage( - new NullifierLeaf(new Fr(value)), - new Fr(nextValue), - BigInt(nextIndex), - ).toHashInputs(); -}; - -const createPublicDataTreeLeaf = (slot: number, value: number) => { - return new PublicDataTreeLeaf(new Fr(slot), new Fr(value)); -}; - -const createPublicDataTreeLeafHashInputs = (slot: number, value: number, nextIndex: number, nextSlot: number) => { - return new PublicDataTreeLeafPreimage( - new PublicDataTreeLeaf(new Fr(slot), new Fr(value)), - new Fr(nextSlot), - BigInt(nextIndex), - ).toHashInputs(); -}; - -const verifyCommittedState = async ( - tree: MerkleTree, - root: Buffer, - siblingPathIndex: bigint, - emptySiblingPath: SiblingPath, -) => { - expect(tree.getRoot(false)).toEqual(root); - expect(tree.getNumLeaves(false)).toEqual(1n); - expect(await tree.getSiblingPath(siblingPathIndex, false)).toEqual(emptySiblingPath); -}; - -const TEST_TREE_DEPTH = 3; - -treeTestSuite('StandardIndexedTree', createDb, createFromName); - -describe('StandardIndexedTreeSpecific', () => { - let poseidon: Poseidon; - - beforeEach(() => { - poseidon = new Poseidon(); - }); - - it('produces the correct roots and sibling paths', async () => { - // Create a depth-3 indexed merkle tree - const db = openTmpStore(); - const tree = await createDb(db, poseidon, 'test', 3); - - /** - * Initial state: - * - * index 0 1 2 3 4 5 6 7 - * --------------------------------------------------------------------- - * val 0 0 0 0 0 0 0 0 - * nextIdx 0 0 0 0 0 0 0 0 - * nextVal 0 0 0 0 0 0 0 0. - */ - - const initialLeafHash = poseidon.hashInputs(createNullifierTreeLeafHashInputs(0, 0, 0)); - const level1ZeroHash = poseidon.hash(INITIAL_LEAF, INITIAL_LEAF); - const level2ZeroHash = poseidon.hash(level1ZeroHash, level1ZeroHash); - - let index0Hash = initialLeafHash; - // Each element is named by the level followed by the index on that level. E.g. e10 -> level 1, index 0, e21 -> level 2, index 1 - let e10 = poseidon.hash(index0Hash, INITIAL_LEAF); - let e20 = poseidon.hash(e10, level1ZeroHash); - - const initialE20 = e20; // Kept for calculating committed state later - const initialE10 = e10; - - let root = poseidon.hash(e20, level2ZeroHash); - const initialRoot = root; - - const emptySiblingPath = new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, level1ZeroHash, level2ZeroHash]); - - expect(tree.getRoot(true)).toEqual(root); - expect(tree.getNumLeaves(true)).toEqual(1n); - expect(await tree.getSiblingPath(0n, true)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, level1ZeroHash, level2ZeroHash]), - ); - - await verifyCommittedState(tree, initialRoot, 0n, emptySiblingPath); - - /** - * Add new value 30: - * - * index 0 1 2 3 4 5 6 7 - * --------------------------------------------------------------------- - * val 0 30 0 0 0 0 0 0 - * nextIdx 1 0 0 0 0 0 0 0 - * nextVal 30 0 0 0 0 0 0 0. - */ - index0Hash = poseidon.hashInputs(createNullifierTreeLeafHashInputs(0, 1, 30)); - let index1Hash = poseidon.hashInputs(createNullifierTreeLeafHashInputs(30, 0, 0)); - e10 = poseidon.hash(index0Hash, index1Hash); - e20 = poseidon.hash(e10, level1ZeroHash); - root = poseidon.hash(e20, level2ZeroHash); - - tree.appendLeaves([toBufferBE(30n, 32)]); - - expect(tree.getRoot(true)).toEqual(root); - expect(tree.getNumLeaves(true)).toEqual(2n); - expect(await tree.getSiblingPath(1n, true)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [index0Hash, level1ZeroHash, level2ZeroHash]), - ); - - // ensure the committed state is correct - const initialSiblingPath = new SiblingPath(TEST_TREE_DEPTH, [initialLeafHash, level1ZeroHash, level2ZeroHash]); - await verifyCommittedState(tree, initialRoot, 1n, initialSiblingPath); - - /** - * Add new value 10: - * - * index 0 1 2 3 4 5 6 7 - * --------------------------------------------------------------------- - * val 0 30 10 0 0 0 0 0 - * nextIdx 2 0 1 0 0 0 0 0 - * nextVal 10 0 30 0 0 0 0 0. - */ - index0Hash = poseidon.hashInputs(createNullifierTreeLeafHashInputs(0, 2, 10)); - let index2Hash = poseidon.hashInputs(createNullifierTreeLeafHashInputs(10, 1, 30)); - e10 = poseidon.hash(index0Hash, index1Hash); - let e11 = poseidon.hash(index2Hash, INITIAL_LEAF); - e20 = poseidon.hash(e10, e11); - root = poseidon.hash(e20, level2ZeroHash); - - tree.appendLeaves([toBufferBE(10n, 32)]); - - expect(tree.getRoot(true)).toEqual(root); - expect(tree.getNumLeaves(true)).toEqual(3n); - expect(await tree.getSiblingPath(2n, true)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, e10, level2ZeroHash]), - ); - - // ensure the committed state is correct - await verifyCommittedState( - tree, - initialRoot, - 2n, - new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, initialE10, level2ZeroHash]), - ); - - /** - * Add new value 20: - * - * index 0 1 2 3 4 5 6 7 - * --------------------------------------------------------------------- - * val 0 30 10 20 0 0 0 0 - * nextIdx 2 0 3 1 0 0 0 0 - * nextVal 10 0 20 30 0 0 0 0. - */ - e10 = poseidon.hash(index0Hash, index1Hash); - index2Hash = poseidon.hashInputs(createNullifierTreeLeafHashInputs(10, 3, 20)); - const index3Hash = poseidon.hashInputs(createNullifierTreeLeafHashInputs(20, 1, 30)); - e11 = poseidon.hash(index2Hash, index3Hash); - e20 = poseidon.hash(e10, e11); - root = poseidon.hash(e20, level2ZeroHash); - - tree.appendLeaves([toBufferBE(20n, 32)]); - - expect(tree.getRoot(true)).toEqual(root); - expect(tree.getNumLeaves(true)).toEqual(4n); - expect(await tree.getSiblingPath(3n, true)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [index2Hash, e10, level2ZeroHash]), - ); - - // ensure the committed state is correct - await verifyCommittedState( - tree, - initialRoot, - 3n, - new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, initialE10, level2ZeroHash]), - ); - - /** - * Add new value 50: - * - * index 0 1 2 3 4 5 6 7 - * --------------------------------------------------------------------- - * val 0 30 10 20 50 0 0 0 - * nextIdx 2 4 3 1 0 0 0 0 - * nextVal 10 50 20 30 0 0 0 0. - */ - index1Hash = poseidon.hashInputs(createNullifierTreeLeafHashInputs(30, 4, 50)); - const index4Hash = poseidon.hashInputs(createNullifierTreeLeafHashInputs(50, 0, 0)); - e10 = poseidon.hash(index0Hash, index1Hash); - e20 = poseidon.hash(e10, e11); - const e12 = poseidon.hash(index4Hash, INITIAL_LEAF); - const e21 = poseidon.hash(e12, level1ZeroHash); - root = poseidon.hash(e20, e21); - - tree.appendLeaves([toBufferBE(50n, 32)]); - - expect(tree.getRoot(true)).toEqual(root); - expect(tree.getNumLeaves(true)).toEqual(5n); - - // ensure the committed state is correct - await verifyCommittedState( - tree, - initialRoot, - 4n, - new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, level1ZeroHash, initialE20]), - ); - - // check all uncommitted hash paths - expect(await tree.getSiblingPath(0n, true)).toEqual(new SiblingPath(TEST_TREE_DEPTH, [index1Hash, e11, e21])); - expect(await tree.getSiblingPath(1n, true)).toEqual(new SiblingPath(TEST_TREE_DEPTH, [index0Hash, e11, e21])); - expect(await tree.getSiblingPath(2n, true)).toEqual(new SiblingPath(TEST_TREE_DEPTH, [index3Hash, e10, e21])); - expect(await tree.getSiblingPath(3n, true)).toEqual(new SiblingPath(TEST_TREE_DEPTH, [index2Hash, e10, e21])); - expect(await tree.getSiblingPath(4n, true)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, level1ZeroHash, e20]), - ); - expect(await tree.getSiblingPath(5n, true)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [index4Hash, level1ZeroHash, e20]), - ); - expect(await tree.getSiblingPath(6n, true)).toEqual(new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, e12, e20])); - expect(await tree.getSiblingPath(7n, true)).toEqual(new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, e12, e20])); - - // check all committed hash paths - expect(await tree.getSiblingPath(0n, false)).toEqual(emptySiblingPath); - expect(await tree.getSiblingPath(1n, false)).toEqual(initialSiblingPath); - expect(await tree.getSiblingPath(2n, false)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, initialE10, level2ZeroHash]), - ); - expect(await tree.getSiblingPath(3n, false)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, initialE10, level2ZeroHash]), - ); - const e2SiblingPath = new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, level1ZeroHash, initialE20]); - expect(await tree.getSiblingPath(4n, false)).toEqual(e2SiblingPath); - expect(await tree.getSiblingPath(5n, false)).toEqual(e2SiblingPath); - expect(await tree.getSiblingPath(6n, false)).toEqual(e2SiblingPath); - expect(await tree.getSiblingPath(7n, false)).toEqual(e2SiblingPath); - - await tree.commit(); - // check all committed hash paths equal uncommitted hash paths - for (let i = 0; i < 8; i++) { - expect(await tree.getSiblingPath(BigInt(i), false)).toEqual(await tree.getSiblingPath(BigInt(i), true)); - } - }); - - it('Can append empty leaves and handle insertions', async () => { - // Create a depth-3 indexed merkle tree - const tree = await createDb(openTmpStore(), poseidon, 'test', 3); - - /** - * Initial state: - * - * index 0 1 2 3 4 5 6 7 - * --------------------------------------------------------------------- - * val 0 0 0 0 0 0 0 0 - * nextIdx 0 0 0 0 0 0 0 0 - * nextVal 0 0 0 0 0 0 0 0. - */ - - const INITIAL_LEAF = toBufferBE(0n, 32); - const initialLeafHash = poseidon.hashInputs(createNullifierTreeLeafHashInputs(0, 0, 0)); - const level1ZeroHash = poseidon.hash(INITIAL_LEAF, INITIAL_LEAF); - const level2ZeroHash = poseidon.hash(level1ZeroHash, level1ZeroHash); - let index0Hash = initialLeafHash; - - let e10 = poseidon.hash(index0Hash, INITIAL_LEAF); - let e20 = poseidon.hash(e10, level1ZeroHash); - - const inite10 = e10; - const inite20 = e20; - - let root = poseidon.hash(e20, level2ZeroHash); - const initialRoot = root; - - const emptySiblingPath = new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, level1ZeroHash, level2ZeroHash]); - const initialSiblingPath = new SiblingPath(TEST_TREE_DEPTH, [initialLeafHash, level1ZeroHash, level2ZeroHash]); - - expect(tree.getRoot(true)).toEqual(root); - expect(tree.getNumLeaves(true)).toEqual(1n); - expect(await tree.getSiblingPath(0n, true)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, level1ZeroHash, level2ZeroHash]), - ); - - await verifyCommittedState(tree, initialRoot, 0n, emptySiblingPath); - - /** - * Add new value 30: - * - * index 0 1 2 3 4 5 6 7 - * --------------------------------------------------------------------- - * val 0 30 0 0 0 0 0 0 - * nextIdx 1 0 0 0 0 0 0 0 - * nextVal 30 0 0 0 0 0 0 0. - */ - index0Hash = poseidon.hashInputs(createNullifierTreeLeafHashInputs(0, 1, 30)); - let index1Hash = poseidon.hashInputs(createNullifierTreeLeafHashInputs(30, 0, 0)); - e10 = poseidon.hash(index0Hash, index1Hash); - e20 = poseidon.hash(e10, level1ZeroHash); - root = poseidon.hash(e20, level2ZeroHash); - - tree.appendLeaves([toBufferBE(30n, 32)]); - - expect(tree.getRoot(true)).toEqual(root); - expect(tree.getNumLeaves(true)).toEqual(2n); - expect(await tree.getSiblingPath(1n, true)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [index0Hash, level1ZeroHash, level2ZeroHash]), - ); - - // ensure the committed state is correct - await verifyCommittedState(tree, initialRoot, 1n, initialSiblingPath); - - /** - * Add new value 10: - * - * index 0 1 2 3 4 5 6 7 - * --------------------------------------------------------------------- - * val 0 30 10 0 0 0 0 0 - * nextIdx 2 0 1 0 0 0 0 0 - * nextVal 10 0 30 0 0 0 0 0. - */ - index0Hash = poseidon.hashInputs(createNullifierTreeLeafHashInputs(0, 2, 10)); - let index2Hash = poseidon.hashInputs(createNullifierTreeLeafHashInputs(10, 1, 30)); - e10 = poseidon.hash(index0Hash, index1Hash); - let e11 = poseidon.hash(index2Hash, INITIAL_LEAF); - e20 = poseidon.hash(e10, e11); - root = poseidon.hash(e20, level2ZeroHash); - - tree.appendLeaves([toBufferBE(10n, 32)]); - - expect(tree.getRoot(true)).toEqual(root); - expect(tree.getNumLeaves(true)).toEqual(3n); - expect(await tree.getSiblingPath(2n, true)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, e10, level2ZeroHash]), - ); - - // ensure the committed state is correct - await verifyCommittedState( - tree, - initialRoot, - 2n, - new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, inite10, level2ZeroHash]), - ); - - /** - * Add new value 20: - * - * index 0 1 2 3 4 5 6 7 - * --------------------------------------------------------------------- - * val 0 30 10 20 0 0 0 0 - * nextIdx 2 0 3 1 0 0 0 0 - * nextVal 10 0 20 30 0 0 0 0. - */ - e10 = poseidon.hash(index0Hash, index1Hash); - index2Hash = poseidon.hashInputs(createNullifierTreeLeafHashInputs(10, 3, 20)); - const index3Hash = poseidon.hashInputs(createNullifierTreeLeafHashInputs(20, 1, 30)); - e11 = poseidon.hash(index2Hash, index3Hash); - e20 = poseidon.hash(e10, e11); - root = poseidon.hash(e20, level2ZeroHash); - - tree.appendLeaves([toBufferBE(20n, 32)]); - - expect(tree.getRoot(true)).toEqual(root); - expect(tree.getNumLeaves(true)).toEqual(4n); - expect(await tree.getSiblingPath(3n, true)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [index2Hash, e10, level2ZeroHash]), - ); - - // ensure the committed state is correct - await verifyCommittedState( - tree, - initialRoot, - 3n, - new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, inite10, level2ZeroHash]), - ); - - // Add 2 empty values - const emptyLeaves = [toBufferBE(0n, 32), toBufferBE(0n, 32)]; - tree.appendLeaves(emptyLeaves); - - // The root should be the same but the size should have increased - expect(tree.getRoot(true)).toEqual(root); - expect(tree.getNumLeaves(true)).toEqual(6n); - - /** - * Add new value 50: - * - * index 0 1 2 3 4 5 6 7 - * -------------------------------------------------------------------- - * val 0 30 10 20 0 0 50 0 - * nextIdx 2 6 3 1 0 0 0 0 - * nextVal 10 50 20 30 0 0 0 0. - */ - index1Hash = poseidon.hashInputs(createNullifierTreeLeafHashInputs(30, 6, 50)); - const index6Hash = poseidon.hashInputs(createNullifierTreeLeafHashInputs(50, 0, 0)); - e10 = poseidon.hash(index0Hash, index1Hash); - e20 = poseidon.hash(e10, e11); - const e13 = poseidon.hash(index6Hash, INITIAL_LEAF); - const e21 = poseidon.hash(level1ZeroHash, e13); - root = poseidon.hash(e20, e21); - - tree.appendLeaves([toBufferBE(50n, 32)]); - - expect(tree.getRoot(true)).toEqual(root); - expect(tree.getNumLeaves(true)).toEqual(7n); - - // ensure the committed state is correct - await verifyCommittedState( - tree, - initialRoot, - 6n, - new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, level1ZeroHash, inite20]), - ); - - // // check all uncommitted hash paths - expect(await tree.getSiblingPath(0n, true)).toEqual(new SiblingPath(TEST_TREE_DEPTH, [index1Hash, e11, e21])); - expect(await tree.getSiblingPath(1n, true)).toEqual(new SiblingPath(TEST_TREE_DEPTH, [index0Hash, e11, e21])); - expect(await tree.getSiblingPath(2n, true)).toEqual(new SiblingPath(TEST_TREE_DEPTH, [index3Hash, e10, e21])); - expect(await tree.getSiblingPath(3n, true)).toEqual(new SiblingPath(TEST_TREE_DEPTH, [index2Hash, e10, e21])); - expect(await tree.getSiblingPath(4n, true)).toEqual(new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, e13, e20])); - expect(await tree.getSiblingPath(5n, true)).toEqual(new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, e13, e20])); - expect(await tree.getSiblingPath(6n, true)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, level1ZeroHash, e20]), - ); - expect(await tree.getSiblingPath(7n, true)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [index6Hash, level1ZeroHash, e20]), - ); - - // check all committed hash paths - expect(await tree.getSiblingPath(0n, false)).toEqual(emptySiblingPath); - expect(await tree.getSiblingPath(1n, false)).toEqual(initialSiblingPath); - expect(await tree.getSiblingPath(2n, false)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, inite10, level2ZeroHash]), - ); - expect(await tree.getSiblingPath(3n, false)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, inite10, level2ZeroHash]), - ); - const e2SiblingPath = new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, level1ZeroHash, inite20]); - expect(await tree.getSiblingPath(4n, false)).toEqual(e2SiblingPath); - expect(await tree.getSiblingPath(5n, false)).toEqual(e2SiblingPath); - expect(await tree.getSiblingPath(6n, false)).toEqual(e2SiblingPath); - expect(await tree.getSiblingPath(7n, false)).toEqual(e2SiblingPath); - - await tree.commit(); - // check all committed hash paths equal uncommitted hash paths - for (let i = 0; i < 8; i++) { - expect(await tree.getSiblingPath(BigInt(i), false)).toEqual(await tree.getSiblingPath(BigInt(i), true)); - } - }); - - // For varying orders of insertions assert the local batch insertion generator creates the correct proofs - it.each([ - // These are arbitrary but it needs to be higher than the constant `INITIAL_NULLIFIER_TREE_SIZE` and `KERNEL_NEW_NULLIFIERS_LENGTH * 2` - [[1003, 1002, 1001, 1000, 0, 0, 0, 0]], - [[1003, 1004, 1005, 1006, 0, 0, 0, 0]], - [[1234, 1098, 0, 0, 99999, 1096, 1054, 0]], - [[1970, 1980, 1040, 0, 99999, 1880, 100001, 9000000]], - ] as const)('performs nullifier tree batch insertion correctly', async nullifiers => { - const leaves = nullifiers.map(i => toBufferBE(BigInt(i), 32)); - - const TREE_HEIGHT = 16; // originally from NULLIFIER_TREE_HEIGHT - const INITIAL_TREE_SIZE = 8; // originally from INITIAL_NULLIFIER_TREE_SIZE - const SUBTREE_HEIGHT = 5; // originally from NULLIFIER_SUBTREE_HEIGHT - - // Create a depth-3 indexed merkle tree - const appendTree = await createDb(openTmpStore(), poseidon, 'test', TREE_HEIGHT, INITIAL_TREE_SIZE); - const insertTree = await createDb(openTmpStore(), poseidon, 'test', TREE_HEIGHT, INITIAL_TREE_SIZE); - - appendTree.appendLeaves(leaves); - await insertTree.batchInsert(leaves, SUBTREE_HEIGHT); - - const expectedRoot = appendTree.getRoot(true); - const actualRoot = insertTree.getRoot(true); - expect(actualRoot).toEqual(expectedRoot); - }); - - it('should be able to find indexes of leaves', async () => { - const db = openTmpStore(); - const tree = await createDb(db, poseidon, 'test', 3); - const values = [Buffer.alloc(32, 1), Buffer.alloc(32, 2)]; - - tree.appendLeaves([values[0]]); - - expect(tree.findLeafIndex(values[0], true)).toBeDefined(); - expect(tree.findLeafIndex(values[0], false)).toBe(undefined); - expect(tree.findLeafIndex(values[1], true)).toBe(undefined); - - await tree.commit(); - - expect(tree.findLeafIndex(values[0], false)).toBeDefined(); - }); - - describe('Updatable leaves', () => { - it('should be able to upsert leaves', async () => { - // Create a depth-3 indexed merkle tree - const db = openTmpStore(); - const tree = await newTree(PublicDataTree, db, poseidon, 'test', {}, 3, 1); - - /** - * Initial state: - * - * index 0 1 2 3 4 5 6 7 - * --------------------------------------------------------------------- - * slot 0 0 0 0 0 0 0 0 - * value 0 0 0 0 0 0 0 0 - * nextIdx 0 0 0 0 0 0 0 0 - * nextSlot 0 0 0 0 0 0 0 0. - */ - - const EMPTY_LEAF = toBufferBE(0n, 32); - const initialLeafHash = poseidon.hashInputs(createPublicDataTreeLeafHashInputs(0, 0, 0, 0)); - const level1ZeroHash = poseidon.hash(EMPTY_LEAF, EMPTY_LEAF); - const level2ZeroHash = poseidon.hash(level1ZeroHash, level1ZeroHash); - let index0Hash = initialLeafHash; - - let e10 = poseidon.hash(index0Hash, EMPTY_LEAF); - let e20 = poseidon.hash(e10, level1ZeroHash); - - const inite10 = e10; - - let root = poseidon.hash(e20, level2ZeroHash); - const initialRoot = root; - - const emptySiblingPath = new SiblingPath(TEST_TREE_DEPTH, [EMPTY_LEAF, level1ZeroHash, level2ZeroHash]); - const initialSiblingPath = new SiblingPath(TEST_TREE_DEPTH, [initialLeafHash, level1ZeroHash, level2ZeroHash]); - - expect(tree.getRoot(true)).toEqual(root); - expect(tree.getNumLeaves(true)).toEqual(1n); - expect(await tree.getSiblingPath(0n, true)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [EMPTY_LEAF, level1ZeroHash, level2ZeroHash]), - ); - - await verifyCommittedState(tree, initialRoot, 0n, emptySiblingPath); - - /** - * Add new value 30:5: - * - * index 0 1 2 3 4 5 6 7 - * --------------------------------------------------------------------- - * slot 0 30 0 0 0 0 0 0 - * value 0 5 0 0 0 0 0 0 - * nextIdx 1 0 0 0 0 0 0 0 - * nextSlot 30 0 0 0 0 0 0 0. - */ - index0Hash = poseidon.hashInputs(createPublicDataTreeLeafHashInputs(0, 0, 1, 30)); - let index1Hash = poseidon.hashInputs(createPublicDataTreeLeafHashInputs(30, 5, 0, 0)); - e10 = poseidon.hash(index0Hash, index1Hash); - e20 = poseidon.hash(e10, level1ZeroHash); - root = poseidon.hash(e20, level2ZeroHash); - - tree.appendLeaves([createPublicDataTreeLeaf(30, 5).toBuffer()]); - - expect(tree.getRoot(true)).toEqual(root); - expect(tree.getNumLeaves(true)).toEqual(2n); - expect(await tree.getSiblingPath(1n, true)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [index0Hash, level1ZeroHash, level2ZeroHash]), - ); - - // ensure the committed state is correct - await verifyCommittedState(tree, initialRoot, 1n, initialSiblingPath); - - /** - * Update the value of 30 to 10: - * - * index 0 1 2 3 4 5 6 7 - * --------------------------------------------------------------------- - * slot 0 30 0 0 0 0 0 0 - * value 0 10 0 0 0 0 0 0 - * nextIdx 1 0 0 0 0 0 0 0 - * nextSlot 30 0 0 0 0 0 0 0. - */ - index1Hash = poseidon.hashInputs(createPublicDataTreeLeafHashInputs(30, 10, 0, 0)); - e10 = poseidon.hash(index0Hash, index1Hash); - e20 = poseidon.hash(e10, level1ZeroHash); - root = poseidon.hash(e20, level2ZeroHash); - - tree.appendLeaves([createPublicDataTreeLeaf(30, 10).toBuffer()]); - - expect(tree.getRoot(true)).toEqual(root); - expect(tree.getNumLeaves(true)).toEqual(3n); - expect(await tree.getSiblingPath(2n, true)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [EMPTY_LEAF, e10, level2ZeroHash]), - ); - - // ensure the committed state is correct - await verifyCommittedState( - tree, - initialRoot, - 2n, - new SiblingPath(TEST_TREE_DEPTH, [EMPTY_LEAF, inite10, level2ZeroHash]), - ); - }); - - it.each([ - [[createPublicDataTreeLeaf(1, 10)], [createPublicDataTreeLeaf(1, 20)]], - [[createPublicDataTreeLeaf(1, 10)], [createPublicDataTreeLeaf(1, 20), createPublicDataTreeLeaf(2, 5)]], - [ - [createPublicDataTreeLeaf(1, 10), createPublicDataTreeLeaf(2, 10)], - [createPublicDataTreeLeaf(1, 20), createPublicDataTreeLeaf(10, 50), createPublicDataTreeLeaf(2, 5)], - ], - ] as const)('performs batch upsert correctly', async (initialState, batch) => { - const TREE_HEIGHT = 16; - const INITIAL_TREE_SIZE = 8; - const SUBTREE_HEIGHT = 5; - - const db = openTmpStore(); - const appendTree = await newTree(PublicDataTree, db, poseidon, 'test', TREE_HEIGHT, INITIAL_TREE_SIZE); - const insertTree = await newTree(PublicDataTree, db, poseidon, 'test', TREE_HEIGHT, INITIAL_TREE_SIZE); - - appendTree.appendLeaves(initialState.map(leaf => leaf.toBuffer())); - insertTree.appendLeaves(initialState.map(leaf => leaf.toBuffer())); - - appendTree.appendLeaves(batch.map(leaf => leaf.toBuffer())); - await insertTree.batchInsert( - batch.map(leaf => leaf.toBuffer()), - SUBTREE_HEIGHT, - ); - - const expectedRoot = appendTree.getRoot(true); - const actualRoot = insertTree.getRoot(true); - expect(actualRoot).toEqual(expectedRoot); - }); - }); -}); diff --git a/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree_with_append.ts b/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree_with_append.ts deleted file mode 100644 index c10273575290..000000000000 --- a/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree_with_append.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { StandardIndexedTree } from '../standard_indexed_tree.js'; - -/** - * A testing utility which is here to store the original implementation of StandardIndexedTree.appendLeaves method - * that was replaced by the more efficient batchInsert method. We keep the original implementation around as it useful - * for testing that the more complex batchInsert method works correctly. - */ -export class StandardIndexedTreeWithAppend extends StandardIndexedTree { - /** - * Appends the given leaves to the tree. - * @param leaves - The leaves to append. - * @returns Empty promise. - * @remarks This method is inefficient and is here mostly for testing. Use batchInsert instead. - */ - public override appendLeaves(leaves: Buffer[]) { - for (const leaf of leaves) { - this.appendLeaf(leaf); - } - } - - private appendEmptyLeaf() { - const newSize = (this.cachedSize ?? this.size) + 1n; - if (newSize - 1n > this.maxIndex) { - throw Error(`Can't append beyond max index. Max index: ${this.maxIndex}`); - } - this.cachedSize = newSize; - } - - /** - * Appends the given leaf to the tree. - * @param leaf - The leaf to append. - * @returns Empty promise. - */ - private appendLeaf(leaf: Buffer): void { - const newLeaf = this.leafFactory.fromBuffer(leaf); - - // Special case when appending zero - if (newLeaf.getKey() === 0n) { - this.appendEmptyLeaf(); - return; - } - - const lowLeafIndex = this.findIndexOfPreviousKey(newLeaf.getKey(), true); - if (lowLeafIndex === undefined) { - throw new Error(`Previous leaf not found!`); - } - - const isUpdate = lowLeafIndex.alreadyPresent; - const lowLeafPreimage = this.getLatestLeafPreimageCopy(lowLeafIndex.index, true)!; - const currentSize = this.getNumLeaves(true); - - if (isUpdate) { - const newLowLeaf = lowLeafPreimage.asLeaf().updateTo(newLeaf); - const newLowLeafPreimage = this.leafPreimageFactory.fromLeaf( - newLowLeaf, - lowLeafPreimage.getNextKey(), - lowLeafPreimage.getNextIndex(), - ); - - this.updateLeaf(newLowLeafPreimage, BigInt(lowLeafIndex.index)); - this.appendEmptyLeaf(); - } else { - const newLeafPreimage = this.leafPreimageFactory.fromLeaf( - newLeaf, - lowLeafPreimage.getNextKey(), - lowLeafPreimage.getNextIndex(), - ); - - // insert a new leaf at the highest index and update the values of our previous leaf copy - const newLowLeafPreimage = this.leafPreimageFactory.fromLeaf( - lowLeafPreimage.asLeaf(), - newLeaf.getKey(), - BigInt(currentSize), - ); - this.updateLeaf(newLowLeafPreimage, BigInt(lowLeafIndex.index)); - this.updateLeaf(newLeafPreimage, currentSize); - } - } -} diff --git a/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts b/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts deleted file mode 100644 index 29e17406dbcc..000000000000 --- a/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { Fr } from '@aztec/foundation/curves/bn254'; -import type { FromBuffer } from '@aztec/foundation/serialize'; -import type { Hasher } from '@aztec/foundation/trees'; -import type { AztecKVStore } from '@aztec/kv-store'; -import { openTmpStore } from '@aztec/kv-store/lmdb'; - -import { loadTree } from '../load_tree.js'; -import { newTree } from '../new_tree.js'; -import { standardBasedTreeTestSuite } from '../test/standard_based_test_suite.js'; -import { treeTestSuite } from '../test/test_suite.js'; -import { PoseidonWithCounter } from '../test/utils/poseidon_with_counter.js'; -import { INITIAL_LEAF } from '../tree_base.js'; -import { StandardTree } from './standard_tree.js'; - -const noopDeserializer: FromBuffer = { - fromBuffer: (buffer: Buffer) => buffer, -}; - -const createDb = async (store: AztecKVStore, hasher: Hasher, name: string, depth: number) => { - return await newTree(StandardTree, store, hasher, name, noopDeserializer, depth); -}; - -const createFromName = async (store: AztecKVStore, hasher: Hasher, name: string) => { - return await loadTree(StandardTree, store, hasher, name, noopDeserializer); -}; - -treeTestSuite('StandardTree', createDb, createFromName); -standardBasedTreeTestSuite('StandardTree', createDb); - -describe('StandardTree_batchAppend', () => { - let poseidon: PoseidonWithCounter; - - beforeAll(() => { - poseidon = new PoseidonWithCounter(); - }); - - afterEach(() => { - poseidon.resetCounter(); - }); - - it('correctly computes root when batch appending and calls hash function expected num times', async () => { - const db = openTmpStore(); - const tree = await createDb(db, poseidon, 'test', 3); - const leaves = Array.from({ length: 5 }, _ => Fr.random().toBuffer()); - - poseidon.resetCounter(); - tree.appendLeaves(leaves); - - // We append 5 leaves so to update values we do the following hashing on each level: - // level2Node0 level2Node1 level2Node2 - // LEVEL2: [newLeaf0, newLeaf1], [newLeaf2, newLeaf3], [newLeaf4, INITIAL_LEAF]. - // level1Node0 level1Node1 - // LEVEL1: [level2Node0, level2Node1], [level2Node2, level2ZeroHash]. - // ROOT - // LEVEL0: [level1Node0, level1Node1]. - const level2NumHashing = 3; - const level1NumHashing = 2; - const level0NumHashing = 1; - const expectedNumHashing = level2NumHashing + level1NumHashing + level0NumHashing; - - expect(poseidon.hashCounter).toEqual(expectedNumHashing); - - const level2Node0 = poseidon.hash(leaves[0], leaves[1]); - const level2Node1 = poseidon.hash(leaves[2], leaves[3]); - const level2Node2 = poseidon.hash(leaves[4], INITIAL_LEAF); - - const level2ZeroHash = poseidon.hash(INITIAL_LEAF, INITIAL_LEAF); - - const level1Node0 = poseidon.hash(level2Node0, level2Node1); - const level1Node1 = poseidon.hash(level2Node2, level2ZeroHash); - - const root = poseidon.hash(level1Node0, level1Node1); - - expect(tree.getRoot(true)).toEqual(root); - }); - - it('should be able to find indexes of leaves', async () => { - const db = openTmpStore(); - const tree = await createDb(db, poseidon, 'test', 3); - const values = [Buffer.alloc(32, 1), Buffer.alloc(32, 2)]; - - tree.appendLeaves([values[0]]); - - expect(tree.findLeafIndex(values[0], true)).toBeDefined(); - expect(tree.findLeafIndex(values[0], false)).toBe(undefined); - expect(tree.findLeafIndex(values[1], true)).toBe(undefined); - - await tree.commit(); - - expect(tree.findLeafIndex(values[0], false)).toBeDefined(); - }); -}); diff --git a/yarn-project/merkle-tree/src/standard_tree/standard_tree.ts b/yarn-project/merkle-tree/src/standard_tree/standard_tree.ts deleted file mode 100644 index 419095c633a5..000000000000 --- a/yarn-project/merkle-tree/src/standard_tree/standard_tree.ts +++ /dev/null @@ -1,59 +0,0 @@ -import type { BlockNumber } from '@aztec/foundation/branded-types'; -import { type Bufferable, serializeToBuffer } from '@aztec/foundation/serialize'; -import { Timer } from '@aztec/foundation/timer'; -import type { TreeInsertionStats } from '@aztec/stdlib/stats'; - -import type { AppendOnlyTree } from '../interfaces/append_only_tree.js'; -import { AppendOnlySnapshotBuilder } from '../snapshots/append_only_snapshot.js'; -import type { TreeSnapshot } from '../snapshots/snapshot_builder.js'; -import { TreeBase } from '../tree_base.js'; - -/** - * A Merkle tree implementation that uses a LevelDB database to store the tree. - */ -export class StandardTree extends TreeBase implements AppendOnlyTree { - #snapshotBuilder = new AppendOnlySnapshotBuilder(this.store, this, this.hasher, this.deserializer); - - /** - * Appends the given leaves to the tree. - * @param leaves - The leaves to append. - * @returns Empty promise. - */ - public override appendLeaves(leaves: T[]): void { - this.hasher.reset(); - const timer = new Timer(); - super.appendLeaves(leaves); - this.log.debug(`Inserted ${leaves.length} leaves into ${this.getName()} tree`, { - eventName: 'tree-insertion', - duration: timer.ms(), - batchSize: leaves.length, - treeName: this.getName(), - treeDepth: this.getDepth(), - treeType: 'append-only', - ...this.hasher.stats(), - } satisfies TreeInsertionStats); - } - - public snapshot(blockNumber: BlockNumber): Promise> { - return this.#snapshotBuilder.snapshot(blockNumber); - } - - public getSnapshot(blockNumber: BlockNumber): Promise> { - return this.#snapshotBuilder.getSnapshot(blockNumber); - } - - public findLeafIndex(value: T, includeUncommitted: boolean): bigint | undefined { - return this.findLeafIndexAfter(value, 0n, includeUncommitted); - } - - public findLeafIndexAfter(value: T, startIndex: bigint, includeUncommitted: boolean): bigint | undefined { - const buffer = serializeToBuffer(value); - for (let i = startIndex; i < this.getNumLeaves(includeUncommitted); i++) { - const currentValue = this.getLeafValue(i, includeUncommitted); - if (currentValue && serializeToBuffer(currentValue).equals(buffer)) { - return i; - } - } - return undefined; - } -} diff --git a/yarn-project/merkle-tree/src/test/standard_based_test_suite.ts b/yarn-project/merkle-tree/src/test/standard_based_test_suite.ts deleted file mode 100644 index 4a50e654fbfe..000000000000 --- a/yarn-project/merkle-tree/src/test/standard_based_test_suite.ts +++ /dev/null @@ -1,138 +0,0 @@ -import { Fr } from '@aztec/foundation/curves/bn254'; -import { SiblingPath } from '@aztec/foundation/trees'; -import type { Hasher } from '@aztec/foundation/trees'; -import type { AztecKVStore } from '@aztec/kv-store'; -import { openTmpStore } from '@aztec/kv-store/lmdb'; - -import { INITIAL_LEAF, Poseidon } from '../index.js'; -import type { AppendOnlyTree } from '../interfaces/append_only_tree.js'; -import type { UpdateOnlyTree } from '../interfaces/update_only_tree.js'; -import { appendLeaves } from './utils/append_leaves.js'; - -const TEST_TREE_DEPTH = 2; - -export const standardBasedTreeTestSuite = ( - testName: string, - createDb: ( - store: AztecKVStore, - hasher: Hasher, - name: string, - depth: number, - ) => Promise, -) => { - describe(testName, () => { - let poseidon: Poseidon; - const values: Buffer[] = []; - - beforeAll(() => { - poseidon = new Poseidon(); - - for (let i = 0; i < 4; ++i) { - const v = Buffer.alloc(32, i + 1); - v.writeUInt32BE(i, 28); - values[i] = v; - } - }); - - it('should have correct empty tree root for depth 32', async () => { - const db = openTmpStore(); - const tree = await createDb(db, poseidon, 'test', 32); - const root = tree.getRoot(false); - expect(root.toString('hex')).toEqual('0b59baa35b9dc267744f0ccb4e3b0255c1fc512460d91130c6bc19fb2668568d'); - }); - - it('should throw when appending beyond max index', async () => { - const db = openTmpStore(); - const tree = await createDb(db, poseidon, 'test', 2); - const leaves = Array.from({ length: 5 }, _ => Fr.random().toBuffer()); - await expect(appendLeaves(tree, leaves)).rejects.toThrow(); - }); - - it('should have correct root and sibling paths', async () => { - const db = openTmpStore(); - const tree = await createDb(db, poseidon, 'test', 2); - - const level1ZeroHash = poseidon.hash(INITIAL_LEAF, INITIAL_LEAF); - expect(tree.getNumLeaves(false)).toEqual(0n); - expect(tree.getRoot(false)).toEqual(poseidon.hash(level1ZeroHash, level1ZeroHash)); - expect(await tree.getSiblingPath(0n, false)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, level1ZeroHash]), - ); - - await appendLeaves(tree, [values[0]]); - expect(tree.getNumLeaves(true)).toEqual(1n); - expect(tree.getNumLeaves(false)).toEqual(0n); - expect(tree.getRoot(true)).toEqual(poseidon.hash(poseidon.hash(values[0], INITIAL_LEAF), level1ZeroHash)); - expect(await tree.getSiblingPath(0n, true)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, level1ZeroHash]), - ); - expect(tree.getRoot(false)).toEqual(poseidon.hash(level1ZeroHash, level1ZeroHash)); - expect(await tree.getSiblingPath(0n, false)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, level1ZeroHash]), - ); - - await appendLeaves(tree, [values[1]]); - expect(tree.getNumLeaves(true)).toEqual(2n); - expect(tree.getRoot(true)).toEqual(poseidon.hash(poseidon.hash(values[0], values[1]), level1ZeroHash)); - expect(await tree.getSiblingPath(1n, true)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [values[0], level1ZeroHash]), - ); - expect(tree.getNumLeaves(false)).toEqual(0n); - expect(tree.getRoot(false)).toEqual(poseidon.hash(level1ZeroHash, level1ZeroHash)); - expect(await tree.getSiblingPath(1n, false)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, level1ZeroHash]), - ); - - await appendLeaves(tree, [values[2]]); - expect(tree.getNumLeaves(true)).toEqual(3n); - expect(tree.getRoot(true)).toEqual( - poseidon.hash(poseidon.hash(values[0], values[1]), poseidon.hash(values[2], INITIAL_LEAF)), - ); - expect(await tree.getSiblingPath(2n, true)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, poseidon.hash(values[0], values[1])]), - ); - expect(tree.getNumLeaves(false)).toEqual(0n); - expect(tree.getRoot(false)).toEqual(poseidon.hash(level1ZeroHash, level1ZeroHash)); - expect(await tree.getSiblingPath(2n, false)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, level1ZeroHash]), - ); - - await appendLeaves(tree, [values[3]]); - expect(tree.getNumLeaves(true)).toEqual(4n); - expect(tree.getRoot(true)).toEqual( - poseidon.hash(poseidon.hash(values[0], values[1]), poseidon.hash(values[2], values[3])), - ); - expect(await tree.getSiblingPath(3n, true)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [values[2], poseidon.hash(values[0], values[1])]), - ); - expect(tree.getNumLeaves(false)).toEqual(0n); - expect(tree.getRoot(false)).toEqual(poseidon.hash(level1ZeroHash, level1ZeroHash)); - expect(await tree.getSiblingPath(3n, false)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [INITIAL_LEAF, level1ZeroHash]), - ); - // Lifted from memory_tree.test.cpp to ensure consistency. - //expect(root.toString('hex')).toEqual('0bf2e78afd70f72b0e6eafb03c41faef167a82441b05e517cdf35d813302061f'); - expect(await tree.getSiblingPath(0n, true)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [values[1], poseidon.hash(values[2], values[3])]), - ); - expect(await tree.getSiblingPath(1n, true)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [values[0], poseidon.hash(values[2], values[3])]), - ); - expect(await tree.getSiblingPath(2n, true)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [values[3], poseidon.hash(values[0], values[1])]), - ); - expect(await tree.getSiblingPath(3n, true)).toEqual( - new SiblingPath(TEST_TREE_DEPTH, [values[2], poseidon.hash(values[0], values[1])]), - ); - - await tree.commit(); - // now committed state should equal uncommitted state - expect(await tree.getSiblingPath(0n, false)).toEqual(await tree.getSiblingPath(0n, true)); - expect(await tree.getSiblingPath(1n, false)).toEqual(await tree.getSiblingPath(1n, true)); - expect(await tree.getSiblingPath(2n, false)).toEqual(await tree.getSiblingPath(2n, true)); - expect(await tree.getSiblingPath(3n, false)).toEqual(await tree.getSiblingPath(3n, true)); - expect(tree.getNumLeaves(false)).toEqual(tree.getNumLeaves(true)); - expect(tree.getRoot(false)).toEqual(tree.getRoot(true)); - }); - }); -}; diff --git a/yarn-project/merkle-tree/src/test/test_suite.ts b/yarn-project/merkle-tree/src/test/test_suite.ts deleted file mode 100644 index ccb92e67fd55..000000000000 --- a/yarn-project/merkle-tree/src/test/test_suite.ts +++ /dev/null @@ -1,155 +0,0 @@ -import { SiblingPath } from '@aztec/foundation/trees'; -import type { Hasher } from '@aztec/foundation/trees'; -import type { AztecKVStore } from '@aztec/kv-store'; -import { openTmpStore } from '@aztec/kv-store/lmdb'; - -import { Poseidon } from '../index.js'; -import type { AppendOnlyTree } from '../interfaces/append_only_tree.js'; -import type { UpdateOnlyTree } from '../interfaces/update_only_tree.js'; -import { appendLeaves } from './utils/append_leaves.js'; - -const expectSameTrees = async ( - tree1: AppendOnlyTree | UpdateOnlyTree, - tree2: AppendOnlyTree | UpdateOnlyTree, - includeUncommitted = true, -) => { - const size = tree1.getNumLeaves(includeUncommitted); - expect(size).toBe(tree2.getNumLeaves(includeUncommitted)); - expect(tree1.getRoot(includeUncommitted).toString('hex')).toBe(tree2.getRoot(includeUncommitted).toString('hex')); - - for (let i = 0; i < size; ++i) { - const siblingPath1 = await tree1.getSiblingPath(BigInt(i), includeUncommitted); - const siblingPath2 = await tree2.getSiblingPath(BigInt(i), includeUncommitted); - expect(siblingPath2).toStrictEqual(siblingPath1); - } -}; - -export const treeTestSuite = ( - testName: string, - createDb: ( - store: AztecKVStore, - hasher: Hasher, - name: string, - depth: number, - ) => Promise, - createFromName: (store: AztecKVStore, hasher: Hasher, name: string) => Promise, -) => { - describe(testName, () => { - const values: Buffer[] = []; - let poseidon: Poseidon; - - beforeAll(() => { - for (let i = 0; i < 32; ++i) { - const v = Buffer.alloc(32, i + 1); - v.writeUInt32BE(i, 28); - values[i] = v; - } - }); - - beforeEach(() => { - poseidon = new Poseidon(); - }); - - it('should revert changes on rollback', async () => { - const dbEmpty = openTmpStore(); - const emptyTree = await createDb(dbEmpty, poseidon, 'test', 10); - - const db = openTmpStore(); - const tree = await createDb(db, poseidon, 'test2', 10); - await appendLeaves(tree, values.slice(0, 4)); - - const firstRoot = tree.getRoot(true); - expect(firstRoot).not.toEqual(emptyTree.getRoot(true)); - // committed root should still be the empty root - expect(tree.getRoot(false)).toEqual(emptyTree.getRoot(false)); - - await tree.rollback(); - - // both committed and uncommitted trees should be equal to the empty tree - await expectSameTrees(tree, emptyTree, true); - await expectSameTrees(tree, emptyTree, false); - - // append the leaves again - await appendLeaves(tree, values.slice(0, 4)); - - expect(tree.getRoot(true)).toEqual(firstRoot); - // committed root should still be the empty root - expect(tree.getRoot(false)).toEqual(emptyTree.getRoot(false)); - - expect(firstRoot).not.toEqual(emptyTree.getRoot(true)); - - await tree.rollback(); - - // both committed and uncommitted trees should be equal to the empty tree - await expectSameTrees(tree, emptyTree, true); - await expectSameTrees(tree, emptyTree, false); - }); - - it('should not revert changes after commit', async () => { - const dbEmpty = openTmpStore(); - const emptyTree = await createDb(dbEmpty, poseidon, 'test', 10); - - const db = openTmpStore(); - const tree = await createDb(db, poseidon, 'test2', 10); - await appendLeaves(tree, values.slice(0, 4)); - - expect(tree.getRoot(true)).not.toEqual(emptyTree.getRoot(true)); - // committed root should still be the empty root - expect(tree.getRoot(false)).toEqual(emptyTree.getRoot(false)); - - await tree.commit(); - await tree.rollback(); - - expect(tree.getRoot(true)).not.toEqual(emptyTree.getRoot(true)); - expect(tree.getRoot(false)).not.toEqual(emptyTree.getRoot(true)); - }); - - it('should be able to restore from previous committed data', async () => { - const db = openTmpStore(); - const tree = await createDb(db, poseidon, 'test', 10); - await appendLeaves(tree, values.slice(0, 4)); - await tree.commit(); - - const tree2 = await createFromName(db, poseidon, 'test'); - - // both committed and uncommitted should be equal to the restored data - expect(tree.getRoot(true)).toEqual(tree2.getRoot(true)); - expect(tree.getRoot(false)).toEqual(tree2.getRoot(false)); - for (let i = 0; i < 4; ++i) { - expect(await tree.getSiblingPath(BigInt(i), true)).toEqual(await tree2.getSiblingPath(BigInt(i), true)); - expect(await tree.getSiblingPath(BigInt(i), false)).toEqual(await tree2.getSiblingPath(BigInt(i), false)); - } - }); - - it('should throw an error if previous data does not exist for the given name', async () => { - const db = openTmpStore(); - await expect( - (async () => { - await createFromName(db, poseidon, 'a_whole_new_tree'); - })(), - ).rejects.toThrow(); - }); - - it('should serialize sibling path data to a buffer and be able to deserialize it back', async () => { - const db = openTmpStore(); - const tree = await createDb(db, poseidon, 'test', 10); - await appendLeaves(tree, values.slice(0, 1)); - - const siblingPath = await tree.getSiblingPath(0n, true); - const buf = siblingPath.toBuffer(); - const recovered = SiblingPath.fromBuffer(buf); - expect(recovered).toEqual(siblingPath); - const deserialized = SiblingPath.deserialize(buf); - expect(deserialized.elem).toEqual(siblingPath); - expect(deserialized.adv).toBe(4 + 10 * 32); - - const dummyData = Buffer.alloc(23, 1); - const paddedBuf = Buffer.concat([dummyData, buf]); - const recovered2 = SiblingPath.fromBuffer(paddedBuf, 23); - expect(recovered2).toEqual(siblingPath); - const deserialized2 = SiblingPath.deserialize(buf); - expect(deserialized2.elem).toEqual(siblingPath); - expect(deserialized2.adv).toBe(4 + 10 * 32); - }); - }); -}; diff --git a/yarn-project/merkle-tree/src/test/utils/append_leaves.ts b/yarn-project/merkle-tree/src/test/utils/append_leaves.ts deleted file mode 100644 index 0ed03e9f556c..000000000000 --- a/yarn-project/merkle-tree/src/test/utils/append_leaves.ts +++ /dev/null @@ -1,15 +0,0 @@ -import type { AppendOnlyTree } from '../../interfaces/append_only_tree.js'; -import type { UpdateOnlyTree } from '../../interfaces/update_only_tree.js'; - -export const appendLeaves = async (tree: AppendOnlyTree | UpdateOnlyTree, leaves: Buffer[]) => { - if ('appendLeaves' in tree) { - // This branch is used by the standard tree test suite, which implements appendLeaves - tree.appendLeaves(leaves); - } else { - // This branch is used by the sparse tree test suite, which does not implement appendLeaves - for (const value of leaves) { - const index = tree.getNumLeaves(true); - await tree.updateLeaf(value, index); - } - } -}; diff --git a/yarn-project/merkle-tree/src/test/utils/poseidon_with_counter.ts b/yarn-project/merkle-tree/src/test/utils/poseidon_with_counter.ts deleted file mode 100644 index 9271456a97cd..000000000000 --- a/yarn-project/merkle-tree/src/test/utils/poseidon_with_counter.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { Poseidon } from '../../index.js'; - -/** - * A test utility allowing us to count the number of times the hash function has been called. - * @deprecated Don't call poseidon2 directly in production code. Instead, create suitably-named functions for specific - * purposes. - */ -export class PoseidonWithCounter extends Poseidon { - /** - * The number of times the hash function has been called. - */ - public hashCounter = 0; - - /** - * Hashes two 32-byte arrays. - * @param lhs - The first 32-byte array. - * @param rhs - The second 32-byte array. - * @returns The new 32-byte hash. - * @deprecated Don't call poseidon2 directly in production code. Instead, create suitably-named functions for specific - * purposes. - */ - public override hash(lhs: Uint8Array, rhs: Uint8Array) { - this.hashCounter++; - return super.hash(lhs, rhs); - } - - /** - * Resets the hash counter. - * @returns void - */ - public resetCounter() { - this.hashCounter = 0; - } -} diff --git a/yarn-project/merkle-tree/src/tree_base.ts b/yarn-project/merkle-tree/src/tree_base.ts deleted file mode 100644 index 0a95c630fc0b..000000000000 --- a/yarn-project/merkle-tree/src/tree_base.ts +++ /dev/null @@ -1,359 +0,0 @@ -import { toBigIntLE, toBufferLE } from '@aztec/foundation/bigint-buffer'; -import { type Logger, createLogger } from '@aztec/foundation/log'; -import { type Bufferable, type FromBuffer, serializeToBuffer } from '@aztec/foundation/serialize'; -import { type Hasher, SiblingPath } from '@aztec/foundation/trees'; -import type { AztecKVStore, AztecMap, AztecSingleton } from '@aztec/kv-store'; - -import { HasherWithStats } from './hasher_with_stats.js'; -import type { MerkleTree } from './interfaces/merkle_tree.js'; - -const MAX_DEPTH = 254; - -const indexToKeyHash = (name: string, level: number, index: bigint) => `${name}:${level}:${index}`; -const encodeMeta = (root: Buffer, depth: number, size: bigint) => { - const data = Buffer.alloc(36); - root.copy(data); - data.writeUInt32LE(depth, 32); - return Buffer.concat([data, toBufferLE(size, 32)]); -}; -const decodeMeta = (meta: Buffer) => { - const root = meta.subarray(0, 32); - const depth = meta.readUInt32LE(32); - const size = toBigIntLE(meta.subarray(36)); - return { - root, - depth, - size, - }; -}; - -const openTreeMetaSingleton = (store: AztecKVStore, treeName: string): AztecSingleton => - store.openSingleton(`merkle_tree_${treeName}_meta`); - -export const getTreeMeta = (store: AztecKVStore, treeName: string) => { - const singleton = openTreeMetaSingleton(store, treeName); - const val = singleton.get(); - if (!val) { - throw new Error(); - } - return decodeMeta(val); -}; - -export const INITIAL_LEAF = Buffer.from('0000000000000000000000000000000000000000000000000000000000000000', 'hex'); - -/** - * A Merkle tree implementation that uses a LevelDB database to store the tree. - */ -export abstract class TreeBase implements MerkleTree { - protected readonly maxIndex: bigint; - protected cachedSize?: bigint; - private root!: Buffer; - private zeroHashes: Buffer[] = []; - private cache: { [key: string]: Buffer } = {}; - protected log: Logger; - protected hasher: HasherWithStats; - - private nodes: AztecMap; - private meta: AztecSingleton; - - public constructor( - protected store: AztecKVStore, - hasher: Hasher, - private name: string, - private depth: number, - protected size: bigint = 0n, - protected deserializer: FromBuffer, - root?: Buffer, - ) { - if (!(depth >= 1 && depth <= MAX_DEPTH)) { - throw Error('Invalid depth'); - } - - this.hasher = new HasherWithStats(hasher); - this.nodes = store.openMap('merkle_tree_' + name); - this.meta = openTreeMetaSingleton(store, name); - - // Compute the zero values at each layer. - let current = INITIAL_LEAF; - for (let i = depth - 1; i >= 0; --i) { - this.zeroHashes[i] = current; - current = hasher.hash(current, current); - } - - this.root = root ? root : current; - this.maxIndex = 2n ** BigInt(depth) - 1n; - - this.log = createLogger(`merkle-tree:${name.toLowerCase()}`); - } - - /** - * Returns the root of the tree. - * @param includeUncommitted - If true, root incorporating uncommitted changes is returned. - * @returns The root of the tree. - */ - public getRoot(includeUncommitted: boolean): Buffer { - return !includeUncommitted ? this.root : (this.cache[indexToKeyHash(this.name, 0, 0n)] ?? this.root); - } - - /** - * Returns the number of leaves in the tree. - * @param includeUncommitted - If true, the returned number of leaves includes uncommitted changes. - * @returns The number of leaves in the tree. - */ - public getNumLeaves(includeUncommitted: boolean) { - return !includeUncommitted ? this.size : (this.cachedSize ?? this.size); - } - - /** - * Returns the name of the tree. - * @returns The name of the tree. - */ - public getName(): string { - return this.name; - } - - /** - * Returns the depth of the tree. - * @returns The depth of the tree. - */ - public getDepth(): number { - return this.depth; - } - - /** - * Returns a sibling path for the element at the given index. - * @param index - The index of the element. - * @param includeUncommitted - Indicates whether to get a sibling path incorporating uncommitted changes. - * @returns A sibling path for the element at the given index. - * Note: The sibling path is an array of sibling hashes, with the lowest hash (leaf hash) first, and the highest hash last. - */ - public getSiblingPath(index: bigint, includeUncommitted: boolean): Promise> { - const path: Buffer[] = []; - let level = this.depth; - while (level > 0) { - const isRight = index & 0x01n; - const sibling = this.getLatestValueAtIndex(level, isRight ? index - 1n : index + 1n, includeUncommitted); - path.push(sibling); - level -= 1; - index >>= 1n; - } - return Promise.resolve(new SiblingPath(this.depth as N, path)); - } - - /** - * Commits the changes to the database. - * @returns Empty promise. - */ - public commit(): Promise { - return this.store.transaction(() => { - const keys = Object.getOwnPropertyNames(this.cache); - for (const key of keys) { - void this.nodes.set(key, this.cache[key]); - } - this.size = this.getNumLeaves(true); - this.root = this.getRoot(true); - - this.clearCache(); - - void this.writeMeta(); - }); - } - - /** - * Rolls back the not-yet-committed changes. - * @returns Empty promise. - */ - public rollback(): Promise { - this.clearCache(); - return Promise.resolve(); - } - - /** - * Gets the value at the given index. - * @param index - The index of the leaf. - * @param includeUncommitted - Indicates whether to include uncommitted changes. - * @returns Leaf value at the given index or undefined. - */ - public getLeafValue(index: bigint, includeUncommitted: boolean): T | undefined { - const buf = this.getLatestValueAtIndex(this.depth, index, includeUncommitted); - if (buf) { - return this.deserializer.fromBuffer(buf); - } else { - return undefined; - } - } - - /** - * Gets the value at the given index. - * @param index - The index of the leaf. - * @param includeUncommitted - Indicates whether to include uncommitted changes. - * @returns Leaf value at the given index or undefined. - */ - public getLeafBuffer(index: bigint, includeUncommitted: boolean): Buffer | undefined { - return this.getLatestValueAtIndex(this.depth, index, includeUncommitted); - } - - public getNode(level: number, index: bigint): Buffer | undefined { - if (level < 0 || level > this.depth) { - throw Error('Invalid level: ' + level); - } - - if (index < 0 || index >= 2n ** BigInt(level)) { - throw Error('Invalid index: ' + index); - } - - return this.dbGet(indexToKeyHash(this.name, level, index)); - } - - public getZeroHash(level: number): Buffer { - if (level <= 0 || level > this.depth) { - throw new Error('Invalid level'); - } - - return this.zeroHashes[level - 1]; - } - - /** - * Clears the cache. - */ - private clearCache() { - this.cache = {}; - this.cachedSize = undefined; - } - - /** - * Adds a leaf and all the hashes above it to the cache. - * @param leaf - Leaf to add to cache. - * @param index - Index of the leaf (used to derive the cache key). - */ - protected addLeafToCacheAndHashToRoot(leaf: Buffer, index: bigint) { - const key = indexToKeyHash(this.name, this.depth, index); - let current = leaf; - this.cache[key] = current; - let level = this.depth; - while (level > 0) { - const isRight = index & 0x01n; - const sibling = this.getLatestValueAtIndex(level, isRight ? index - 1n : index + 1n, true); - const lhs = isRight ? sibling : current; - const rhs = isRight ? current : sibling; - current = this.hasher.hash(lhs, rhs); - level -= 1; - index >>= 1n; - const cacheKey = indexToKeyHash(this.name, level, index); - this.cache[cacheKey] = current; - } - } - - /** - * Returns the latest value at the given index. - * @param level - The level of the tree. - * @param index - The index of the element. - * @param includeUncommitted - Indicates, whether to get include uncommitted changes. - * @returns The latest value at the given index. - * Note: If the value is not in the cache, it will be fetched from the database. - */ - private getLatestValueAtIndex(level: number, index: bigint, includeUncommitted: boolean): Buffer { - const key = indexToKeyHash(this.name, level, index); - if (includeUncommitted && this.cache[key] !== undefined) { - return this.cache[key]; - } - const committed = this.dbGet(key); - if (committed !== undefined) { - return committed; - } - return this.zeroHashes[level - 1]; - } - - /** - * Gets a value from db by key. - * @param key - The key to by which to get the value. - * @returns A value from the db based on the key. - */ - private dbGet(key: string): Buffer | undefined { - return this.nodes.get(key); - } - - /** - * Initializes the tree. - * @param prefilledSize - A number of leaves that are prefilled with values. - * @returns Empty promise. - */ - // eslint-disable-next-line @typescript-eslint/no-unused-vars - public async init(prefilledSize: number): Promise { - // prefilledSize is used only by Indexed Tree. - await this.writeMeta(); - } - - /** - * Writes meta data to the provided batch. - * @param batch - The batch to which to write the meta data. - */ - protected writeMeta() { - const data = encodeMeta(this.getRoot(true), this.depth, this.getNumLeaves(true)); - return this.meta.set(data); - } - - /** - * Appends the given leaves to the tree. - * @param leaves - The leaves to append. - * @returns Empty promise. - * - * @remarks The batch insertion algorithm works as follows: - * 1. Insert all the leaves, - * 2. start iterating over levels from the bottom up, - * 3. on each level iterate over all the affected nodes (i.e. nodes whose preimages have changed), - * 4. fetch the preimage, hash it and insert the updated value. - * @remarks This algorithm is optimal when it comes to the number of hashing operations. It might not be optimal when - * it comes to the number of database reads, but that should be irrelevant given that most of the time - * `getLatestValueAtIndex` will return a value from cache (because at least one of the 2 children was - * touched in previous iteration). - */ - protected appendLeaves(leaves: T[]): void { - const numLeaves = this.getNumLeaves(true); - if (numLeaves + BigInt(leaves.length) - 1n > this.maxIndex) { - throw Error(`Can't append beyond max index. Max index: ${this.maxIndex}`); - } - - // 1. Insert all the leaves - let firstIndex = numLeaves; - let level = this.depth; - for (let i = 0; i < leaves.length; i++) { - const cacheKey = indexToKeyHash(this.name, level, firstIndex + BigInt(i)); - this.cache[cacheKey] = serializeToBuffer(leaves[i]); - } - - let lastIndex = firstIndex + BigInt(leaves.length); - // 2. Iterate over all the levels from the bottom up - while (level > 0) { - firstIndex >>= 1n; - lastIndex >>= 1n; - // 3.Iterate over all the affected nodes at this level and update them - for (let index = firstIndex; index <= lastIndex; index++) { - const lhs = this.getLatestValueAtIndex(level, index * 2n, true); - const rhs = this.getLatestValueAtIndex(level, index * 2n + 1n, true); - const cacheKey = indexToKeyHash(this.name, level - 1, index); - this.cache[cacheKey] = this.hasher.hash(lhs, rhs); - } - - level -= 1; - } - this.cachedSize = numLeaves + BigInt(leaves.length); - } - - /** - * Returns the index of a leaf given its value, or undefined if no leaf with that value is found. - * @param value - The leaf value to look for. - * @param includeUncommitted - Indicates whether to include uncommitted data. - * @returns The index of the first leaf found with a given value (undefined if not found). - */ - abstract findLeafIndex(value: T, includeUncommitted: boolean): bigint | undefined; - - /** - * Returns the first index containing a leaf value after `startIndex`. - * @param leaf - The leaf value to look for. - * @param startIndex - The index to start searching from (used when skipping nullified messages) - * @param includeUncommitted - Indicates whether to include uncommitted data. - * @returns The index of the first leaf found with a given value (undefined if not found). - */ - abstract findLeafIndexAfter(leaf: T, startIndex: bigint, includeUncommitted: boolean): bigint | undefined; -} diff --git a/yarn-project/merkle-tree/src/unbalanced_tree.test.ts b/yarn-project/merkle-tree/src/unbalanced_tree.test.ts deleted file mode 100644 index 92f136172505..000000000000 --- a/yarn-project/merkle-tree/src/unbalanced_tree.test.ts +++ /dev/null @@ -1,271 +0,0 @@ -import { SHA256Trunc, sha256Trunc } from '@aztec/foundation/crypto/sha256'; -import { Fr } from '@aztec/foundation/curves/bn254'; -import type { FromBuffer } from '@aztec/foundation/serialize'; -import type { Hasher } from '@aztec/foundation/trees'; -import { openTmpStore } from '@aztec/kv-store/lmdb'; - -import { StandardTree } from './standard_tree/standard_tree.js'; -import { UnbalancedTree } from './unbalanced_tree.js'; - -const noopDeserializer: FromBuffer = { - fromBuffer: (buffer: Buffer) => buffer, -}; - -describe('Unbalanced tree', () => { - let hasher: Hasher; - let tree: UnbalancedTree; - let leaves: Buffer[]; - - const createAndFillTree = async (size: number) => { - const depth = Math.ceil(Math.log2(size)); - const tree = new UnbalancedTree(hasher, `test`, depth, noopDeserializer); - const leaves = Array(size) - .fill(0) - .map((_, i) => sha256Trunc(new Fr(i).toBuffer())); - // For the final test, we make the final (shifted up) leaf be H(1, 2), so we can calculate the root - // with a standard tree easily. - if (leaves[30]) { - leaves[30] = hasher.hash(new Fr(1).toBuffer(), new Fr(2).toBuffer()); - } - await tree.appendLeaves(leaves); - return { tree, leaves }; - }; - - beforeAll(() => { - hasher = new SHA256Trunc(); - }); - - // Example - 2 txs: - // - // root - // / \ - // base base - describe('2 Transactions', () => { - beforeAll(async () => { - const res = await createAndFillTree(2); - tree = res.tree; - leaves = res.leaves; - }); - - it("Shouldn't accept more leaves", () => { - expect(() => tree.appendLeaves([Buffer.alloc(32)])).toThrow( - "Can't re-append to an unbalanced tree. Current has 2 leaves.", - ); - }); - - it('Correctly computes tree information', () => { - expect(tree.getNumLeaves()).toEqual(BigInt(leaves.length)); - expect(tree.getDepth()).toEqual(1); - expect(tree.findLeafIndex(leaves[0])).toEqual(0n); - }); - - it('Correctly computes root', () => { - const root = tree.getRoot(); - const expectedRoot = sha256Trunc(Buffer.concat([leaves[0], leaves[1]])); - expect(root).toEqual(expectedRoot); - }); - - it('Correctly computes sibling path', async () => { - const sibPath = await tree.getSiblingPath(BigInt('0x' + leaves[0].toString('hex'))); - expect(sibPath.pathSize).toEqual(1); - const expectedSibPath = [leaves[1]]; - expect(sibPath.toBufferArray()).toEqual(expectedSibPath); - }); - }); - - // Example - 3 txs: - // - // root - // / \ - // merge base - // / \ - // base base - describe('3 Transactions', () => { - beforeAll(async () => { - const res = await createAndFillTree(3); - tree = res.tree; - leaves = res.leaves; - }); - - it('Correctly computes tree information', () => { - expect(tree.getNumLeaves()).toEqual(BigInt(leaves.length)); - expect(tree.getDepth()).toEqual(2); - expect(tree.findLeafIndex(leaves[0])).toEqual(0n); - }); - - it('Correctly computes root', () => { - const root = tree.getRoot(); - const mergeNode = sha256Trunc(Buffer.concat([leaves[0], leaves[1]])); - const expectedRoot = sha256Trunc(Buffer.concat([mergeNode, leaves[2]])); - expect(root).toEqual(expectedRoot); - }); - - it('Correctly computes sibling path', async () => { - const sibPath = await tree.getSiblingPath(BigInt('0x' + leaves[0].toString('hex'))); - expect(sibPath.pathSize).toEqual(2); - const expectedSibPath = [leaves[1], leaves[2]]; - expect(sibPath.toBufferArray()).toEqual(expectedSibPath); - }); - }); - - // Example - 5 txs: - // - // root - // / \ - // merge base - // / \ - // merge merge - // / \ / \ - // base base base base - describe('5 Transactions', () => { - beforeAll(async () => { - const res = await createAndFillTree(5); - tree = res.tree; - leaves = res.leaves; - }); - - it('Correctly computes tree information', () => { - expect(tree.getNumLeaves()).toEqual(BigInt(leaves.length)); - expect(tree.getDepth()).toEqual(3); - expect(tree.findLeafIndex(leaves[0])).toEqual(0n); - }); - - it('Correctly computes root', () => { - const root = tree.getRoot(); - let leftMergeNode = sha256Trunc(Buffer.concat([leaves[0], leaves[1]])); - const rightMergeNode = sha256Trunc(Buffer.concat([leaves[2], leaves[3]])); - leftMergeNode = sha256Trunc(Buffer.concat([leftMergeNode, rightMergeNode])); - const expectedRoot = sha256Trunc(Buffer.concat([leftMergeNode, leaves[4]])); - expect(root).toEqual(expectedRoot); - }); - - it('Correctly computes sibling path', async () => { - const sibPath = await tree.getSiblingPath(BigInt('0x' + leaves[0].toString('hex'))); - expect(sibPath.pathSize).toEqual(3); - const expectedSibPath = [leaves[1], sha256Trunc(Buffer.concat([leaves[2], leaves[3]])), leaves[4]]; - expect(sibPath.toBufferArray()).toEqual(expectedSibPath); - }); - }); - - // Example - 6 txs: - // - // root - // / \ - // merge4 merge3 - // / \ / \ - // merge1 merge2 base base - // / \ / \ - // base base base base - describe('6 Transactions', () => { - beforeAll(async () => { - const res = await createAndFillTree(6); - tree = res.tree; - leaves = res.leaves; - }); - - it('Correctly computes tree information', () => { - expect(tree.getNumLeaves()).toEqual(BigInt(leaves.length)); - expect(tree.getDepth()).toEqual(3); - expect(tree.findLeafIndex(leaves[0])).toEqual(0n); - }); - - it('Correctly computes root', () => { - const root = tree.getRoot(); - let leftMergeNode = sha256Trunc(Buffer.concat([leaves[0], leaves[1]])); - let rightMergeNode = sha256Trunc(Buffer.concat([leaves[2], leaves[3]])); - leftMergeNode = sha256Trunc(Buffer.concat([leftMergeNode, rightMergeNode])); - rightMergeNode = sha256Trunc(Buffer.concat([leaves[4], leaves[5]])); - const expectedRoot = sha256Trunc(Buffer.concat([leftMergeNode, rightMergeNode])); - expect(root).toEqual(expectedRoot); - }); - - it('Correctly computes sibling path', async () => { - const sibPath = await tree.getSiblingPath(BigInt('0x' + leaves[0].toString('hex'))); - expect(sibPath.pathSize).toEqual(3); - const expectedSibPath = [ - leaves[1], - sha256Trunc(Buffer.concat([leaves[2], leaves[3]])), - sha256Trunc(Buffer.concat([leaves[4], leaves[5]])), - ]; - expect(sibPath.toBufferArray()).toEqual(expectedSibPath); - }); - }); - - // Example - 7 txs: - // - // root - // / \ - // merge3 merge5 - // / \ / \ - // merge1 merge2 merge4 base - // / \ / \ / \ - // base base base base base base - describe('7 Transactions', () => { - let secondMergeNode: Buffer; - let fifthMergeNode: Buffer; - beforeAll(async () => { - const res = await createAndFillTree(7); - tree = res.tree; - leaves = res.leaves; - }); - - it('Correctly computes tree information', () => { - expect(tree.getNumLeaves()).toEqual(BigInt(leaves.length)); - expect(tree.getDepth()).toEqual(3); - expect(tree.findLeafIndex(leaves[0])).toEqual(0n); - }); - - it('Correctly computes root', () => { - const root = tree.getRoot(); - const firstMergeNode = sha256Trunc(Buffer.concat([leaves[0], leaves[1]])); - secondMergeNode = sha256Trunc(Buffer.concat([leaves[2], leaves[3]])); - const thirdMergeNode = sha256Trunc(Buffer.concat([firstMergeNode, secondMergeNode])); - const fourthMergeNode = sha256Trunc(Buffer.concat([leaves[4], leaves[5]])); - fifthMergeNode = sha256Trunc(Buffer.concat([fourthMergeNode, leaves[6]])); - const expectedRoot = sha256Trunc(Buffer.concat([thirdMergeNode, fifthMergeNode])); - expect(root).toEqual(expectedRoot); - }); - - it('Correctly computes sibling path', async () => { - const sibPath = await tree.getSiblingPath(BigInt('0x' + leaves[0].toString('hex'))); - expect(sibPath.pathSize).toEqual(3); - const expectedSibPath = [leaves[1], secondMergeNode, fifthMergeNode]; - expect(sibPath.toBufferArray()).toEqual(expectedSibPath); - }); - }); - - // Example - 31 txs: - // The same as a standard 32 leaf balanced tree, but with the last 'leaf' shifted up one. - describe('31 Transactions', () => { - let stdTree: StandardTree; - beforeAll(async () => { - const res = await createAndFillTree(31); - tree = res.tree; - leaves = res.leaves; - stdTree = new StandardTree(openTmpStore(true), hasher, `temp`, 5, 0n, noopDeserializer); - // We have set the last leaf to be H(1, 2), so we can fill a 32 size tree with: - stdTree.appendLeaves([...res.leaves.slice(0, 30), new Fr(1).toBuffer(), new Fr(2).toBuffer()]); - }); - - it('Correctly computes tree information', () => { - expect(tree.getNumLeaves()).toEqual(BigInt(leaves.length)); - expect(tree.getDepth()).toEqual(5); - expect(tree.findLeafIndex(leaves[0])).toEqual(0n); - }); - - it('Correctly computes root', () => { - const root = tree.getRoot(); - const expectedRoot = stdTree.getRoot(true); - expect(root).toEqual(expectedRoot); - }); - - it('Correctly computes sibling paths', async () => { - let sibPath = await tree.getSiblingPath(BigInt('0x' + leaves[0].toString('hex'))); - let expectedSibPath = await stdTree.getSiblingPath(0n, true); - expect(sibPath).toEqual(expectedSibPath); - sibPath = await tree.getSiblingPath(BigInt('0x' + leaves[27].toString('hex'))); - expectedSibPath = await stdTree.getSiblingPath(27n, true); - expect(sibPath).toEqual(expectedSibPath); - }); - }); -}); diff --git a/yarn-project/merkle-tree/src/unbalanced_tree.ts b/yarn-project/merkle-tree/src/unbalanced_tree.ts deleted file mode 100644 index 2d97bdafd74d..000000000000 --- a/yarn-project/merkle-tree/src/unbalanced_tree.ts +++ /dev/null @@ -1,239 +0,0 @@ -import { type Bufferable, type FromBuffer, serializeToBuffer } from '@aztec/foundation/serialize'; -import { SiblingPath } from '@aztec/foundation/trees'; -import type { Hasher } from '@aztec/foundation/trees'; - -import { HasherWithStats } from './hasher_with_stats.js'; -import type { MerkleTree } from './interfaces/merkle_tree.js'; - -const indexToKeyHash = (name: string, level: number, index: bigint) => `${name}:${level}:${index}`; - -/** - * An ephemeral unbalanced Merkle tree implementation. - * Follows the rollup implementation which greedily hashes pairs of nodes up the tree. - * Remaining rightmost nodes are shifted up until they can be paired. - */ -export class UnbalancedTree implements MerkleTree { - // This map stores index and depth -> value - private cache: { [key: string]: Buffer } = {}; - // This map stores value -> index and depth, since we have variable depth - private valueCache: { [key: string]: string } = {}; - protected size: bigint = 0n; - protected readonly maxIndex: bigint; - - protected hasher: HasherWithStats; - root: Buffer = Buffer.alloc(32); - - public constructor( - hasher: Hasher, - private name: string, - private maxDepth: number = 0, - protected deserializer: FromBuffer, - ) { - this.hasher = new HasherWithStats(hasher); - this.maxIndex = 2n ** BigInt(this.maxDepth) - 1n; - } - - /** - * Returns the root of the tree. - * @returns The root of the tree. - */ - public getRoot(): Buffer { - return this.root; - } - - /** - * Returns the number of leaves in the tree. - * @returns The number of leaves in the tree. - */ - public getNumLeaves() { - return this.size; - } - - /** - * Returns the max depth of the tree. - * @returns The max depth of the tree. - */ - public getDepth(): number { - return this.maxDepth; - } - - /** - * @remark A wonky tree is (currently) only ever ephemeral, so we don't use any db to commit to. - * The fn must exist to implement MerkleTree however. - */ - public commit(): Promise { - throw new Error("Unsupported function - cannot commit on an unbalanced tree as it's always ephemeral."); - return Promise.resolve(); - } - - /** - * Rolls back the not-yet-committed changes. - * @returns Empty promise. - */ - public rollback(): Promise { - this.clearCache(); - return Promise.resolve(); - } - - /** - * Clears the cache. - */ - private clearCache() { - this.cache = {}; - this.size = 0n; - } - - /** - * @remark A wonky tree can validly have duplicate indices: - * e.g. 001 (index 1 at level 3) and 01 (index 1 at level 2) - * So this function cannot reliably give the expected leaf value. - * We cannot add level as an input as its based on the MerkleTree class's function. - */ - public getLeafValue(_index: bigint): undefined { - throw new Error('Unsupported function - cannot get leaf value from an index in an unbalanced tree.'); - } - - /** - * Returns the index of a leaf given its value, or undefined if no leaf with that value is found. - * @param leaf - The leaf value to look for. - * @returns The index of the first leaf found with a given value (undefined if not found). - * @remark This is NOT the index as inserted, but the index which will be used to calculate path structure. - */ - public findLeafIndex(value: T): bigint | undefined { - const key = this.valueCache[serializeToBuffer(value).toString('hex')]; - const [, , index] = key.split(':'); - return BigInt(index); - } - - /** - * Returns the first index containing a leaf value after `startIndex`. - * @param value - The leaf value to look for. - * @param startIndex - The index to start searching from. - * @returns The index of the first leaf found with a given value (undefined if not found). - * @remark This is not really used for a wonky tree, but required to implement MerkleTree. - */ - public findLeafIndexAfter(value: T, startIndex: bigint): bigint | undefined { - const index = this.findLeafIndex(value); - if (!index || index < startIndex) { - return undefined; - } - return index; - } - - /** - * Returns the node at the given level and index - * @param level - The level of the element (root is at level 0). - * @param index - The index of the element. - * @returns Leaf or node value, or undefined. - */ - public getNode(level: number, index: bigint): Buffer | undefined { - if (level < 0 || level > this.maxDepth) { - throw Error('Invalid level: ' + level); - } - - if (index < 0 || index >= this.maxIndex) { - throw Error('Invalid index: ' + index); - } - - return this.cache[indexToKeyHash(this.name, level, index)]; - } - - /** - * Returns a sibling path for the element at the given index. - * @param value - The value of the element. - * @returns A sibling path for the element. - * Note: The sibling path is an array of sibling hashes, with the lowest hash (leaf hash) first, and the highest hash last. - */ - public getSiblingPath(value: bigint): Promise> { - const path: Buffer[] = []; - const [, depth, _index] = this.valueCache[serializeToBuffer(value).toString('hex')].split(':'); - let level = parseInt(depth, 10); - let index = BigInt(_index); - while (level > 0) { - const isRight = index & 0x01n; - const key = indexToKeyHash(this.name, level, isRight ? index - 1n : index + 1n); - const sibling = this.cache[key]; - path.push(sibling); - level -= 1; - index >>= 1n; - } - return Promise.resolve(new SiblingPath(parseInt(depth, 10) as N, path)); - } - - /** - * Appends the given leaves to the tree. - * @param leaves - The leaves to append. - * @returns Empty promise. - */ - public appendLeaves(leaves: T[]): Promise { - this.hasher.reset(); - if (this.size != BigInt(0)) { - throw Error(`Can't re-append to an unbalanced tree. Current has ${this.size} leaves.`); - } - if (this.size + BigInt(leaves.length) - 1n > this.maxIndex) { - throw Error(`Can't append beyond max index. Max index: ${this.maxIndex}`); - } - const root = this.batchInsert(leaves); - this.root = root; - - return Promise.resolve(); - } - - /** - * Calculates root while adding leaves and nodes to the cache. - * @param leaves - The leaves to append. - * @returns Resulting root of the tree. - */ - private batchInsert(_leaves: T[]): Buffer { - // If we have an even number of leaves, hash them all in pairs - // Otherwise, store the final leaf to be shifted up to the next odd sized level - let [layerWidth, nodeToShift] = - _leaves.length & 1 - ? [_leaves.length - 1, serializeToBuffer(_leaves[_leaves.length - 1])] - : [_leaves.length, Buffer.alloc(0)]; - // Allocate this layer's leaves and init the next layer up - let thisLayer = _leaves.slice(0, layerWidth).map(l => serializeToBuffer(l)); - let nextLayer = []; - // Store the bottom level leaves - thisLayer.forEach((leaf, i) => this.storeNode(leaf, this.maxDepth, BigInt(i))); - for (let i = 0; i < this.maxDepth; i++) { - for (let j = 0; j < layerWidth; j += 2) { - // Store the hash of each pair one layer up - nextLayer[j / 2] = this.hasher.hash(serializeToBuffer(thisLayer[j]), serializeToBuffer(thisLayer[j + 1])); - this.storeNode(nextLayer[j / 2], this.maxDepth - i - 1, BigInt(j >> 1)); - } - layerWidth /= 2; - if (layerWidth & 1) { - if (nodeToShift.length) { - // If the next layer has odd length, and we have a node that needs to be shifted up, add it here - nextLayer.push(serializeToBuffer(nodeToShift)); - this.storeNode(nodeToShift, this.maxDepth - i - 1, BigInt((layerWidth * 2) >> 1)); - layerWidth += 1; - nodeToShift = Buffer.alloc(0); - } else { - // If we don't have a node waiting to be shifted, store the next layer's final node to be shifted - layerWidth -= 1; - nodeToShift = nextLayer[layerWidth]; - } - } - // reset the layers - thisLayer = nextLayer; - nextLayer = []; - } - this.size += BigInt(_leaves.length); - // return the root - return thisLayer[0]; - } - - /** - * Stores the given node in the cache. - * @param value - The value to store. - * @param depth - The depth of the node in the full tree. - * @param index - The index of the node at the given depth. - */ - private storeNode(value: T | Buffer, depth: number, index: bigint) { - const key = indexToKeyHash(this.name, depth, index); - this.cache[key] = serializeToBuffer(value); - this.valueCache[serializeToBuffer(value).toString('hex')] = key; - } -} diff --git a/yarn-project/merkle-tree/tsconfig.json b/yarn-project/merkle-tree/tsconfig.json deleted file mode 100644 index b9af945fbd98..000000000000 --- a/yarn-project/merkle-tree/tsconfig.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "extends": "..", - "compilerOptions": { - "outDir": "dest", - "rootDir": "src", - "tsBuildInfoFile": ".tsbuildinfo" - }, - "references": [ - { - "path": "../foundation" - }, - { - "path": "../kv-store" - }, - { - "path": "../stdlib" - } - ], - "include": ["src"] -} diff --git a/yarn-project/node-lib/package.json b/yarn-project/node-lib/package.json index 5b160bddfcee..bbe3ee3e7165 100644 --- a/yarn-project/node-lib/package.json +++ b/yarn-project/node-lib/package.json @@ -74,7 +74,6 @@ "@aztec/ethereum": "workspace:^", "@aztec/foundation": "workspace:^", "@aztec/kv-store": "workspace:^", - "@aztec/merkle-tree": "workspace:^", "@aztec/p2p": "workspace:^", "@aztec/protocol-contracts": "workspace:^", "@aztec/prover-client": "workspace:^", diff --git a/yarn-project/node-lib/tsconfig.json b/yarn-project/node-lib/tsconfig.json index 456917e31fde..a9fb78a6f642 100644 --- a/yarn-project/node-lib/tsconfig.json +++ b/yarn-project/node-lib/tsconfig.json @@ -30,9 +30,6 @@ { "path": "../kv-store" }, - { - "path": "../merkle-tree" - }, { "path": "../p2p" }, diff --git a/yarn-project/noir-protocol-circuits-types/package.json b/yarn-project/noir-protocol-circuits-types/package.json index 7c034dd4fad3..a842e5e00024 100644 --- a/yarn-project/noir-protocol-circuits-types/package.json +++ b/yarn-project/noir-protocol-circuits-types/package.json @@ -76,7 +76,6 @@ }, "devDependencies": { "@aztec/kv-store": "workspace:^", - "@aztec/merkle-tree": "workspace:^", "@jest/globals": "^30.0.0", "@swc/helpers": "^0.5.15", "@types/jest": "^30.0.0", diff --git a/yarn-project/noir-protocol-circuits-types/tsconfig.json b/yarn-project/noir-protocol-circuits-types/tsconfig.json index 1e6763d09e77..77050135d28f 100644 --- a/yarn-project/noir-protocol-circuits-types/tsconfig.json +++ b/yarn-project/noir-protocol-circuits-types/tsconfig.json @@ -21,9 +21,6 @@ }, { "path": "../kv-store" - }, - { - "path": "../merkle-tree" } ], "include": ["src", "artifacts/*.d.json.ts", "artifacts/**/*.d.json.ts"], diff --git a/yarn-project/package.json b/yarn-project/package.json index 6524ec3ba123..c981fb62445b 100644 --- a/yarn-project/package.json +++ b/yarn-project/package.json @@ -40,7 +40,6 @@ "key-store", "kv-store", "l1-artifacts", - "merkle-tree", "native", "node-keystore", "node-lib", diff --git a/yarn-project/pxe/package.json b/yarn-project/pxe/package.json index 4c42df489cb4..a10b2bf35c6e 100644 --- a/yarn-project/pxe/package.json +++ b/yarn-project/pxe/package.json @@ -91,8 +91,8 @@ "viem": "npm:@aztec/viem@2.38.2" }, "devDependencies": { - "@aztec/merkle-tree": "workspace:^", "@aztec/noir-test-contracts.js": "workspace:^", + "@aztec/world-state": "workspace:^", "@jest/globals": "^30.0.0", "@types/jest": "^30.0.0", "@types/lodash.omit": "^4.5.7", diff --git a/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution.test.ts b/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution.test.ts index e82ca7a4003f..0387085d1097 100644 --- a/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution.test.ts +++ b/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution.test.ts @@ -1,10 +1,4 @@ -import { - DomainSeparator, - L1_TO_L2_MSG_TREE_HEIGHT, - NOTE_HASH_TREE_HEIGHT, - NULL_MSG_SENDER_CONTRACT_ADDRESS, - PUBLIC_DATA_TREE_HEIGHT, -} from '@aztec/constants'; +import { DomainSeparator, NULL_MSG_SENDER_CONTRACT_ADDRESS } from '@aztec/constants'; import { asyncMap } from '@aztec/foundation/async-map'; import { BlockNumber } from '@aztec/foundation/branded-types'; import { times } from '@aztec/foundation/collection'; @@ -17,8 +11,6 @@ import { EthAddress } from '@aztec/foundation/eth-address'; import { type Logger, createLogger } from '@aztec/foundation/log'; import type { FieldsOf } from '@aztec/foundation/types'; import { KeyStore } from '@aztec/key-store'; -import { openTmpStore } from '@aztec/kv-store/lmdb'; -import { type AppendOnlyTree, Poseidon, StandardTree, newTree } from '@aztec/merkle-tree'; import { ChildContractArtifact } from '@aztec/noir-test-contracts.js/Child'; import { ParentContractArtifact } from '@aztec/noir-test-contracts.js/Parent'; import { PendingNoteHashesContractArtifact } from '@aztec/noir-test-contracts.js/PendingNoteHashes'; @@ -44,22 +36,16 @@ import { import { GasFees, GasSettings } from '@aztec/stdlib/gas'; import { computeNoteHashNonce, computeSecretHash, computeUniqueNoteHash, siloNoteHash } from '@aztec/stdlib/hash'; import type { AztecNode } from '@aztec/stdlib/interfaces/client'; +import type { MerkleTreeWriteOperations } from '@aztec/stdlib/interfaces/server'; import { KeyValidationRequest } from '@aztec/stdlib/kernel'; import { computeAppNullifierHidingKey, deriveKeys } from '@aztec/stdlib/keys'; import type { SiloedTag } from '@aztec/stdlib/logs'; import { L1Actor, L1ToL2Message, L2Actor } from '@aztec/stdlib/messaging'; import { Note, NoteDao } from '@aztec/stdlib/note'; import { makeBlockHeader, makeL2Tips } from '@aztec/stdlib/testing'; -import { AppendOnlyTreeSnapshot } from '@aztec/stdlib/trees'; -import { - BlockHeader, - HashedValues, - PartialStateReference, - StateReference, - TxContext, - TxExecutionRequest, - TxHash, -} from '@aztec/stdlib/tx'; +import { MerkleTreeId } from '@aztec/stdlib/trees'; +import { BlockHeader, HashedValues, TxContext, TxExecutionRequest, TxHash } from '@aztec/stdlib/tx'; +import { NativeWorldStateService } from '@aztec/world-state'; import { jest } from '@jest/globals'; import { Matcher, type MatcherCreator, type MockProxy, mock } from 'jest-mock-extended'; @@ -150,13 +136,14 @@ describe('Private Execution test suite', () => { const TEST_JOB_ID = 'test-job-id'; - const treeHeights: { [name: string]: number } = { - noteHash: NOTE_HASH_TREE_HEIGHT, - l1ToL2Messages: L1_TO_L2_MSG_TREE_HEIGHT, - publicData: PUBLIC_DATA_TREE_HEIGHT, + const treeNameToId: { [name: string]: MerkleTreeId } = { + noteHash: MerkleTreeId.NOTE_HASH_TREE, + l1ToL2Messages: MerkleTreeId.L1_TO_L2_MESSAGE_TREE, + publicData: MerkleTreeId.PUBLIC_DATA_TREE, }; - let trees: { [name: keyof typeof treeHeights]: AppendOnlyTree } = {}; + let ws: NativeWorldStateService; + let fork: MerkleTreeWriteOperations; const txContextFields: FieldsOf = { chainId: new Fr(10), version: new Fr(20), @@ -229,49 +216,22 @@ describe('Private Execution test suite', () => { }; const insertLeaves = async (leaves: Fr[], name = 'noteHash') => { - if (!treeHeights[name]) { + const treeId = treeNameToId[name]; + if (treeId === undefined) { throw new Error(`Unknown tree ${name}`); } - if (!trees[name]) { - const db = openTmpStore(); - const poseidon = new Poseidon(); - trees[name] = await newTree(StandardTree, db, poseidon, name, Fr, treeHeights[name]); - } - const tree = trees[name]; - - tree.appendLeaves(leaves); - - // Create a new snapshot. - const newSnap = new AppendOnlyTreeSnapshot(Fr.fromBuffer(tree.getRoot(true)), Number(tree.getNumLeaves(true))); - - if (name === 'noteHash' || name === 'l1ToL2Messages' || name === 'publicData') { - anchorBlockHeader = new BlockHeader( - anchorBlockHeader.lastArchive, - new StateReference( - name === 'l1ToL2Messages' ? newSnap : anchorBlockHeader.state.l1ToL2MessageTree, - new PartialStateReference( - name === 'noteHash' ? newSnap : anchorBlockHeader.state.partial.noteHashTree, - anchorBlockHeader.state.partial.nullifierTree, - name === 'publicData' ? newSnap : anchorBlockHeader.state.partial.publicDataTree, - ), - ), - anchorBlockHeader.spongeBlobHash, - anchorBlockHeader.globalVariables, - anchorBlockHeader.totalFees, - anchorBlockHeader.totalManaUsed, - ); - } else { - anchorBlockHeader = new BlockHeader( - anchorBlockHeader.lastArchive, - new StateReference(newSnap, anchorBlockHeader.state.partial), - anchorBlockHeader.spongeBlobHash, - anchorBlockHeader.globalVariables, - anchorBlockHeader.totalFees, - anchorBlockHeader.totalManaUsed, - ); - } - return trees[name]; + await fork.appendLeaves(treeId, leaves); + const state = await fork.getStateReference(); + + anchorBlockHeader = new BlockHeader( + anchorBlockHeader.lastArchive, + state, + anchorBlockHeader.spongeBlobHash, + anchorBlockHeader.globalVariables, + anchorBlockHeader.totalFees, + anchorBlockHeader.totalManaUsed, + ); }; const computeNoteHash = (note: Note, owner: AztecAddress, storageSlot: Fr, randomness: Fr) => { @@ -312,8 +272,14 @@ describe('Private Execution test suite', () => { defaultContractAddress = await AztecAddress.random(); }); + afterEach(async () => { + await fork?.close(); + await ws?.close(); + }); + beforeEach(async () => { - trees = {}; + ws = await NativeWorldStateService.tmp(); + fork = await ws.fork(); contractStore = mock(); noteStore = mock(); noteStore.getNotes.mockResolvedValue([]); @@ -813,9 +779,9 @@ describe('Private Execution test suite', () => { ]; const mockOracles = async () => { - const tree = await insertLeaves([preimage.hash()], 'l1ToL2Messages'); + await insertLeaves([preimage.hash()], 'l1ToL2Messages'); aztecNode.getL1ToL2MessageMembershipWitness.mockImplementation(async () => { - return Promise.resolve([0n, await tree.getSiblingPath(0n, true)]); + return Promise.resolve([0n, await fork.getSiblingPath(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, 0n)]); }); aztecNode.findLeavesIndexes.mockImplementation(() => { return Promise.resolve([]); diff --git a/yarn-project/pxe/tsconfig.json b/yarn-project/pxe/tsconfig.json index ddcba16ddda4..e26348ab59f4 100644 --- a/yarn-project/pxe/tsconfig.json +++ b/yarn-project/pxe/tsconfig.json @@ -40,7 +40,7 @@ "path": "../stdlib" }, { - "path": "../merkle-tree" + "path": "../world-state" }, { "path": "../noir-test-contracts.js" diff --git a/yarn-project/sequencer-client/package.json b/yarn-project/sequencer-client/package.json index 862ef20f54bc..22b4af6455cb 100644 --- a/yarn-project/sequencer-client/package.json +++ b/yarn-project/sequencer-client/package.json @@ -35,7 +35,6 @@ "@aztec/ethereum": "workspace:^", "@aztec/foundation": "workspace:^", "@aztec/l1-artifacts": "workspace:^", - "@aztec/merkle-tree": "workspace:^", "@aztec/node-keystore": "workspace:^", "@aztec/noir-acvm_js": "portal:../../noir/packages/acvm_js", "@aztec/noir-contracts.js": "workspace:^", diff --git a/yarn-project/sequencer-client/tsconfig.json b/yarn-project/sequencer-client/tsconfig.json index 4abe6f06daee..3cd3bce0d7f0 100644 --- a/yarn-project/sequencer-client/tsconfig.json +++ b/yarn-project/sequencer-client/tsconfig.json @@ -33,9 +33,6 @@ { "path": "../l1-artifacts" }, - { - "path": "../merkle-tree" - }, { "path": "../node-keystore" }, diff --git a/yarn-project/simulator/package.json b/yarn-project/simulator/package.json index 2d727b483aa6..46d81e33a242 100644 --- a/yarn-project/simulator/package.json +++ b/yarn-project/simulator/package.json @@ -81,7 +81,6 @@ }, "devDependencies": { "@aztec/kv-store": "workspace:^", - "@aztec/merkle-tree": "workspace:^", "@aztec/noir-contracts.js": "workspace:^", "@aztec/noir-test-contracts.js": "workspace:^", "@jest/globals": "^30.0.0", diff --git a/yarn-project/simulator/tsconfig.json b/yarn-project/simulator/tsconfig.json index e9d50ce7351e..26ec1662eecd 100644 --- a/yarn-project/simulator/tsconfig.json +++ b/yarn-project/simulator/tsconfig.json @@ -33,9 +33,6 @@ { "path": "../kv-store" }, - { - "path": "../merkle-tree" - }, { "path": "../noir-contracts.js" }, diff --git a/yarn-project/tsconfig.json b/yarn-project/tsconfig.json index ff81b33d59d8..41951800acf7 100644 --- a/yarn-project/tsconfig.json +++ b/yarn-project/tsconfig.json @@ -42,7 +42,6 @@ { "path": "key-store/tsconfig.json" }, { "path": "l1-artifacts/tsconfig.json" }, { "path": "ethereum/tsconfig.json" }, - { "path": "merkle-tree/tsconfig.json" }, { "path": "native/tsconfig.json" }, { "path": "kv-store/tsconfig.json" }, { "path": "epoch-cache/tsconfig.json" }, diff --git a/yarn-project/typedoc.json b/yarn-project/typedoc.json index 90edf2de43f1..ea2fe0ec75e0 100644 --- a/yarn-project/typedoc.json +++ b/yarn-project/typedoc.json @@ -17,7 +17,6 @@ "aztec-node", "sequencer-client", "types", - "world-state", - "merkle-tree" + "world-state" ] } diff --git a/yarn-project/world-state/package.json b/yarn-project/world-state/package.json index 6bb598e1e186..18d23db27ea1 100644 --- a/yarn-project/world-state/package.json +++ b/yarn-project/world-state/package.json @@ -67,7 +67,6 @@ "@aztec/constants": "workspace:^", "@aztec/foundation": "workspace:^", "@aztec/kv-store": "workspace:^", - "@aztec/merkle-tree": "workspace:^", "@aztec/native": "workspace:^", "@aztec/protocol-contracts": "workspace:^", "@aztec/stdlib": "workspace:^", diff --git a/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts b/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts index 09beb473a79d..86ff3ee57655 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts @@ -1,14 +1,12 @@ import { MAX_NULLIFIERS_PER_TX, MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX } from '@aztec/constants'; import type { BlockNumber } from '@aztec/foundation/branded-types'; import type { Fr } from '@aztec/foundation/curves/bn254'; -import type { IndexedTreeSnapshot, TreeSnapshot } from '@aztec/merkle-tree'; import type { L2Block } from '@aztec/stdlib/block'; import type { ForkMerkleTreeOperations, MerkleTreeReadOperations, ReadonlyWorldStateAccess, } from '@aztec/stdlib/interfaces/server'; -import type { MerkleTreeId } from '@aztec/stdlib/trees'; import type { WorldStateStatusFull, WorldStateStatusSummary } from '../native/message.js'; @@ -31,14 +29,6 @@ export const INITIAL_NULLIFIER_TREE_SIZE = 2 * MAX_NULLIFIERS_PER_TX; export const INITIAL_PUBLIC_DATA_TREE_SIZE = 2 * MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX; -export type TreeSnapshots = { - [MerkleTreeId.NULLIFIER_TREE]: IndexedTreeSnapshot; - [MerkleTreeId.NOTE_HASH_TREE]: TreeSnapshot; - [MerkleTreeId.PUBLIC_DATA_TREE]: IndexedTreeSnapshot; - [MerkleTreeId.L1_TO_L2_MESSAGE_TREE]: TreeSnapshot; - [MerkleTreeId.ARCHIVE]: TreeSnapshot; -}; - export interface MerkleTreeAdminDatabase extends ForkMerkleTreeOperations, ReadonlyWorldStateAccess { /** * Handles a single L2 block (i.e. Inserts the new note hashes into the merkle tree). diff --git a/yarn-project/world-state/tsconfig.json b/yarn-project/world-state/tsconfig.json index c370ed9f98a1..ad5c5ae3de92 100644 --- a/yarn-project/world-state/tsconfig.json +++ b/yarn-project/world-state/tsconfig.json @@ -15,9 +15,6 @@ { "path": "../kv-store" }, - { - "path": "../merkle-tree" - }, { "path": "../native" }, diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index bea8063ceed0..5ae2655a3b8e 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -786,7 +786,6 @@ __metadata: "@aztec/foundation": "workspace:^" "@aztec/kv-store": "workspace:^" "@aztec/l1-artifacts": "workspace:^" - "@aztec/merkle-tree": "workspace:^" "@aztec/node-keystore": "workspace:^" "@aztec/node-lib": "workspace:^" "@aztec/noir-protocol-circuits-types": "workspace:^" @@ -1232,7 +1231,6 @@ __metadata: "@aztec/foundation": "workspace:^" "@aztec/kv-store": "workspace:^" "@aztec/l1-artifacts": "workspace:^" - "@aztec/merkle-tree": "workspace:^" "@aztec/node-keystore": "workspace:^" "@aztec/noir-contracts.js": "workspace:^" "@aztec/noir-noirc_abi": "portal:../../noir/packages/noirc_abi" @@ -1548,26 +1546,6 @@ __metadata: languageName: unknown linkType: soft -"@aztec/merkle-tree@workspace:^, @aztec/merkle-tree@workspace:merkle-tree": - version: 0.0.0-use.local - resolution: "@aztec/merkle-tree@workspace:merkle-tree" - dependencies: - "@aztec/foundation": "workspace:^" - "@aztec/kv-store": "workspace:^" - "@aztec/stdlib": "workspace:^" - "@jest/globals": "npm:^30.0.0" - "@types/jest": "npm:^30.0.0" - "@types/node": "npm:^22.15.17" - "@types/sha256": "npm:^0.2.0" - "@typescript/native-preview": "npm:7.0.0-dev.20260113.1" - jest: "npm:^30.0.0" - sha256: "npm:^0.2.0" - ts-node: "npm:^10.9.1" - tslib: "npm:^2.4.0" - typescript: "npm:^5.3.3" - languageName: unknown - linkType: soft - "@aztec/native@workspace:^, @aztec/native@workspace:native": version: 0.0.0-use.local resolution: "@aztec/native@workspace:native" @@ -1619,7 +1597,6 @@ __metadata: "@aztec/ethereum": "workspace:^" "@aztec/foundation": "workspace:^" "@aztec/kv-store": "workspace:^" - "@aztec/merkle-tree": "workspace:^" "@aztec/node-keystore": "workspace:^" "@aztec/p2p": "workspace:^" "@aztec/protocol-contracts": "workspace:^" @@ -1705,7 +1682,6 @@ __metadata: "@aztec/constants": "workspace:^" "@aztec/foundation": "workspace:^" "@aztec/kv-store": "workspace:^" - "@aztec/merkle-tree": "workspace:^" "@aztec/noir-acvm_js": "portal:../../noir/packages/acvm_js" "@aztec/noir-noir_codegen": "portal:../../noir/packages/noir_codegen" "@aztec/noir-noirc_abi": "portal:../../noir/packages/noirc_abi" @@ -1941,13 +1917,13 @@ __metadata: "@aztec/foundation": "workspace:^" "@aztec/key-store": "workspace:^" "@aztec/kv-store": "workspace:^" - "@aztec/merkle-tree": "workspace:^" "@aztec/noir-protocol-circuits-types": "workspace:^" "@aztec/noir-test-contracts.js": "workspace:^" "@aztec/noir-types": "workspace:*" "@aztec/protocol-contracts": "workspace:^" "@aztec/simulator": "workspace:^" "@aztec/stdlib": "workspace:^" + "@aztec/world-state": "workspace:^" "@jest/globals": "npm:^30.0.0" "@types/jest": "npm:^30.0.0" "@types/lodash.omit": "npm:^4.5.7" @@ -2005,7 +1981,6 @@ __metadata: "@aztec/foundation": "workspace:^" "@aztec/kv-store": "workspace:^" "@aztec/l1-artifacts": "workspace:^" - "@aztec/merkle-tree": "workspace:^" "@aztec/node-keystore": "workspace:^" "@aztec/noir-acvm_js": "portal:../../noir/packages/acvm_js" "@aztec/noir-contracts.js": "workspace:^" @@ -2049,7 +2024,6 @@ __metadata: "@aztec/constants": "workspace:^" "@aztec/foundation": "workspace:^" "@aztec/kv-store": "workspace:^" - "@aztec/merkle-tree": "workspace:^" "@aztec/native": "workspace:^" "@aztec/noir-acvm_js": "portal:../../noir/packages/acvm_js" "@aztec/noir-contracts.js": "workspace:^" @@ -2337,7 +2311,6 @@ __metadata: "@aztec/constants": "workspace:^" "@aztec/foundation": "workspace:^" "@aztec/kv-store": "workspace:^" - "@aztec/merkle-tree": "workspace:^" "@aztec/native": "workspace:^" "@aztec/protocol-contracts": "workspace:^" "@aztec/stdlib": "workspace:^" @@ -8066,15 +8039,6 @@ __metadata: languageName: node linkType: hard -"@types/sha256@npm:^0.2.0": - version: 0.2.2 - resolution: "@types/sha256@npm:0.2.2" - dependencies: - "@types/node": "npm:*" - checksum: 10/7701b9dc105e7b877090c9bb9b02e10953831737b599bfc7658635ae35d2b21927f77028f8090d50ea0281058ee975f190d664e5351c5aaf5535a1c26ba01f1f - languageName: node - linkType: hard - "@types/sinon@npm:^17.0.3": version: 17.0.3 resolution: "@types/sinon@npm:17.0.3" @@ -11182,13 +11146,6 @@ __metadata: languageName: node linkType: hard -"convert-hex@npm:~0.1.0": - version: 0.1.0 - resolution: "convert-hex@npm:0.1.0" - checksum: 10/eacb880dbc45a36a0e6b5f5674f7e57bdce59bbf5a3ebfba980f694e2be81f1b2c81c9c89834f8054f23cc9c21d1fd210265e2000287a1cd0426657797b2f462 - languageName: node - linkType: hard - "convert-source-map@npm:^1.5.1": version: 1.9.0 resolution: "convert-source-map@npm:1.9.0" @@ -11210,13 +11167,6 @@ __metadata: languageName: node linkType: hard -"convert-string@npm:~0.1.0": - version: 0.1.0 - resolution: "convert-string@npm:0.1.0" - checksum: 10/a1775cb186d2fbf175486f02e3f7cc68c75e7a0c7609bf434d2a933e801b3a0499ab57de4230919ec824351dc344055bf639a1db5e44a976787145817106d9aa - languageName: node - linkType: hard - "cookie-signature@npm:1.0.6": version: 1.0.6 resolution: "cookie-signature@npm:1.0.6" @@ -19908,16 +19858,6 @@ __metadata: languageName: node linkType: hard -"sha256@npm:^0.2.0": - version: 0.2.0 - resolution: "sha256@npm:0.2.0" - dependencies: - convert-hex: "npm:~0.1.0" - convert-string: "npm:~0.1.0" - checksum: 10/95017cec85533c3b12dd7bde04166e859aa8648b567c5c8c5a2e62f213cb03f3d017239fa8b7cc6fc9e741de57613e97a9741e756e7df31d56203095a1ea626d - languageName: node - linkType: hard - "sha3@npm:^2.1.4": version: 2.1.4 resolution: "sha3@npm:2.1.4"