diff --git a/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp b/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp index 0317fd4826be..65e3d30f7400 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp @@ -27,24 +27,79 @@ const size_t MAX_BATCH_SIZE = 64; template void add_values(TreeType& tree, const std::vector& values) { Signal signal(1); - typename TreeType::AddCompletionCallback completion = [&](const auto&) -> void { signal.signal_level(0); }; + bool success = true; + std::string error_message; + typename TreeType::AddCompletionCallback completion = [&](const auto& result) -> void { + success = result.success; + error_message = result.message; + signal.signal_level(0); + }; tree.add_or_update_values(values, completion); signal.wait_for_level(0); + if (!success) { + throw std::runtime_error(format("Failed to add values: ", error_message)); + } } template void add_values_with_witness(TreeType& tree, const std::vector& values) { + bool success = true; + std::string error_message; Signal signal(1); - typename TreeType::AddCompletionCallbackWithWitness completion = [&](const auto&) -> void { + typename TreeType::AddCompletionCallbackWithWitness completion = [&](const auto& result) -> void { + success = result.success; + error_message = result.message; signal.signal_level(0); }; tree.add_or_update_values(values, completion); signal.wait_for_level(0); + if (!success) { + throw std::runtime_error(format("Failed to add values with witness: ", error_message)); + } +} + +template void add_values_sequentially(TreeType& tree, const std::vector& values) +{ + bool success = true; + std::string error_message; + Signal signal(1); + typename TreeType::AddCompletionCallback completion = [&](const auto& result) -> void { + success = result.success; + error_message = result.message; + signal.signal_level(0); + }; + + tree.add_or_update_values_sequentially(values, completion); + signal.wait_for_level(0); + if (!success) { + throw std::runtime_error(format("Failed to add values sequentially: ", error_message)); + } +} + +template +void add_values_sequentially_with_witness(TreeType& tree, const std::vector& values) +{ + bool success = true; + std::string error_message; + Signal signal(1); + typename TreeType::AddSequentiallyCompletionCallbackWithWitness completion = [&](const auto& result) -> void { + success = result.success; + error_message = result.message; + signal.signal_level(0); + }; + + tree.add_or_update_values_sequentially(values, completion); + signal.wait_for_level(0); + if (!success) { + throw std::runtime_error(format("Failed to add values sequentially with witness: ", error_message)); + } } -template void multi_thread_indexed_tree_bench(State& state) noexcept +enum InsertionStrategy { SEQUENTIAL, BATCH }; + +template void multi_thread_indexed_tree_bench(State& state) noexcept { const size_t batch_size = size_t(state.range(0)); const size_t depth = TREE_DEPTH; @@ -61,10 +116,14 @@ template void multi_thread_indexed_tree_bench(State& state) const size_t initial_size = 1024 * 16; std::vector initial_batch(initial_size); - for (size_t i = 0; i < batch_size; ++i) { + for (size_t i = 0; i < initial_size; ++i) { initial_batch[i] = fr(random_engine.get_random_uint256()); } - add_values(tree, initial_batch); + if (strategy == SEQUENTIAL) { + add_values_sequentially(tree, initial_batch); + } else { + add_values(tree, initial_batch); + } for (auto _ : state) { state.PauseTiming(); @@ -73,11 +132,15 @@ template void multi_thread_indexed_tree_bench(State& state) values[i] = fr(random_engine.get_random_uint256()); } state.ResumeTiming(); - add_values(tree, values); + if (strategy == SEQUENTIAL) { + add_values_sequentially(tree, values); + } else { + add_values(tree, values); + } } } -template void single_thread_indexed_tree_bench(State& state) noexcept +template void single_thread_indexed_tree_bench(State& state) noexcept { const size_t batch_size = size_t(state.range(0)); const size_t depth = TREE_DEPTH; @@ -94,10 +157,14 @@ template void single_thread_indexed_tree_bench(State& state) const size_t initial_size = 1024 * 16; std::vector initial_batch(initial_size); - for (size_t i = 0; i < batch_size; ++i) { + for (size_t i = 0; i < initial_size; ++i) { initial_batch[i] = fr(random_engine.get_random_uint256()); } - add_values(tree, initial_batch); + if (strategy == SEQUENTIAL) { + add_values_sequentially(tree, initial_batch); + } else { + add_values(tree, initial_batch); + } for (auto _ : state) { state.PauseTiming(); @@ -106,11 +173,16 @@ template void single_thread_indexed_tree_bench(State& state) values[i] = fr(random_engine.get_random_uint256()); } state.ResumeTiming(); - add_values(tree, values); + if (strategy == SEQUENTIAL) { + add_values_sequentially(tree, values); + } else { + add_values(tree, values); + } } } -template void multi_thread_indexed_tree_with_witness_bench(State& state) noexcept +template +void multi_thread_indexed_tree_with_witness_bench(State& state) noexcept { const size_t batch_size = size_t(state.range(0)); const size_t depth = TREE_DEPTH; @@ -127,10 +199,14 @@ template void multi_thread_indexed_tree_with_witness_bench(S const size_t initial_size = 1024 * 16; std::vector initial_batch(initial_size); - for (size_t i = 0; i < batch_size; ++i) { + for (size_t i = 0; i < initial_size; ++i) { initial_batch[i] = fr(random_engine.get_random_uint256()); } - add_values(tree, initial_batch); + if (strategy == SEQUENTIAL) { + add_values_sequentially(tree, initial_batch); + } else { + add_values(tree, initial_batch); + } for (auto _ : state) { state.PauseTiming(); @@ -139,11 +215,16 @@ template void multi_thread_indexed_tree_with_witness_bench(S values[i] = fr(random_engine.get_random_uint256()); } state.ResumeTiming(); - add_values_with_witness(tree, values); + if (strategy == SEQUENTIAL) { + add_values_sequentially_with_witness(tree, values); + } else { + add_values_with_witness(tree, values); + } } } -template void single_thread_indexed_tree_with_witness_bench(State& state) noexcept +template +void single_thread_indexed_tree_with_witness_bench(State& state) noexcept { const size_t batch_size = size_t(state.range(0)); const size_t depth = TREE_DEPTH; @@ -160,10 +241,14 @@ template void single_thread_indexed_tree_with_witness_bench( const size_t initial_size = 1024 * 16; std::vector initial_batch(initial_size); - for (size_t i = 0; i < batch_size; ++i) { + for (size_t i = 0; i < initial_size; ++i) { initial_batch[i] = fr(random_engine.get_random_uint256()); } - add_values(tree, initial_batch); + if (strategy == SEQUENTIAL) { + add_values_sequentially(tree, initial_batch); + } else { + add_values(tree, initial_batch); + } for (auto _ : state) { state.PauseTiming(); @@ -172,53 +257,105 @@ template void single_thread_indexed_tree_with_witness_bench( values[i] = fr(random_engine.get_random_uint256()); } state.ResumeTiming(); - add_values_with_witness(tree, values); + if (strategy == SEQUENTIAL) { + add_values_sequentially_with_witness(tree, values); + } else { + add_values_with_witness(tree, values); + } } } -BENCHMARK(single_thread_indexed_tree_with_witness_bench) +BENCHMARK(single_thread_indexed_tree_with_witness_bench) ->Unit(benchmark::kMillisecond) ->RangeMultiplier(2) ->Range(2, MAX_BATCH_SIZE) ->Iterations(1000); -BENCHMARK(single_thread_indexed_tree_with_witness_bench) +BENCHMARK(single_thread_indexed_tree_with_witness_bench) ->Unit(benchmark::kMillisecond) ->RangeMultiplier(2) ->Range(512, 8192) ->Iterations(10); -BENCHMARK(multi_thread_indexed_tree_with_witness_bench) +BENCHMARK(single_thread_indexed_tree_with_witness_bench) ->Unit(benchmark::kMillisecond) ->RangeMultiplier(2) ->Range(2, MAX_BATCH_SIZE) ->Iterations(1000); -BENCHMARK(multi_thread_indexed_tree_with_witness_bench) +BENCHMARK(single_thread_indexed_tree_with_witness_bench) ->Unit(benchmark::kMillisecond) ->RangeMultiplier(2) ->Range(512, 8192) ->Iterations(10); -BENCHMARK(single_thread_indexed_tree_bench) +BENCHMARK(multi_thread_indexed_tree_with_witness_bench) ->Unit(benchmark::kMillisecond) ->RangeMultiplier(2) ->Range(2, MAX_BATCH_SIZE) ->Iterations(1000); -BENCHMARK(single_thread_indexed_tree_bench) +BENCHMARK(multi_thread_indexed_tree_with_witness_bench) ->Unit(benchmark::kMillisecond) ->RangeMultiplier(2) ->Range(512, 8192) ->Iterations(10); -BENCHMARK(multi_thread_indexed_tree_bench) +BENCHMARK(multi_thread_indexed_tree_with_witness_bench) + ->Unit(benchmark::kMillisecond) + ->RangeMultiplier(2) + ->Range(2, MAX_BATCH_SIZE) + ->Iterations(1000); + +BENCHMARK(multi_thread_indexed_tree_with_witness_bench) + ->Unit(benchmark::kMillisecond) + ->RangeMultiplier(2) + ->Range(512, 8192) + ->Iterations(10); + +BENCHMARK(single_thread_indexed_tree_bench) + ->Unit(benchmark::kMillisecond) + ->RangeMultiplier(2) + ->Range(2, MAX_BATCH_SIZE) + ->Iterations(1000); + +BENCHMARK(single_thread_indexed_tree_bench) + ->Unit(benchmark::kMillisecond) + ->RangeMultiplier(2) + ->Range(512, 8192) + ->Iterations(10); + +BENCHMARK(single_thread_indexed_tree_bench) + ->Unit(benchmark::kMillisecond) + ->RangeMultiplier(2) + ->Range(2, MAX_BATCH_SIZE) + ->Iterations(1000); + +BENCHMARK(single_thread_indexed_tree_bench) + ->Unit(benchmark::kMillisecond) + ->RangeMultiplier(2) + ->Range(512, 8192) + ->Iterations(10); + +BENCHMARK(multi_thread_indexed_tree_bench) + ->Unit(benchmark::kMillisecond) + ->RangeMultiplier(2) + ->Range(2, MAX_BATCH_SIZE) + ->Iterations(1000); + +BENCHMARK(multi_thread_indexed_tree_bench) + ->Unit(benchmark::kMillisecond) + ->RangeMultiplier(2) + ->Range(512, 8192) + ->Iterations(100); + +BENCHMARK(multi_thread_indexed_tree_bench) ->Unit(benchmark::kMillisecond) ->RangeMultiplier(2) ->Range(2, MAX_BATCH_SIZE) ->Iterations(1000); -BENCHMARK(multi_thread_indexed_tree_bench) +BENCHMARK(multi_thread_indexed_tree_bench) ->Unit(benchmark::kMillisecond) ->RangeMultiplier(2) ->Range(512, 8192) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp index 4a303ccc362e..f1ceef8eebb2 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp @@ -489,6 +489,16 @@ std::optional ContentAddressedAppendOnlyTree::find_lea if (!child.has_value()) { // std::cout << "No child" << std::endl; + // We still need to update the cache with the sibling. The fact that under us there is an empty subtree + // doesn't mean that same is happening with our sibling. + if (updateNodesByIndexCache) { + child_index_at_level = is_right ? (child_index_at_level * 2) + 1 : (child_index_at_level * 2); + std::optional sibling = is_right ? nodePayload.left : nodePayload.right; + index_t sibling_index_at_level = is_right ? child_index_at_level - 1 : child_index_at_level + 1; + if (sibling.has_value()) { + store_->put_cached_node_by_index(i + 1, sibling_index_at_level, sibling.value(), false); + } + } return std::nullopt; } // std::cout << "Found child " << child.value() << std::endl; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp index 8aa2412208af..197f784d59fc 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp @@ -49,7 +49,10 @@ class ContentAddressedIndexedTree : public ContentAddressedAppendOnlyTree>&)>; + using AddSequentiallyCompletionCallbackWithWitness = + std::function>&)>; using AddCompletionCallback = std::function&)>; + using LeafCallback = std::function>&)>; using FindLowLeafCallback = std::function&)>; @@ -63,12 +66,12 @@ class ContentAddressedIndexedTree : public ContentAddressedAppendOnlyTree& values, const AddCompletionCallback& completion); /** - * @brief Adds or updates the given set of values in the tree (updates not currently supported) + * @brief Adds or updates the given set of values in the tree using subtree insertion. * @param values The values to be added or updated * @param subtree_depth The height of the subtree to be inserted. * @param completion The callback to be triggered once the values have been added @@ -107,6 +110,22 @@ class ContentAddressedIndexedTree : public ContentAddressedAppendOnlyTree& values, + const AddSequentiallyCompletionCallbackWithWitness& completion); + + /** + * @brief Adds or updates the given set of values in the tree one by one + * @param values The values to be added or updated + * @param completion The callback to be triggered once the values have been added + */ + void add_or_update_values_sequentially(const std::vector& values, + const AddCompletionCallback& completion); + void get_leaf(const index_t& index, bool includeUncommitted, const LeafCallback& completion) const; /** @@ -182,9 +201,9 @@ class ContentAddressedIndexedTree : public ContentAddressedAppendOnlyTree sparse_batch_update(const index_t& start_index, const index_t& num_leaves_to_be_inserted, const uint32_t& root_level, - const std::vector& insertions); + const std::vector& updates); void sparse_batch_update(const std::vector>& hashes_at_level, uint32_t level); @@ -212,9 +231,19 @@ class ContentAddressedIndexedTree : public ContentAddressedAppendOnlyTree& values, + const AddSequentiallyCompletionCallbackWithWitness& completion, + bool capture_witness); + struct InsertionGenerationResponse { - std::shared_ptr> insertions; - std::shared_ptr> indexed_leaves; + std::shared_ptr> low_leaf_updates; + std::shared_ptr> leaves_to_append; index_t highest_index; }; @@ -222,17 +251,33 @@ class ContentAddressedIndexedTree : public ContentAddressedAppendOnlyTree>>& values_to_be_sorted, const InsertionGenerationCallback& completion); - struct InsertionCompletionResponse { - std::shared_ptr>> low_leaf_witness_data; + struct InsertionUpdates { + LeafUpdate low_leaf_update; + IndexedLeafValueType new_leaf; + index_t new_leaf_index; + }; + + struct SequentialInsertionGenerationResponse { + std::shared_ptr> updates_to_perform; + index_t highest_index; + }; + + using SequentialInsertionGenerationCallback = + std::function&)>; + void generate_sequential_insertions(const std::vector& values, + const SequentialInsertionGenerationCallback& completion); + + struct UpdatesCompletionResponse { + std::shared_ptr>> update_witnesses; }; - using InsertionCompletionCallback = std::function&)>; - void perform_insertions(size_t total_leaves, - std::shared_ptr> insertions, - const InsertionCompletionCallback& completion); - void perform_insertions_without_witness(const index_t& highest_index, - std::shared_ptr> insertions, - const InsertionCompletionCallback& completion); + using UpdatesCompletionCallback = std::function&)>; + void perform_updates(size_t total_leaves, + std::shared_ptr> updates, + const UpdatesCompletionCallback& completion); + void perform_updates_without_witness(const index_t& highest_index, + std::shared_ptr> updates, + const UpdatesCompletionCallback& completion); struct HashGenerationResponse { std::shared_ptr> hashes; @@ -618,7 +663,7 @@ void ContentAddressedIndexedTree::add_or_update_values_int // new hashes that will be appended to the tree std::shared_ptr> hashes_to_append; // info about the low leaves that have been updated - std::shared_ptr>> low_leaf_witness_data; + std::shared_ptr>> low_leaf_witness_data; fr_sibling_path subtree_path; std::atomic count; Status status; @@ -700,12 +745,12 @@ void ContentAddressedIndexedTree::add_or_update_values_int // This signals the completion of the low leaf updates // If the append hash generation has also copleted then the hashes can be appended - InsertionCompletionCallback insertion_completion = - [=, this](const TypedResponse& insertion_response) { - if (!insertion_response.success) { - results->status.set_failure(insertion_response.message); + UpdatesCompletionCallback updates_completion = + [=, this](const TypedResponse& updates_response) { + if (!updates_response.success) { + results->status.set_failure(updates_response.message); } else if (capture_witness) { - results->low_leaf_witness_data = insertion_response.inner.low_leaf_witness_data; + results->low_leaf_witness_data = updates_response.inner.update_witnesses; } if (results->count.fetch_sub(1) == 1) { @@ -726,7 +771,7 @@ void ContentAddressedIndexedTree::add_or_update_values_int }; // This signals the completion of the insertion data generation - // Here we will enqueue both the generation of the appended hashes and the low leaf updates (insertions) + // Here we will enqueue both the generation of the appended hashes and the low leaf updates InsertionGenerationCallback insertion_generation_completed = [=, this](const TypedResponse& insertion_response) { if (!insertion_response.success) { @@ -734,35 +779,36 @@ void ContentAddressedIndexedTree::add_or_update_values_int return; } workers_->enqueue([=, this]() { - generate_hashes_for_appending(insertion_response.inner.indexed_leaves, hash_completion); + generate_hashes_for_appending(insertion_response.inner.leaves_to_append, hash_completion); }); if (capture_witness) { - perform_insertions(values.size(), insertion_response.inner.insertions, insertion_completion); + perform_updates(values.size(), insertion_response.inner.low_leaf_updates, updates_completion); return; } - perform_insertions_without_witness( - insertion_response.inner.highest_index, insertion_response.inner.insertions, insertion_completion); + perform_updates_without_witness( + insertion_response.inner.highest_index, insertion_response.inner.low_leaf_updates, updates_completion); }; // We start by enqueueing the insertion data generation workers_->enqueue([=, this]() { generate_insertions(values_to_be_sorted, insertion_generation_completed); }); } +// Performs a number of leaf updates in the tree, fetching witnesses for the updates in the order they've been applied, +// with the caveat that all nodes fetched need to be in the cache. Otherwise, they'll be assumed to be empty, +// potentially erasing part of the tree. This function won't fetch nodes from DB. template -void ContentAddressedIndexedTree::perform_insertions( - size_t total_leaves, - std::shared_ptr> insertions, - const InsertionCompletionCallback& completion) +void ContentAddressedIndexedTree::perform_updates( + size_t total_leaves, std::shared_ptr> updates, const UpdatesCompletionCallback& completion) { - auto low_leaf_witness_data = std::make_shared>>( + auto update_witnesses = std::make_shared>>( total_leaves, - LowLeafWitnessData{ IndexedLeafValueType::empty(), 0, fr_sibling_path(depth_, fr::zero()) }); + LeafUpdateWitnessData{ IndexedLeafValueType::empty(), 0, fr_sibling_path(depth_, fr::zero()) }); - // early return, no insertions to perform - if (insertions->size() == 0) { - TypedResponse response; + // early return, no updates to perform + if (updates->size() == 0) { + TypedResponse response; response.success = true; - response.inner.low_leaf_witness_data = low_leaf_witness_data; + response.inner.update_witnesses = update_witnesses; completion(response); return; } @@ -776,7 +822,7 @@ void ContentAddressedIndexedTree::perform_insertions( // The first signal is set to 0. This ensures the first worker up the tree is not impeded signals->emplace_back(0); // Workers will follow their leaders up the tree, being triggered by the signal in front of them - for (size_t i = 0; i < insertions->size(); ++i) { + for (size_t i = 0; i < updates->size(); ++i) { signals->emplace_back(uint32_t(1 + depth_)); } @@ -812,19 +858,19 @@ void ContentAddressedIndexedTree::perform_insertions( std::shared_ptr enqueuedOperations = std::make_shared(); - for (uint32_t i = 0; i < insertions->size(); ++i) { + for (uint32_t i = 0; i < updates->size(); ++i) { std::function op = [=, this]() { - LeafInsertion& insertion = (*insertions)[i]; + LeafUpdate& update = (*updates)[i]; Signal& leaderSignal = (*signals)[i]; Signal& followerSignal = (*signals)[i + 1]; try { - auto& current_witness_data = low_leaf_witness_data->at(i); - current_witness_data.leaf = insertion.original_low_leaf; - current_witness_data.index = insertion.low_leaf_index; + auto& current_witness_data = update_witnesses->at(i); + current_witness_data.leaf = update.original_leaf; + current_witness_data.index = update.leaf_index; current_witness_data.path.clear(); - update_leaf_and_hash_to_root(insertion.low_leaf_index, - insertion.low_leaf, + update_leaf_and_hash_to_root(update.leaf_index, + update.updated_leaf, leaderSignal, followerSignal, current_witness_data.path); @@ -839,12 +885,12 @@ void ContentAddressedIndexedTree::perform_insertions( enqueuedOperations->enqueue_next(*workers_); } - if (i == insertions->size() - 1) { - TypedResponse response; + if (i == updates->size() - 1) { + TypedResponse response; response.success = status->success; response.message = status->message; if (response.success) { - response.inner.low_leaf_witness_data = low_leaf_witness_data; + response.inner.update_witnesses = update_witnesses; } completion(response); } @@ -861,15 +907,18 @@ void ContentAddressedIndexedTree::perform_insertions( } } +// Performs a number of leaf updates in the tree, with the caveat that all nodes fetched need to be in the cache +// Otherwise, they'll be assumed to be empty, potentially erasing part of the tree. This function won't fetch nodes from +// DB. template -void ContentAddressedIndexedTree::perform_insertions_without_witness( +void ContentAddressedIndexedTree::perform_updates_without_witness( const index_t& highest_index, - std::shared_ptr> insertions, - const InsertionCompletionCallback& completion) + std::shared_ptr> updates, + const UpdatesCompletionCallback& completion) { - // early return, no insertions to perform - if (insertions->size() == 0) { - TypedResponse response; + // early return, no updates to perform + if (updates->size() == 0) { + TypedResponse response; response.success = true; completion(response); return; @@ -913,7 +962,7 @@ void ContentAddressedIndexedTree::perform_insertions_witho try { bool withinRange = startIndex <= highest_index; if (withinRange) { - opCount->roots[i] = sparse_batch_update(startIndex, batchSize, rootLevel, *insertions); + opCount->roots[i] = sparse_batch_update(startIndex, batchSize, rootLevel, *updates); } } catch (std::exception& e) { status->set_failure(e.what()); @@ -928,7 +977,7 @@ void ContentAddressedIndexedTree::perform_insertions_witho } sparse_batch_update(hashes_at_level, rootLevel); - TypedResponse response; + TypedResponse response; response.success = true; completion(response); } @@ -948,7 +997,7 @@ void ContentAddressedIndexedTree::generate_hashes_for_appe std::vector& leaves = *leaves_to_hash; for (uint32_t i = 0; i < leaves.size(); ++i) { IndexedLeafValueType& leaf = leaves[i]; - fr hash = leaf.is_empty() ? 0 : HashingPolicy::hash(leaf.get_hash_inputs()); + fr hash = leaf.is_empty() ? fr::zero() : HashingPolicy::hash(leaf.get_hash_inputs()); (*response.inner.hashes)[i] = hash; store_->put_leaf_by_hash(hash, leaf); } @@ -980,11 +1029,11 @@ void ContentAddressedIndexedTree::generate_insertions( // std::cout << "Generating insertions " << std::endl; // Now that we have the sorted values we need to identify the leaves that need updating. - // This is performed sequentially and is stored in this 'leaf_insertion' struct + // This is performed sequentially and is stored in this 'leaf_update' struct response.inner.highest_index = 0; - response.inner.insertions = std::make_shared>(); - response.inner.insertions->reserve(values.size()); - response.inner.indexed_leaves = + response.inner.low_leaf_updates = std::make_shared>(); + response.inner.low_leaf_updates->reserve(values.size()); + response.inner.leaves_to_append = std::make_shared>(values.size(), IndexedLeafValueType::empty()); index_t num_leaves_to_be_inserted = values.size(); std::set unique_values; @@ -1065,10 +1114,10 @@ void ContentAddressedIndexedTree::generate_insertions( low_leaf = low_leaf_option.value(); } - LeafInsertion insertion = { - .low_leaf_index = low_leaf_index, - .low_leaf = IndexedLeafValueType::empty(), - .original_low_leaf = low_leaf, + LeafUpdate low_update = { + .leaf_index = low_leaf_index, + .updated_leaf = IndexedLeafValueType::empty(), + .original_leaf = low_leaf, }; // Capture the index and original value of the 'low' leaf @@ -1090,10 +1139,10 @@ void ContentAddressedIndexedTree::generate_insertions( store_->put_cached_leaf_by_index(low_leaf_index, low_leaf); // leaves_pre[low_leaf_index] = low_leaf; - insertion.low_leaf = low_leaf; + low_update.updated_leaf = low_leaf; // Update the set of leaves to append - (*response.inner.indexed_leaves)[index_into_appended_leaves] = new_leaf; + (*response.inner.leaves_to_append)[index_into_appended_leaves] = new_leaf; } else if (IndexedLeafValueType::is_updateable()) { // Update the current leaf's value, don't change it's link IndexedLeafValueType replacement_leaf = @@ -1104,7 +1153,7 @@ void ContentAddressedIndexedTree::generate_insertions( // << index_of_new_leaf << std::endl; // store_->set_leaf_key_at_index(index_of_new_leaf, empty_leaf); store_->put_cached_leaf_by_index(low_leaf_index, replacement_leaf); - insertion.low_leaf = replacement_leaf; + low_update.updated_leaf = replacement_leaf; // The set of appended leaves already has an empty leaf in the slot at index // 'index_into_appended_leaves' } else { @@ -1112,11 +1161,13 @@ void ContentAddressedIndexedTree::generate_insertions( meta.name, " leaf type ", IndexedLeafValueType::name(), - " is not updateable")); + " is not updateable and ", + value_pair.first.get_key(), + " is already present")); } response.inner.highest_index = std::max(response.inner.highest_index, low_leaf_index); - response.inner.insertions->push_back(insertion); + response.inner.low_leaf_updates->push_back(low_update); } } }, @@ -1144,7 +1195,7 @@ void ContentAddressedIndexedTree::update_leaf_and_hash_to_ // 3. Write the new node value index_t index = leaf_index; uint32_t level = depth_; - fr new_hash = HashingPolicy::hash(leaf.get_hash_inputs()); + fr new_hash = leaf.value.is_empty() ? fr::zero() : HashingPolicy::hash(leaf.get_hash_inputs()); // Wait until we see that our leader has cleared 'depth_ - 1' (i.e. the level above the leaves that we are about // to write into) this ensures that our leader is not still reading the leaves @@ -1257,7 +1308,7 @@ std::pair ContentAddressedIndexedTree::sparse_ba const index_t& start_index, const index_t& num_leaves_to_be_inserted, const uint32_t& root_level, - const std::vector& insertions) + const std::vector& updates) { auto get_optional_node = [&](uint32_t level, index_t index) -> std::optional { fr value = fr::zero(); @@ -1269,7 +1320,7 @@ std::pair ContentAddressedIndexedTree::sparse_ba uint32_t level = depth_; std::vector indices; - indices.reserve(insertions.size()); + indices.reserve(updates.size()); fr new_hash = fr::zero(); @@ -1277,28 +1328,29 @@ std::pair ContentAddressedIndexedTree::sparse_ba std::unordered_map hashes; index_t end_index = start_index + num_leaves_to_be_inserted; // Insert the leaves - for (size_t i = 0; i < insertions.size(); ++i) { + for (size_t i = 0; i < updates.size(); ++i) { - const LeafInsertion& insertion = insertions[i]; - if (insertion.low_leaf_index < start_index || insertion.low_leaf_index >= end_index) { + const LeafUpdate& update = updates[i]; + if (update.leaf_index < start_index || update.leaf_index >= end_index) { continue; } // one of our leaves - new_hash = HashingPolicy::hash(insertion.low_leaf.get_hash_inputs()); + new_hash = update.updated_leaf.value.is_empty() ? fr::zero() + : HashingPolicy::hash(update.updated_leaf.get_hash_inputs()); - // std::cout << "Hashing leaf at level " << level << " index " << insertion.low_leaf_index << " batch start " + // std::cout << "Hashing leaf at level " << level << " index " << update.leaf_index << " batch start " // << start_index << " hash " << leaf_hash << std::endl; // Write the new leaf hash in place - store_->put_cached_node_by_index(level, insertion.low_leaf_index, new_hash); + store_->put_cached_node_by_index(level, update.leaf_index, new_hash); // std::cout << "Writing leaf hash: " << new_hash << " at index " << index << std::endl; - store_->put_leaf_by_hash(new_hash, insertion.low_leaf); + store_->put_leaf_by_hash(new_hash, update.updated_leaf); // std::cout << "Writing level: " << level << std::endl; store_->put_node_by_hash(new_hash, { .left = std::nullopt, .right = std::nullopt, .ref = 1 }); - indices.push_back(insertion.low_leaf_index); - hashes[insertion.low_leaf_index] = new_hash; - // std::cout << "Leaf " << new_hash << " at index " << insertion.low_leaf_index << std::endl; + indices.push_back(update.leaf_index); + hashes[update.leaf_index] = new_hash; + // std::cout << "Leaf " << new_hash << " at index " << update.leaf_index << std::endl; } if (indices.empty()) { @@ -1339,4 +1391,250 @@ std::pair ContentAddressedIndexedTree::sparse_ba return std::make_pair(true, new_hash); } +template +void ContentAddressedIndexedTree::add_or_update_values_sequentially( + const std::vector& values, const AddSequentiallyCompletionCallbackWithWitness& completion) +{ + add_or_update_values_sequentially_internal(values, completion, true); +} + +template +void ContentAddressedIndexedTree::add_or_update_values_sequentially( + const std::vector& values, const AddCompletionCallback& completion) +{ + auto final_completion = + [=](const TypedResponse>& add_data_response) { + TypedResponse response; + response.success = add_data_response.success; + response.message = add_data_response.message; + if (add_data_response.success) { + response.inner = add_data_response.inner.add_data_result; + } + // Trigger the client's provided callback + completion(response); + }; + add_or_update_values_sequentially_internal(values, final_completion, false); +} + +template +void ContentAddressedIndexedTree::add_or_update_values_sequentially_internal( + const std::vector& values, + const AddSequentiallyCompletionCallbackWithWitness& completion, + bool capture_witness) +{ + auto on_error = [=](const std::string& message) { + try { + TypedResponse> response; + response.success = false; + response.message = message; + completion(response); + } catch (std::exception&) { + } + }; + + // This is the final callback triggered once all the leaves have been inserted in the tree + auto final_completion = [=, this](const TypedResponse& updates_completion_response) { + TypedResponse> response; + response.success = updates_completion_response.success; + response.message = updates_completion_response.message; + if (updates_completion_response.success) { + { + TreeMeta meta; + ReadTransactionPtr tx = store_->create_read_transaction(); + store_->get_meta(meta, *tx, true); + + index_t new_total_size = values.size() + meta.size; + meta.size = new_total_size; + meta.root = store_->get_current_root(*tx, true); + + store_->put_meta(meta); + } + + if (capture_witness) { + // Split results->update_witnesses between low_leaf_witness_data and insertion_witness_data + // Currently we always insert an empty leaf, even if it's an update, so we can split based + // on the index of the witness data + response.inner.insertion_witness_data = + std::make_shared>>(); + ; + response.inner.low_leaf_witness_data = + std::make_shared>>(); + ; + + for (size_t i = 0; i < updates_completion_response.inner.update_witnesses->size(); ++i) { + LeafUpdateWitnessData& witness_data = + updates_completion_response.inner.update_witnesses->at(i); + // If even, it's a low leaf, if odd, it's an insertion witness + if (i % 2 == 0) { + response.inner.low_leaf_witness_data->push_back(witness_data); + } else { + response.inner.insertion_witness_data->push_back(witness_data); + } + } + } + } + // Trigger the client's provided callback + completion(response); + }; + + // This signals the completion of the insertion data generation + // Here we'll perform all updates to the tree + SequentialInsertionGenerationCallback insertion_generation_completed = + [=, this](const TypedResponse& insertion_response) { + if (!insertion_response.success) { + on_error(insertion_response.message); + return; + } + + std::shared_ptr> flat_updates = std::make_shared>(); + for (size_t i = 0; i < insertion_response.inner.updates_to_perform->size(); ++i) { + InsertionUpdates& insertion_update = insertion_response.inner.updates_to_perform->at(i); + flat_updates->push_back(insertion_update.low_leaf_update); + flat_updates->push_back(LeafUpdate{ + .leaf_index = insertion_update.new_leaf_index, + .updated_leaf = insertion_update.new_leaf, + .original_leaf = IndexedLeafValueType::empty(), + }); + } + + if (capture_witness) { + perform_updates(flat_updates->size(), flat_updates, final_completion); + return; + } + perform_updates_without_witness(insertion_response.inner.highest_index, flat_updates, final_completion); + }; + + // Enqueue the insertion data generation + workers_->enqueue([=, this]() { generate_sequential_insertions(values, insertion_generation_completed); }); +} + +template +void ContentAddressedIndexedTree::generate_sequential_insertions( + const std::vector& values, const SequentialInsertionGenerationCallback& completion) +{ + execute_and_report( + [=, this](TypedResponse& response) { + response.inner.highest_index = 0; + response.inner.updates_to_perform = std::make_shared>(); + + TreeMeta meta; + ReadTransactionPtr tx = store_->create_read_transaction(); + store_->get_meta(meta, *tx, true); + + RequestContext requestContext; + requestContext.includeUncommitted = true; + // Ensure that the tree is not going to be overfilled + index_t new_total_size = values.size() + meta.size; + if (new_total_size > max_size_) { + throw std::runtime_error(format("Unable to insert values into tree ", + meta.name, + " new size: ", + new_total_size, + " max size: ", + max_size_)); + } + // The highest index touched will be the last leaf index, since we append a zero for updates + response.inner.highest_index = new_total_size - 1; + requestContext.root = store_->get_current_root(*tx, true); + // Fetch the frontier (non empty nodes to the right) of the tree. This will ensure that perform_updates or + // perform_updates_without_witness has all the cached nodes it needs to perform the insertions. See comment + // above those functions. + if (meta.size > 0) { + find_leaf_hash(meta.size - 1, requestContext, *tx, true); + } + + for (size_t i = 0; i < values.size(); ++i) { + const LeafValueType& new_payload = values[i]; + if (new_payload.is_empty()) { + continue; + } + index_t index_of_new_leaf = i + meta.size; + fr value = new_payload.get_key(); + + // This gives us the leaf that need updating + index_t low_leaf_index = 0; + bool is_already_present = false; + + std::tie(is_already_present, low_leaf_index) = + store_->find_low_value(new_payload.get_key(), requestContext, *tx); + + // Try and retrieve the leaf pre-image from the cache first. + // If unsuccessful, derive from the tree and hash based lookup + std::optional optional_low_leaf = + store_->get_cached_leaf_by_index(low_leaf_index); + IndexedLeafValueType low_leaf; + + if (optional_low_leaf.has_value()) { + low_leaf = optional_low_leaf.value(); + } else { + std::optional low_leaf_hash = find_leaf_hash(low_leaf_index, requestContext, *tx, true); + + if (!low_leaf_hash.has_value()) { + throw std::runtime_error(format("Unable to insert values into tree ", + meta.name, + " failed to find low leaf at index ", + low_leaf_index)); + } + + std::optional low_leaf_option = + store_->get_leaf_by_hash(low_leaf_hash.value(), *tx, true); + + if (!low_leaf_option.has_value()) { + throw std::runtime_error(format("Unable to insert values into tree ", + meta.name, + " failed to get leaf pre-image by hash for index ", + low_leaf_index)); + } + low_leaf = low_leaf_option.value(); + }; + + InsertionUpdates insertion_update = { + .low_leaf_update = + LeafUpdate{ + .leaf_index = low_leaf_index, + .updated_leaf = IndexedLeafValueType::empty(), + .original_leaf = low_leaf, + }, + .new_leaf = IndexedLeafValueType::empty(), + .new_leaf_index = index_of_new_leaf, + }; + + if (!is_already_present) { + // Update the current leaf to point it to the new leaf + IndexedLeafValueType new_leaf = + IndexedLeafValueType(new_payload, low_leaf.nextIndex, low_leaf.nextValue); + + low_leaf.nextIndex = index_of_new_leaf; + low_leaf.nextValue = value; + // Cache the new leaf + store_->set_leaf_key_at_index(index_of_new_leaf, new_leaf); + store_->put_cached_leaf_by_index(index_of_new_leaf, new_leaf); + // Update cached low leaf + store_->put_cached_leaf_by_index(low_leaf_index, low_leaf); + + insertion_update.low_leaf_update.updated_leaf = low_leaf; + insertion_update.new_leaf = new_leaf; + } else if (IndexedLeafValueType::is_updateable()) { + // Update the current leaf's value, don't change it's link + IndexedLeafValueType replacement_leaf = + IndexedLeafValueType(new_payload, low_leaf.nextIndex, low_leaf.nextValue); + + store_->put_cached_leaf_by_index(low_leaf_index, replacement_leaf); + insertion_update.low_leaf_update.updated_leaf = replacement_leaf; + } else { + throw std::runtime_error(format("Unable to insert values into tree ", + meta.name, + " leaf type ", + IndexedLeafValueType::name(), + " is not updateable and ", + new_payload.get_key(), + " is already present")); + } + + response.inner.updates_to_perform->push_back(insertion_update); + } + }, + completion); +} + } // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp index 52a2296a086c..5177b3708333 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp @@ -33,6 +33,7 @@ using Store = ContentAddressedCachedTreeStore; using TreeType = ContentAddressedIndexedTree; using CompletionCallback = TreeType::AddCompletionCallbackWithWitness; +using SequentialCompletionCallback = TreeType::AddSequentiallyCompletionCallbackWithWitness; class PersistedContentAddressedIndexedTreeTest : public testing::Test { protected: @@ -370,6 +371,19 @@ void add_values(TypeOfTree& tree, const std::vector& values, bool signal.wait_for_level(); } +template +void add_values_sequentially(TypeOfTree& tree, const std::vector& values, bool expectedSuccess = true) +{ + Signal signal; + auto completion = [&](const TypedResponse>& response) -> void { + EXPECT_EQ(response.success, expectedSuccess); + signal.signal_level(); + }; + + tree.add_or_update_values_sequentially(values, completion); + signal.wait_for_level(); +} + template void block_sync_values(TypeOfTree& tree, const std::vector& values, bool expectedSuccess = true) { @@ -383,6 +397,21 @@ void block_sync_values(TypeOfTree& tree, const std::vector& value signal.wait_for_level(); } +template +void block_sync_values_sequential(TypeOfTree& tree, + const std::vector& values, + bool expectedSuccess = true) +{ + Signal signal; + auto completion = [&](const TypedResponse& response) -> void { + EXPECT_EQ(response.success, expectedSuccess); + signal.signal_level(); + }; + + tree.add_or_update_values_sequentially(values, completion); + signal.wait_for_level(); +} + template void remove_historic_block(TypeOfTree& tree, const index_t& blockNumber, bool expected_success = true) { @@ -717,8 +746,8 @@ void test_batch_insert(uint32_t batchSize, std::string directory, uint64_t mapSi fr_sibling_path path = memdb.update_element(batch[j].value); memory_tree_sibling_paths.push_back(path); } - std::shared_ptr>> tree1_low_leaf_witness_data; - std::shared_ptr>> tree2_low_leaf_witness_data; + std::shared_ptr>> tree1_low_leaf_witness_data; + std::shared_ptr>> tree2_low_leaf_witness_data; { Signal signal; CompletionCallback completion = @@ -804,8 +833,8 @@ void test_batch_insert_with_commit_restore(uint32_t batchSize, fr_sibling_path path = memdb.update_element(batch[j].value); memory_tree_sibling_paths.push_back(path); } - std::shared_ptr>> tree1_low_leaf_witness_data; - std::shared_ptr>> tree2_low_leaf_witness_data; + std::shared_ptr>> tree1_low_leaf_witness_data; + std::shared_ptr>> tree2_low_leaf_witness_data; { Signal signal; CompletionCallback completion = @@ -955,6 +984,149 @@ TEST_F(PersistedContentAddressedIndexedTreeTest, reports_an_error_if_batch_conta signal.wait_for_level(); } +void test_sequential_insert_vs_batch(uint32_t batchSize, std::string directory, uint64_t mapSize, uint64_t maxReaders) +{ + auto& random_engine = numeric::get_randomness(); + const uint32_t batch_size = batchSize; + const uint32_t num_batches = 16; + uint32_t depth = 10; + ThreadPoolPtr workers = make_thread_pool(1); + ThreadPoolPtr multi_workers = make_thread_pool(8); + NullifierMemoryTree memdb(depth, batch_size); + + auto sequential_tree_1 = create_tree(directory, mapSize, maxReaders, depth, batch_size, workers); + auto sequential_tree_2 = create_tree(directory, mapSize, maxReaders, depth, batch_size, multi_workers); + auto sequential_tree_3 = create_tree(directory, mapSize, maxReaders, depth, batch_size, multi_workers); + auto batch_tree = create_tree(directory, mapSize, maxReaders, depth, batch_size, multi_workers); + + for (uint32_t i = 0; i < num_batches; i++) { + + check_root(*sequential_tree_1, memdb.root()); + check_root(*sequential_tree_2, memdb.root()); + check_root(*sequential_tree_3, memdb.root()); + check_root(*batch_tree, memdb.root()); + check_sibling_path(*sequential_tree_1, 0, memdb.get_sibling_path(0)); + check_sibling_path(*sequential_tree_2, 0, memdb.get_sibling_path(0)); + check_sibling_path(*sequential_tree_3, 0, memdb.get_sibling_path(0)); + check_sibling_path(*batch_tree, 0, memdb.get_sibling_path(0)); + + check_sibling_path(*sequential_tree_1, 512, memdb.get_sibling_path(512)); + check_sibling_path(*sequential_tree_2, 512, memdb.get_sibling_path(512)); + check_sibling_path(*sequential_tree_3, 512, memdb.get_sibling_path(512)); + check_sibling_path(*batch_tree, 512, memdb.get_sibling_path(512)); + + std::vector batch; + std::vector memory_tree_sibling_paths; + for (uint32_t j = 0; j < batch_size; j++) { + batch.emplace_back(random_engine.get_random_uint256()); + fr_sibling_path path = memdb.update_element(batch[j].value); + memory_tree_sibling_paths.push_back(path); + } + std::shared_ptr>> sequential_tree_1_low_leaf_witness_data; + std::shared_ptr>> + sequential_tree_1_insertion_witness_data; + std::shared_ptr>> sequential_tree_2_low_leaf_witness_data; + std::shared_ptr>> + sequential_tree_2_insertion_witness_data; + + { + Signal signal; + SequentialCompletionCallback completion = + [&](const TypedResponse>& response) { + sequential_tree_1_low_leaf_witness_data = response.inner.low_leaf_witness_data; + sequential_tree_1_insertion_witness_data = response.inner.insertion_witness_data; + signal.signal_level(); + }; + sequential_tree_1->add_or_update_values_sequentially(batch, completion); + signal.wait_for_level(); + } + + { + Signal signal; + SequentialCompletionCallback completion = + [&](const TypedResponse>& response) { + sequential_tree_2_low_leaf_witness_data = response.inner.low_leaf_witness_data; + sequential_tree_2_insertion_witness_data = response.inner.insertion_witness_data; + signal.signal_level(); + }; + sequential_tree_2->add_or_update_values_sequentially(batch, completion); + signal.wait_for_level(); + } + + { + Signal signal; + auto completion = [&](const TypedResponse&) { signal.signal_level(); }; + sequential_tree_3->add_or_update_values_sequentially(batch, completion); + signal.wait_for_level(); + } + + { + Signal signal; + auto completion = [&](const TypedResponse&) { signal.signal_level(); }; + batch_tree->add_or_update_values(batch, completion); + signal.wait_for_level(); + } + check_root(*sequential_tree_1, memdb.root()); + check_root(*sequential_tree_2, memdb.root()); + check_root(*sequential_tree_3, memdb.root()); + check_root(*batch_tree, memdb.root()); + + check_sibling_path(*sequential_tree_1, 0, memdb.get_sibling_path(0)); + check_sibling_path(*sequential_tree_2, 0, memdb.get_sibling_path(0)); + check_sibling_path(*sequential_tree_3, 0, memdb.get_sibling_path(0)); + check_sibling_path(*batch_tree, 0, memdb.get_sibling_path(0)); + + check_sibling_path(*sequential_tree_1, 512, memdb.get_sibling_path(512)); + check_sibling_path(*sequential_tree_2, 512, memdb.get_sibling_path(512)); + check_sibling_path(*sequential_tree_3, 512, memdb.get_sibling_path(512)); + check_sibling_path(*batch_tree, 512, memdb.get_sibling_path(512)); + + for (uint32_t j = 0; j < batch_size; j++) { + EXPECT_EQ(sequential_tree_1_low_leaf_witness_data->at(j).leaf, + sequential_tree_2_low_leaf_witness_data->at(j).leaf); + EXPECT_EQ(sequential_tree_1_low_leaf_witness_data->at(j).index, + sequential_tree_2_low_leaf_witness_data->at(j).index); + EXPECT_EQ(sequential_tree_1_low_leaf_witness_data->at(j).path, + sequential_tree_2_low_leaf_witness_data->at(j).path); + + EXPECT_EQ(sequential_tree_1_insertion_witness_data->at(j).leaf, + sequential_tree_2_insertion_witness_data->at(j).leaf); + EXPECT_EQ(sequential_tree_1_insertion_witness_data->at(j).index, + sequential_tree_2_insertion_witness_data->at(j).index); + EXPECT_EQ(sequential_tree_1_insertion_witness_data->at(j).path, + sequential_tree_2_insertion_witness_data->at(j).path); + } + } +} + +TEST_F(PersistedContentAddressedIndexedTreeTest, test_sequential_insert_vs_batch) +{ + uint32_t batchSize = 2; + while (batchSize <= 2) { + test_sequential_insert_vs_batch(batchSize, _directory, _mapSize, _maxReaders); + batchSize <<= 1; + } +} + +TEST_F(PersistedContentAddressedIndexedTreeTest, sequential_insert_allows_multiple_inserts_to_the_same_key) +{ + index_t current_size = 2; + ThreadPoolPtr workers = make_thread_pool(8); + // Create a depth-3 indexed merkle tree + constexpr size_t depth = 3; + std::string name = random_string(); + LMDBTreeStore::SharedPtr db = std::make_shared(_directory, name, _mapSize, _maxReaders); + std::unique_ptr> store = + std::make_unique>(name, depth, db); + auto tree = ContentAddressedIndexedTree, Poseidon2HashPolicy>( + std::move(store), workers, current_size); + + std::vector values{ PublicDataLeafValue(42, 27), PublicDataLeafValue(42, 28) }; + add_values_sequentially(tree, values); + + EXPECT_EQ(get_leaf(tree, 2).value, values[1]); +} + template fr hash_leaf(const IndexedLeaf& leaf) { return HashPolicy::hash(leaf.get_hash_inputs()); @@ -1475,8 +1647,8 @@ TEST_F(PersistedContentAddressedIndexedTreeTest, test_historic_sibling_path_retr memdb.update_element(batch[j].value); } memory_tree_sibling_paths_index_0.push_back(memdb.get_sibling_path(0)); - std::shared_ptr>> tree1_low_leaf_witness_data; - std::shared_ptr>> tree2_low_leaf_witness_data; + std::shared_ptr>> tree1_low_leaf_witness_data; + std::shared_ptr>> tree2_low_leaf_witness_data; { Signal signal; CompletionCallback completion = diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp index 6d7765e520f1..5fdfd19b85b9 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp @@ -27,7 +27,7 @@ struct GetSiblingPathResponse { fr_sibling_path path; }; -template struct LowLeafWitnessData { +template struct LeafUpdateWitnessData { IndexedLeaf leaf; index_t index; fr_sibling_path path; @@ -39,7 +39,13 @@ template struct AddIndexedDataResponse { AddDataResponse add_data_result; fr_sibling_path subtree_path; std::shared_ptr>> sorted_leaves; - std::shared_ptr>> low_leaf_witness_data; + std::shared_ptr>> low_leaf_witness_data; +}; + +template struct AddIndexedDataSequentiallyResponse { + AddDataResponse add_data_result; + std::shared_ptr>> low_leaf_witness_data; + std::shared_ptr>> insertion_witness_data; }; struct FindLeafIndexResponse { diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp index 3d81fbb58b5c..e2efc72bb1c5 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp @@ -469,13 +469,12 @@ void WorldState::rollback() signal.wait_for_level(); } -WorldStateStatusFull WorldState::sync_block( - const StateReference& block_state_ref, - const bb::fr& block_header_hash, - const std::vector& notes, - const std::vector& l1_to_l2_messages, - const std::vector& nullifiers, - const std::vector>& public_writes) +WorldStateStatusFull WorldState::sync_block(const StateReference& block_state_ref, + const bb::fr& block_header_hash, + const std::vector& notes, + const std::vector& l1_to_l2_messages, + const std::vector& nullifiers, + const std::vector& public_writes) { validate_trees_are_equally_synched(); WorldStateStatusFull status; @@ -525,31 +524,14 @@ WorldStateStatusFull WorldState::sync_block( wrapper.tree->add_value(block_header_hash, decr); } - // finally insert the public writes and wait for all the operations to end { - // insert public writes in batches so that we can have different transactions modifying the same slot in the - // same L2 block auto& wrapper = std::get>(fork->_trees.at(MerkleTreeId::PUBLIC_DATA_TREE)); - std::atomic_uint64_t current_batch = 0; - PublicDataTree::AddCompletionCallback completion = [&](const auto& resp) -> void { - current_batch++; - if (current_batch == public_writes.size()) { - decr(resp); - } else { - wrapper.tree->add_or_update_values(public_writes[current_batch], 0, completion); - } - }; - - if (public_writes.empty()) { - signal.signal_decrement(); - } else { - wrapper.tree->add_or_update_values(public_writes[current_batch], 0, completion); - } - - // block inside this scope in order to keep current_batch/completion alive until the end of all operations - signal.wait_for_level(); + PublicDataTree::AddCompletionCallback completion = [&](const auto&) -> void { signal.signal_decrement(); }; + wrapper.tree->add_or_update_values_sequentially(public_writes, completion); } + signal.wait_for_level(); + if (!success) { throw std::runtime_error("Failed to sync block: " + err_message); } diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp index 5c08a3d65593..5afa4a010ef7 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp @@ -35,13 +35,20 @@ namespace bb::world_state { using crypto::merkle_tree::index_t; template struct BatchInsertionResult { - std::vector> low_leaf_witness_data; + std::vector> low_leaf_witness_data; std::vector> sorted_leaves; crypto::merkle_tree::fr_sibling_path subtree_path; MSGPACK_FIELDS(low_leaf_witness_data, sorted_leaves, subtree_path); }; +template struct SequentialInsertionResult { + std::vector> low_leaf_witness_data; + std::vector> insertion_witness_data; + + MSGPACK_FIELDS(low_leaf_witness_data, insertion_witness_data); +}; + /** * @brief Holds the Merkle trees responsible for storing the state of the Aztec protocol. * @@ -178,6 +185,19 @@ class WorldState { uint32_t subtree_depth, Fork::Id fork_id = CANONICAL_FORK_ID); + /** + * @brief Inserts a set of leaves sequentially into an indexed Merkle Tree. + * + * @tparam T The type of the leaves. + * @param tree_id The ID of the Merkle Tree. + * @param leaves The leaves to insert. + * @return SequentialInsertionResult + */ + template + SequentialInsertionResult insert_indexed_leaves(MerkleTreeId tree_id, + const std::vector& leaves, + Fork::Id fork_id = CANONICAL_FORK_ID); + /** * @brief Updates a leaf in an existing Merkle Tree. * @@ -215,13 +235,12 @@ class WorldState { WorldStateStatusFull remove_historical_blocks(const index_t& toBlockNumber); void get_status_summary(WorldStateStatusSummary& status) const; - WorldStateStatusFull sync_block( - const StateReference& block_state_ref, - const bb::fr& block_header_hash, - const std::vector& notes, - const std::vector& l1_to_l2_messages, - const std::vector& nullifiers, - const std::vector>& public_writes); + WorldStateStatusFull sync_block(const StateReference& block_state_ref, + const bb::fr& block_header_hash, + const std::vector& notes, + const std::vector& l1_to_l2_messages, + const std::vector& nullifiers, + const std::vector& public_writes); private: std::shared_ptr _workers; @@ -562,6 +581,45 @@ BatchInsertionResult WorldState::batch_insert_indexed_leaves(MerkleTreeId id, return result; } + +template +SequentialInsertionResult WorldState::insert_indexed_leaves(MerkleTreeId id, + const std::vector& leaves, + Fork::Id fork_id) +{ + using namespace crypto::merkle_tree; + using Store = ContentAddressedCachedTreeStore; + using Tree = ContentAddressedIndexedTree; + + Fork::SharedPtr fork = retrieve_fork(fork_id); + + Signal signal; + SequentialInsertionResult result; + const auto& wrapper = std::get>(fork->_trees.at(id)); + bool success = true; + std::string error_msg; + + wrapper.tree->add_or_update_values_sequentially( + leaves, [&](const TypedResponse>& response) { + if (response.success) { + result.low_leaf_witness_data = *response.inner.low_leaf_witness_data; + result.insertion_witness_data = *response.inner.insertion_witness_data; + } else { + success = false; + error_msg = response.message; + } + + signal.signal_level(0); + }); + + signal.wait_for_level(); + + if (!success) { + throw std::runtime_error("Failed to sequentially insert indexed leaves: " + error_msg); + } + + return result; +} } // namespace bb::world_state MSGPACK_ADD_ENUM(bb::world_state::MerkleTreeId) diff --git a/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp index f5b1ce4b129e..d8ac68d0db14 100644 --- a/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp @@ -166,6 +166,10 @@ WorldStateAddon::WorldStateAddon(const Napi::CallbackInfo& info) WorldStateMessageType::BATCH_INSERT, [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return batch_insert(obj, buffer); }); + _dispatcher.registerTarget( + WorldStateMessageType::SEQUENTIAL_INSERT, + [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return sequential_insert(obj, buffer); }); + _dispatcher.registerTarget( WorldStateMessageType::UPDATE_ARCHIVE, [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return update_archive(obj, buffer); }); @@ -507,6 +511,42 @@ bool WorldStateAddon::batch_insert(msgpack::object& obj, msgpack::sbuffer& buffe return true; } +bool WorldStateAddon::sequential_insert(msgpack::object& obj, msgpack::sbuffer& buffer) +{ + TypedMessage request; + obj.convert(request); + + switch (request.value.treeId) { + case MerkleTreeId::PUBLIC_DATA_TREE: { + TypedMessage> r1; + obj.convert(r1); + auto result = _ws->insert_indexed_leaves( + request.value.treeId, r1.value.leaves, r1.value.forkId); + MsgHeader header(request.header.messageId); + messaging::TypedMessage> resp_msg( + WorldStateMessageType::SEQUENTIAL_INSERT, header, result); + msgpack::pack(buffer, resp_msg); + + break; + } + case MerkleTreeId::NULLIFIER_TREE: { + TypedMessage> r2; + obj.convert(r2); + auto result = _ws->insert_indexed_leaves( + request.value.treeId, r2.value.leaves, r2.value.forkId); + MsgHeader header(request.header.messageId); + messaging::TypedMessage> resp_msg( + WorldStateMessageType::SEQUENTIAL_INSERT, header, result); + msgpack::pack(buffer, resp_msg); + break; + } + default: + throw std::runtime_error("Unsupported tree type"); + } + + return true; +} + bool WorldStateAddon::update_archive(msgpack::object& obj, msgpack::sbuffer& buf) { TypedMessage request; @@ -560,7 +600,7 @@ bool WorldStateAddon::sync_block(msgpack::object& obj, msgpack::sbuffer& buf) request.value.paddedNoteHashes, request.value.paddedL1ToL2Messages, request.value.paddedNullifiers, - request.value.batchesOfPublicDataWrites); + request.value.publicDataWrites); MsgHeader header(request.header.messageId); messaging::TypedMessage resp_msg(WorldStateMessageType::SYNC_BLOCK, header, { status }); diff --git a/barretenberg/cpp/src/barretenberg/world_state_napi/addon.hpp b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.hpp index 034ca9cd0326..1a077a946e96 100644 --- a/barretenberg/cpp/src/barretenberg/world_state_napi/addon.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.hpp @@ -44,6 +44,7 @@ class WorldStateAddon : public Napi::ObjectWrap { bool append_leaves(msgpack::object& obj, msgpack::sbuffer& buffer); bool batch_insert(msgpack::object& obj, msgpack::sbuffer& buffer); + bool sequential_insert(msgpack::object& obj, msgpack::sbuffer& buffer); bool update_archive(msgpack::object& obj, msgpack::sbuffer& buffer); diff --git a/barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp b/barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp index 23f293fbebeb..c876f5993bc7 100644 --- a/barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp @@ -26,6 +26,7 @@ enum WorldStateMessageType { APPEND_LEAVES, BATCH_INSERT, + SEQUENTIAL_INSERT, UPDATE_ARCHIVE, @@ -168,6 +169,13 @@ template struct BatchInsertRequest { MSGPACK_FIELDS(treeId, leaves, subtreeDepth, forkId); }; +template struct InsertRequest { + MerkleTreeId treeId; + std::vector leaves; + Fork::Id forkId{ CANONICAL_FORK_ID }; + MSGPACK_FIELDS(treeId, leaves, forkId); +}; + struct UpdateArchiveRequest { StateReference blockStateRef; bb::fr blockHeaderHash; @@ -181,7 +189,7 @@ struct SyncBlockRequest { bb::fr blockHeaderHash; std::vector paddedNoteHashes, paddedL1ToL2Messages; std::vector paddedNullifiers; - std::vector> batchesOfPublicDataWrites; + std::vector publicDataWrites; MSGPACK_FIELDS(blockNumber, blockStateRef, @@ -189,7 +197,7 @@ struct SyncBlockRequest { paddedNoteHashes, paddedL1ToL2Messages, paddedNullifiers, - batchesOfPublicDataWrites); + publicDataWrites); }; } // namespace bb::world_state diff --git a/yarn-project/circuit-types/src/interfaces/merkle_tree_operations.ts b/yarn-project/circuit-types/src/interfaces/merkle_tree_operations.ts index 8b4cd35afc34..8520da7ca559 100644 --- a/yarn-project/circuit-types/src/interfaces/merkle_tree_operations.ts +++ b/yarn-project/circuit-types/src/interfaces/merkle_tree_operations.ts @@ -19,19 +19,19 @@ export type IndexedTreeId = MerkleTreeId.NULLIFIER_TREE | MerkleTreeId.PUBLIC_DA export type FrTreeId = Exclude; /** - * All of the data to be return during batch insertion. + * Witness data for a leaf update. */ -export interface LowLeafWitnessData { +export interface LeafUpdateWitnessData { /** - * Preimage of the low nullifier that proves non membership. + * Preimage of the leaf before updating. */ leafPreimage: IndexedTreeLeafPreimage; /** - * Sibling path to prove membership of low nullifier. + * Sibling path to prove membership of the leaf. */ siblingPath: SiblingPath; /** - * The index of low nullifier. + * The index of the leaf. */ index: bigint; } @@ -43,7 +43,7 @@ export interface BatchInsertionResult[]; + lowLeavesWitnessData?: LeafUpdateWitnessData[]; /** * Sibling path "pointing to" where the new subtree should be inserted into the tree. */ @@ -58,6 +58,20 @@ export interface BatchInsertionResult { + /** + * Data for the leaves to be updated when inserting the new ones. + */ + lowLeavesWitnessData: LeafUpdateWitnessData[]; + /** + * Data for the inserted leaves + */ + insertionWitnessData: LeafUpdateWitnessData[]; +} + /** * Defines tree information. */ @@ -215,6 +229,18 @@ export interface MerkleTreeWriteOperations extends MerkleTreeReadOperations { subtreeHeight: number, ): Promise>; + /** + * Inserts multiple leaves into the tree, getting witnesses at every step. + * Note: This method doesn't support inserting empty leaves. + * @param treeId - The tree on which to insert. + * @param leaves - The leaves to insert. + * @returns The witnesses for the low leaf updates and the insertions. + */ + sequentialInsert( + treeId: ID, + leaves: Buffer[], + ): Promise>; + /** * Closes the database, discarding any uncommitted changes. */ diff --git a/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts b/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts index bacfeaa2e027..63823f35620c 100644 --- a/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts +++ b/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts @@ -1,4 +1,4 @@ -import { type BatchInsertionResult, type LowLeafWitnessData, SiblingPath } from '@aztec/circuit-types'; +import { type BatchInsertionResult, type LeafUpdateWitnessData, SiblingPath } from '@aztec/circuit-types'; import { type TreeInsertionStats } from '@aztec/circuit-types/stats'; import { toBufferBE } from '@aztec/foundation/bigint-buffer'; import { type FromBuffer } from '@aztec/foundation/serialize'; @@ -44,7 +44,7 @@ export interface LeafFactory { function getEmptyLowLeafWitness( treeHeight: N, leafPreimageFactory: PreimageFactory, -): LowLeafWitnessData { +): LeafUpdateWitnessData { return { leafPreimage: leafPreimageFactory.empty(), index: 0n, @@ -473,7 +473,7 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree const insertedKeys = new Map(); const emptyLowLeafWitness = getEmptyLowLeafWitness(this.getDepth() as TreeHeight, this.leafPreimageFactory); // Accumulators - const lowLeavesWitnesses: LowLeafWitnessData[] = leaves.map(() => emptyLowLeafWitness); + const lowLeavesWitnesses: LeafUpdateWitnessData[] = leaves.map(() => emptyLowLeafWitness); const pendingInsertionSubtree: IndexedTreeLeafPreimage[] = leaves.map(() => this.leafPreimageFactory.empty()); // Start info @@ -516,7 +516,7 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree const lowLeafPreimage = this.getLatestLeafPreimageCopy(indexOfPrevious.index, true)!; const siblingPath = await this.getSiblingPath(BigInt(indexOfPrevious.index), true); - const witness: LowLeafWitnessData = { + const witness: LeafUpdateWitnessData = { leafPreimage: lowLeafPreimage, index: BigInt(indexOfPrevious.index), siblingPath, diff --git a/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts b/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts index a5f081d6687f..9dc700689b62 100644 --- a/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts +++ b/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts @@ -492,41 +492,30 @@ async function processPublicDataUpdateRequests(tx: ProcessedTx, db: MerkleTreeWr ({ leafSlot, value }) => new PublicDataTreeLeaf(leafSlot, value), ); - const lowPublicDataWritesPreimages = []; - const lowPublicDataWritesMembershipWitnesses = []; - const publicDataWritesSiblingPaths = []; - - for (const write of allPublicDataWrites) { - if (write.isEmpty()) { - throw new Error(`Empty public data write in tx: ${toFriendlyJSON(tx)}`); - } - - // TODO(Alvaro) write a specialized function for this? Internally add_or_update_value uses batch insertion anyway - const { lowLeavesWitnessData, newSubtreeSiblingPath } = await db.batchInsert( - MerkleTreeId.PUBLIC_DATA_TREE, - [write.toBuffer()], - // TODO(#3675) remove oldValue from update requests - 0, - ); - - if (lowLeavesWitnessData === undefined) { - throw new Error(`Could not craft public data batch insertion proofs`); - } - - const [lowLeafWitness] = lowLeavesWitnessData; - lowPublicDataWritesPreimages.push(lowLeafWitness.leafPreimage as PublicDataTreeLeafPreimage); - lowPublicDataWritesMembershipWitnesses.push( - MembershipWitness.fromBufferArray( - lowLeafWitness.index, - assertLength(lowLeafWitness.siblingPath.toBufferArray(), PUBLIC_DATA_TREE_HEIGHT), - ), - ); + const { lowLeavesWitnessData, insertionWitnessData } = await db.sequentialInsert( + MerkleTreeId.PUBLIC_DATA_TREE, + allPublicDataWrites.map(write => { + if (write.isEmpty()) { + throw new Error(`Empty public data write in tx: ${toFriendlyJSON(tx)}`); + } + return write.toBuffer(); + }), + ); - const insertionSiblingPath = newSubtreeSiblingPath.toFields(); + const lowPublicDataWritesPreimages = lowLeavesWitnessData.map( + lowLeafWitness => lowLeafWitness.leafPreimage as PublicDataTreeLeafPreimage, + ); + const lowPublicDataWritesMembershipWitnesses = lowLeavesWitnessData.map(lowLeafWitness => + MembershipWitness.fromBufferArray( + lowLeafWitness.index, + assertLength(lowLeafWitness.siblingPath.toBufferArray(), PUBLIC_DATA_TREE_HEIGHT), + ), + ); + const publicDataWritesSiblingPaths = insertionWitnessData.map(w => { + const insertionSiblingPath = w.siblingPath.toFields(); assertLength(insertionSiblingPath, PUBLIC_DATA_TREE_HEIGHT); - - publicDataWritesSiblingPaths.push(insertionSiblingPath as Tuple); - } + return insertionSiblingPath as Tuple; + }); return { lowPublicDataWritesPreimages, diff --git a/yarn-project/prover-client/src/test/bb_prover_full_rollup.test.ts b/yarn-project/prover-client/src/test/bb_prover_full_rollup.test.ts index ee3dc99956d7..e573e8572f71 100644 --- a/yarn-project/prover-client/src/test/bb_prover_full_rollup.test.ts +++ b/yarn-project/prover-client/src/test/bb_prover_full_rollup.test.ts @@ -22,7 +22,7 @@ describe('prover/bb_prover/full-rollup', () => { return prover; }; log = createDebugLogger('aztec:bb-prover-full-rollup'); - context = await TestContext.new(log, 'legacy', 1, buildProver); + context = await TestContext.new(log, 'native', 1, buildProver); }); afterAll(async () => { diff --git a/yarn-project/world-state/src/native/merkle_trees_facade.ts b/yarn-project/world-state/src/native/merkle_trees_facade.ts index be83ce2c102c..49d53e8f3a51 100644 --- a/yarn-project/world-state/src/native/merkle_trees_facade.ts +++ b/yarn-project/world-state/src/native/merkle_trees_facade.ts @@ -5,6 +5,7 @@ import { type MerkleTreeLeafType, type MerkleTreeReadOperations, type MerkleTreeWriteOperations, + type SequentialInsertionResult, SiblingPath, type TreeInfo, } from '@aztec/circuit-types'; @@ -221,6 +222,31 @@ export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTr }; } + async sequentialInsert( + treeId: ID, + rawLeaves: Buffer[], + ): Promise> { + const leaves = rawLeaves.map((leaf: Buffer) => hydrateLeaf(treeId, leaf)).map(serializeLeaf); + const resp = await this.instance.call(WorldStateMessageType.SEQUENTIAL_INSERT, { + leaves, + treeId, + forkId: this.revision.forkId, + }); + + return { + lowLeavesWitnessData: resp.low_leaf_witness_data.map(data => ({ + index: BigInt(data.index), + leafPreimage: deserializeIndexedLeaf(data.leaf), + siblingPath: new SiblingPath(data.path.length as any, data.path), + })), + insertionWitnessData: resp.insertion_witness_data.map(data => ({ + index: BigInt(data.index), + leafPreimage: deserializeIndexedLeaf(data.leaf), + siblingPath: new SiblingPath(data.path.length as any, data.path), + })), + }; + } + public async close(): Promise { assert.notEqual(this.revision.forkId, 0, 'Fork ID must be set'); await this.instance.call(WorldStateMessageType.DELETE_FORK, { forkId: this.revision.forkId }); diff --git a/yarn-project/world-state/src/native/message.ts b/yarn-project/world-state/src/native/message.ts index 8013e81e772d..12c4c410edc7 100644 --- a/yarn-project/world-state/src/native/message.ts +++ b/yarn-project/world-state/src/native/message.ts @@ -60,6 +60,7 @@ export enum WorldStateMessageType { APPEND_LEAVES, BATCH_INSERT, + SEQUENTIAL_INSERT, UPDATE_ARCHIVE, @@ -375,6 +376,7 @@ interface AppendLeavesRequest extends WithTreeId, WithForkId, WithLeaves {} interface BatchInsertRequest extends WithTreeId, WithForkId, WithLeaves { subtreeDepth: number; } + interface BatchInsertResponse { low_leaf_witness_data: ReadonlyArray<{ leaf: SerializedIndexedLeaf; @@ -385,6 +387,21 @@ interface BatchInsertResponse { subtree_path: Tuple; } +interface SequentialInsertRequest extends WithTreeId, WithForkId, WithLeaves {} + +interface SequentialInsertResponse { + low_leaf_witness_data: ReadonlyArray<{ + leaf: SerializedIndexedLeaf; + index: bigint | number; + path: Tuple; + }>; + insertion_witness_data: ReadonlyArray<{ + leaf: SerializedIndexedLeaf; + index: bigint | number; + path: Tuple; + }>; +} + interface UpdateArchiveRequest extends WithForkId { blockStateRef: BlockStateReference; blockHeaderHash: Buffer; @@ -397,7 +414,7 @@ interface SyncBlockRequest { paddedNoteHashes: readonly SerializedLeafValue[]; paddedL1ToL2Messages: readonly SerializedLeafValue[]; paddedNullifiers: readonly SerializedLeafValue[]; - batchesOfPublicDataWrites: readonly SerializedLeafValue[][]; + publicDataWrites: readonly SerializedLeafValue[]; } interface CreateForkRequest { @@ -435,6 +452,7 @@ export type WorldStateRequest = { [WorldStateMessageType.APPEND_LEAVES]: AppendLeavesRequest; [WorldStateMessageType.BATCH_INSERT]: BatchInsertRequest; + [WorldStateMessageType.SEQUENTIAL_INSERT]: SequentialInsertRequest; [WorldStateMessageType.UPDATE_ARCHIVE]: UpdateArchiveRequest; @@ -469,6 +487,7 @@ export type WorldStateResponse = { [WorldStateMessageType.APPEND_LEAVES]: void; [WorldStateMessageType.BATCH_INSERT]: BatchInsertResponse; + [WorldStateMessageType.SEQUENTIAL_INSERT]: SequentialInsertResponse; [WorldStateMessageType.UPDATE_ARCHIVE]: void; diff --git a/yarn-project/world-state/src/native/native_world_state.ts b/yarn-project/world-state/src/native/native_world_state.ts index a9336724a685..9e0b175ac6c5 100644 --- a/yarn-project/world-state/src/native/native_world_state.ts +++ b/yarn-project/world-state/src/native/native_world_state.ts @@ -178,18 +178,14 @@ export class NativeWorldStateService implements MerkleTreeDatabase { .flatMap(txEffect => padArrayEnd(txEffect.nullifiers, Fr.ZERO, MAX_NULLIFIERS_PER_TX)) .map(nullifier => new NullifierLeaf(nullifier)); - // We insert the public data tree leaves with one batch per tx to avoid updating the same key twice - const batchesOfPublicDataWrites: PublicDataTreeLeaf[][] = []; - for (const txEffect of paddedTxEffects) { - batchesOfPublicDataWrites.push( - txEffect.publicDataWrites.map(write => { - if (write.isEmpty()) { - throw new Error('Public data write must not be empty when syncing'); - } - return new PublicDataTreeLeaf(write.leafSlot, write.value); - }), - ); - } + const publicDataWrites: PublicDataTreeLeaf[] = paddedTxEffects.flatMap(txEffect => { + return txEffect.publicDataWrites.map(write => { + if (write.isEmpty()) { + throw new Error('Public data write must not be empty when syncing'); + } + return new PublicDataTreeLeaf(write.leafSlot, write.value); + }); + }); return await this.instance.call( WorldStateMessageType.SYNC_BLOCK, @@ -199,7 +195,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase { paddedL1ToL2Messages: paddedL1ToL2Messages.map(serializeLeaf), paddedNoteHashes: paddedNoteHashes.map(serializeLeaf), paddedNullifiers: paddedNullifiers.map(serializeLeaf), - batchesOfPublicDataWrites: batchesOfPublicDataWrites.map(batch => batch.map(serializeLeaf)), + publicDataWrites: publicDataWrites.map(serializeLeaf), blockStateRef: blockStateReference(l2Block.header.state), }, this.sanitiseAndCacheSummaryFromFull.bind(this), diff --git a/yarn-project/world-state/src/native/native_world_state_instance.ts b/yarn-project/world-state/src/native/native_world_state_instance.ts index 0533cf73f160..a1fa6baed48f 100644 --- a/yarn-project/world-state/src/native/native_world_state_instance.ts +++ b/yarn-project/world-state/src/native/native_world_state_instance.ts @@ -200,7 +200,7 @@ export class NativeWorldState implements NativeWorldStateInstance { data['notesCount'] = body.paddedNoteHashes.length; data['nullifiersCount'] = body.paddedNullifiers.length; data['l1ToL2MessagesCount'] = body.paddedL1ToL2Messages.length; - data['publicDataWritesCount'] = body.batchesOfPublicDataWrites.reduce((acc, batch) => acc + batch.length, 0); + data['publicDataWritesCount'] = body.publicDataWrites.length; } this.log.debug(`Calling messageId=${messageId} ${WorldStateMessageType[messageType]} with ${fmtLogData(data)}`); diff --git a/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts b/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts index 1992de2e9d7c..82f5934fb18c 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts @@ -3,6 +3,7 @@ import { type IndexedTreeId, type MerkleTreeLeafType, type MerkleTreeWriteOperations, + type SequentialInsertionResult, type TreeInfo, } from '@aztec/circuit-types/interfaces'; import { type Header, type StateReference } from '@aztec/circuits.js'; @@ -167,6 +168,19 @@ export class MerkleTreeReadOperationsFacade implements MerkleTreeWriteOperations return this.trees.batchInsert(treeId, leaves, subtreeHeight); } + /** + * Sequentially inserts multiple leaves into the tree. + * @param treeId - The ID of the tree. + * @param leaves - Leaves to insert into the tree. + * @returns Witnesses for the operations performed. + */ + public sequentialInsert( + _treeId: IndexedTreeId, + _leaves: Buffer[], + ): Promise> { + throw new Error('Method not implemented in legacy merkle tree'); + } + close(): Promise { return Promise.resolve(); }