feat(avm): merkle db hints (part 3)

barretenberg/cpp/CMakePresets.json

@@ -289,7 +289,8 @@
       "inherits": "clang16",
       "binaryDir": "build-fuzzing",
       "cacheVariables": {
-        "FUZZING": "ON"
+        "FUZZING": "ON",
+        "DISABLE_AZTEC_VM": "ON"
       }
     },
     {
@@ -301,6 +302,7 @@
       "cacheVariables": {
         "SHOW_INFORMATION": "ON",
         "FUZZING": "ON",
+        "DISABLE_AZTEC_VM": "ON",
         "ENABLE_ASAN": "ON",
         "DISABLE_ASM": "ON"
       }

barretenberg/cpp/src/barretenberg/common/utils.hpp

@@ -35,13 +35,19 @@ template <typename... Ts> struct HashableTuple : public std::tuple<Ts...> {
 
 } // namespace bb::utils
 
-// Define hash function so that they can be used as keys in maps.
-// See https://en.cppreference.com/w/cpp/utility/hash.
-template <typename... Ts> struct std::hash<bb::utils::HashableTuple<Ts...>> {
-    std::size_t operator()(const bb::utils::HashableTuple<Ts...>& st) const noexcept { return st.hash(); }
-};
-
 // Needed for HashableTuple to work as a tuple.
 template <typename... Ts> struct std::tuple_size<bb::utils::HashableTuple<Ts...>> {
     static constexpr size_t value = sizeof...(Ts);
 };
+
+// Define std::hash for any type that has a hash() method. This includes HashableTuple.
+template <typename T>
+concept Hashable = requires(const T& t) {
+    {
+        t.hash()
+    } -> std::same_as<std::size_t>;
+};
+
+template <Hashable T> struct std::hash<T> {
+    std::size_t operator()(const T& t) const noexcept { return t.hash(); }
+};

barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp

@@ -1,13 +1,5 @@
 #pragma once
-#include "../hash_path.hpp"
-#include "../node_store//tree_meta.hpp"
-#include "../response.hpp"
-#include "../types.hpp"
-#include "barretenberg/common/thread_pool.hpp"
-#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp"
-#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp"
-#include "barretenberg/crypto/merkle_tree/signal.hpp"
-#include "barretenberg/numeric/bitop/pow.hpp"
+
 #include <cstddef>
 #include <cstdint>
 #include <exception>
@@ -22,6 +14,16 @@
 #include <utility>
 #include <vector>
 
+#include "barretenberg/common/thread_pool.hpp"
+#include "barretenberg/crypto/merkle_tree/hash_path.hpp"
+#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp"
+#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp"
+#include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp"
+#include "barretenberg/crypto/merkle_tree/response.hpp"
+#include "barretenberg/crypto/merkle_tree/signal.hpp"
+#include "barretenberg/crypto/merkle_tree/types.hpp"
+#include "barretenberg/numeric/bitop/pow.hpp"
+
 namespace bb::crypto::merkle_tree {
 
 using namespace bb;

barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp

@@ -1,18 +1,5 @@
 #pragma once
-#include "../hash.hpp"
-#include "../hash_path.hpp"
-#include "../signal.hpp"
-#include "barretenberg/common/assert.hpp"
-#include "barretenberg/common/thread_pool.hpp"
-#include "barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp"
-#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp"
-#include "barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp"
-#include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp"
-#include "barretenberg/crypto/merkle_tree/response.hpp"
-#include "barretenberg/crypto/merkle_tree/types.hpp"
-#include "barretenberg/numeric/bitop/get_msb.hpp"
-#include "barretenberg/numeric/uint256/uint256.hpp"
-#include "indexed_leaf.hpp"
+
 #include <algorithm>
 #include <atomic>
 #include <cmath>
@@ -31,6 +18,22 @@
 #include <utility>
 #include <vector>
 
+#include "barretenberg/common/assert.hpp"
+#include "barretenberg/common/thread_pool.hpp"
+#include "barretenberg/common/utils.hpp"
+#include "barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp"
+#include "barretenberg/crypto/merkle_tree/hash.hpp"
+#include "barretenberg/crypto/merkle_tree/hash_path.hpp"
+#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp"
+#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp"
+#include "barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp"
+#include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp"
+#include "barretenberg/crypto/merkle_tree/response.hpp"
+#include "barretenberg/crypto/merkle_tree/signal.hpp"
+#include "barretenberg/crypto/merkle_tree/types.hpp"
+#include "barretenberg/numeric/bitop/get_msb.hpp"
+#include "barretenberg/numeric/uint256/uint256.hpp"
+
 namespace bb::crypto::merkle_tree {
 
 /**
@@ -630,6 +633,7 @@ void ContentAddressedIndexedTree<Store, HashingPolicy>::add_or_update_values_int
             }
             TypedResponse<GetSiblingPathResponse> response;
             response.success = true;
+
             sibling_path_completion(response);
         }
     };
@@ -706,15 +710,17 @@ void ContentAddressedIndexedTree<Store, HashingPolicy>::perform_updates(
 
     // We now kick off multiple workers to perform the low leaf updates
     // We create set of signals to coordinate the workers as the move up the tree
-    // We don';t want toflood the provided thread pool with jobs that can't be processed so we throttle the rate
+    // We don';t want to flood the provided thread pool with jobs that can't be processed so we throttle the rate
     // at which jobs are added to the thread pool. This enables other trees to utilise the same pool
-    std::shared_ptr<std::vector<Signal>> signals = std::make_shared<std::vector<Signal>>();
+    // NOTE: Wrapping signals with unique_ptr to make them movable (re: mac build).
+    // Feel free to reconsider and make Signal movable.
+    auto signals = std::make_shared<std::vector<std::unique_ptr<Signal>>>();
     std::shared_ptr<Status> status = std::make_shared<Status>();
     // The first signal is set to 0. This ensures the first worker up the tree is not impeded
-    signals->emplace_back(0);
+    signals->emplace_back(std::make_unique<Signal>(0));
     // Workers will follow their leaders up the tree, being triggered by the signal in front of them
     for (size_t i = 0; i < updates->size(); ++i) {
-        signals->emplace_back(uint32_t(1 + depth_));
+        signals->emplace_back(std::make_unique<Signal>(static_cast<uint32_t>(1 + depth_)));
     }
 
     {
@@ -752,8 +758,8 @@ void ContentAddressedIndexedTree<Store, HashingPolicy>::perform_updates(
         for (uint32_t i = 0; i < updates->size(); ++i) {
             std::function<void()> op = [=, this]() {
                 LeafUpdate& update = (*updates)[i];
-                Signal& leaderSignal = (*signals)[i];
-                Signal& followerSignal = (*signals)[i + 1];
+                Signal& leaderSignal = *(*signals)[i];
+                Signal& followerSignal = *(*signals)[i + 1];
                 try {
                     auto& current_witness_data = update_witnesses->at(i);
                     current_witness_data.leaf = update.original_leaf;
@@ -828,7 +834,7 @@ void ContentAddressedIndexedTree<Store, HashingPolicy>::perform_updates_without_
     uint64_t numBatchesPower2Floor = numeric::get_msb(workers_->num_threads());
     index_t numBatches = static_cast<index_t>(std::pow(2UL, numBatchesPower2Floor));
     index_t batchSize = span / numBatches;
-    batchSize = std::max(batchSize, 2UL);
+    batchSize = std::max(batchSize, static_cast<index_t>(2));
     index_t startIndex = 0;
     indexPower2Ceil = log2Ceil(batchSize);
     uint32_t rootLevel = depth_ - static_cast<uint32_t>(indexPower2Ceil);
@@ -899,7 +905,7 @@ void ContentAddressedIndexedTree<Store, HashingPolicy>::generate_hashes_for_appe
 template <typename Store, typename HashingPolicy>
 void ContentAddressedIndexedTree<Store, HashingPolicy>::generate_insertions(
     const std::shared_ptr<std::vector<std::pair<LeafValueType, index_t>>>& values_to_be_sorted,
-    const InsertionGenerationCallback& on_completion)
+    const InsertionGenerationCallback& completion)
 {
     execute_and_report<InsertionGenerationResponse>(
         [=, this](TypedResponse<InsertionGenerationResponse>& response) {
@@ -1064,7 +1070,7 @@ void ContentAddressedIndexedTree<Store, HashingPolicy>::generate_insertions(
                 }
             }
         },
-        on_completion);
+        completion);
 }
 
 template <typename Store, typename HashingPolicy>
@@ -1169,8 +1175,7 @@ void ContentAddressedIndexedTree<Store, HashingPolicy>::sparse_batch_update(
     while (level > 0) {
         std::vector<index_t> next_indices;
         std::unordered_map<index_t, fr> next_hashes;
-        for (size_t i = 0; i < indices.size(); ++i) {
-            index_t index = indices[i];
+        for (index_t index : indices) {
             index_t parent_index = index >> 1;
             auto it = unique_indices.insert(parent_index);
             if (!it.second) {
@@ -1253,8 +1258,7 @@ std::pair<bool, fr> ContentAddressedIndexedTree<Store, HashingPolicy>::sparse_ba
     while (level > root_level) {
         std::vector<index_t> next_indices;
         std::unordered_map<index_t, fr> next_hashes;
-        for (size_t i = 0; i < indices.size(); ++i) {
-            index_t index = indices[i];
+        for (index_t index : indices) {
             index_t parent_index = index >> 1;
             auto it = unique_indices.insert(parent_index);
             if (!it.second) {

barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "barretenberg/common/utils.hpp"
 #include "barretenberg/crypto/merkle_tree/types.hpp"
 #include "barretenberg/ecc/curves/bn254/fr.hpp"
 #include "barretenberg/serialize/msgpack.hpp"
@@ -61,6 +62,8 @@ struct NullifierLeafValue {
     static NullifierLeafValue padding(index_t i) { return { i }; }
 
     static std::string name() { return "NullifierLeafValue"; };
+
+    size_t hash() const noexcept { return std::hash<fr>{}(nullifier); }
 };
 
 struct PublicDataLeafValue {
@@ -121,6 +124,8 @@ struct PublicDataLeafValue {
     static PublicDataLeafValue padding(index_t i) { return { i, fr::zero() }; }
 
     static std::string name() { return "PublicDataLeafValue"; };
+
+    size_t hash() const noexcept { return utils::hash_as_tuple(value, slot); }
 };
 
 template <typename LeafType> struct IndexedLeaf {

barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/content_addressed_cache.hpp

@@ -20,17 +20,6 @@
 #include <utility>
 #include <vector>
 
-template <> struct std::hash<uint256_t> {
-    std::size_t operator()(const uint256_t& k) const { return k.data[0]; }
-};
-template <> struct std::hash<bb::fr> {
-    std::size_t operator()(const bb::fr& k) const
-    {
-        bb::numeric::uint256_t val(k);
-        return val.data[0];
-    }
-};
-
 namespace bb::crypto::merkle_tree {
 
 // Stores all of the penidng updates to a mekle tree indexed for optimal retrieval
@@ -457,4 +446,4 @@ void ContentAddressedCache<LeafValueType>::put_node_by_index(uint32_t level, con
     }
     nodes_by_index_[level][index] = node;
 }
-} // namespace bb::crypto::merkle_tree
+} // namespace bb::crypto::merkle_tree

barretenberg/cpp/src/barretenberg/numeric/uint256/uint256.hpp

@@ -14,6 +14,7 @@
 #include "../uint128/uint128.hpp"
 #include "barretenberg/common/serialize.hpp"
 #include "barretenberg/common/throw_or_abort.hpp"
+#include "barretenberg/common/utils.hpp"
 #include <concepts>
 #include <cstdint>
 #include <iomanip>
@@ -202,6 +203,8 @@ class alignas(32) uint256_t {
 
     [[nodiscard]] constexpr std::pair<uint256_t, uint256_t> divmod(const uint256_t& b) const;
 
+    size_t hash() const noexcept { return utils::hash_as_tuple(data[0], data[1], data[2], data[3]); }
+
   private:
     [[nodiscard]] static constexpr std::pair<uint64_t, uint64_t> mul_wide(uint64_t a, uint64_t b);
     [[nodiscard]] static constexpr std::pair<uint64_t, uint64_t> addc(uint64_t a, uint64_t b, uint64_t carry_in);

barretenberg/cpp/src/barretenberg/vm2/common/avm_inputs.hpp

@@ -7,7 +7,10 @@
 
 #include "barretenberg/common/utils.hpp"
 #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp"
+#include "barretenberg/crypto/merkle_tree/response.hpp"
 #include "barretenberg/serialize/msgpack.hpp"
+#include "barretenberg/world_state/world_state.hpp"
+
 #include "barretenberg/vm2/common/aztec_types.hpp"
 #include "barretenberg/vm2/common/field.hpp"
 #include "barretenberg/world_state/types.hpp"
@@ -158,6 +161,22 @@ struct GetLeafValueHint {
     MSGPACK_FIELDS(hintKey, treeId, index, value);
 };
 
+template <typename Leaf> struct SequentialInsertHint {
+    AppendOnlyTreeSnapshot hintKey;
+    // params
+    world_state::MerkleTreeId treeId;
+    Leaf leaf;
+    // return
+    crypto::merkle_tree::LeafUpdateWitnessData<Leaf> lowLeavesWitnessData;
+    crypto::merkle_tree::LeafUpdateWitnessData<Leaf> insertionWitnessData;
+    // evolved state
+    AppendOnlyTreeSnapshot stateAfter;
+
+    bool operator==(const SequentialInsertHint<Leaf>& other) const = default;
+
+    MSGPACK_FIELDS(hintKey, treeId, leaf, lowLeavesWitnessData, insertionWitnessData, stateAfter);
+};
+
 ////////////////////////////////////////////////////////////////////////////
 // Hints (other)
 ////////////////////////////////////////////////////////////////////////////
@@ -189,6 +208,8 @@ struct ExecutionHints {
     std::vector<GetLeafPreimageHint<crypto::merkle_tree::IndexedLeaf<crypto::merkle_tree::NullifierLeafValue>>>
         getLeafPreimageHintsNullifierTree;
     std::vector<GetLeafValueHint> getLeafValueHints;
+    std::vector<SequentialInsertHint<crypto::merkle_tree::PublicDataLeafValue>> sequentialInsertHintsPublicDataTree;
+    std::vector<SequentialInsertHint<crypto::merkle_tree::NullifierLeafValue>> sequentialInsertHintsNullifierTree;
 
     bool operator==(const ExecutionHints& other) const = default;
 
@@ -200,7 +221,9 @@ struct ExecutionHints {
                    getPreviousValueIndexHints,
                    getLeafPreimageHintsPublicDataTree,
                    getLeafPreimageHintsNullifierTree,
-                   getLeafValueHints);
+                   getLeafValueHints,
+                   sequentialInsertHintsPublicDataTree,
+                   sequentialInsertHintsNullifierTree);
 };
 
 ////////////////////////////////////////////////////////////////////////////
@@ -218,9 +241,6 @@ struct AvmProvingInputs {
 
 } // namespace bb::avm2
 
-// This has to be done outside of the namespace.
-MSGPACK_ADD_ENUM(bb::world_state::MerkleTreeId);
-
 // Define hash function so that they can be used as keys in maps.
 // See https://en.cppreference.com/w/cpp/utility/hash.
 template <> struct std::hash<bb::avm2::AppendOnlyTreeSnapshot> {

barretenberg/cpp/src/barretenberg/vm2/simulation/lib/db_interfaces.hpp

@@ -9,6 +9,7 @@
 #include "barretenberg/vm2/common/avm_inputs.hpp"
 #include "barretenberg/vm2/common/aztec_types.hpp"
 #include "barretenberg/world_state/types.hpp"
+#include "barretenberg/world_state/world_state.hpp"
 
 namespace bb::avm2::simulation {
 
@@ -38,6 +39,16 @@ class LowLevelMerkleDBInterface {
     get_leaf_preimage_public_data_tree(crypto::merkle_tree::index_t leaf_index) const = 0;
     virtual crypto::merkle_tree::IndexedLeaf<crypto::merkle_tree::NullifierLeafValue> get_leaf_preimage_nullifier_tree(
         crypto::merkle_tree::index_t leaf_index) const = 0;
+
+    // Inserts a leaf into the public data tree sequentially, getting witnesses at every step.
+    // Note: This method doesn't support inserting empty leaves.
+    virtual world_state::SequentialInsertionResult<crypto::merkle_tree::PublicDataLeafValue>
+    insert_indexed_leaves_public_data_tree(const crypto::merkle_tree::PublicDataLeafValue& leaf_value) = 0;
+
+    // Inserts a leaf into the nullifier tree sequentially, getting witnesses at every step.
+    // Note: This method doesn't support inserting empty leaves.
+    virtual world_state::SequentialInsertionResult<crypto::merkle_tree::NullifierLeafValue>
+    insert_indexed_leaves_nullifier_tree(const crypto::merkle_tree::NullifierLeafValue& leaf_value) = 0;
 };
 
 // High level access to a merkle db. In general these will be constrained.