msgpack: friendly binary serialization format

.circleci/build

@@ -20,6 +20,8 @@ echo "Repository: $REPOSITORY"
 echo "Working directory: $PWD"
 echo "Dockerfile: $DOCKERFILE"
 
+init_submodules $REPOSITORY
+
 cd $(query_manifest buildDir $REPOSITORY)
 
 function fetch_image() {

.circleci/cond_spot_run_build

@@ -7,6 +7,8 @@ SPEC=$1
 shift
 DOCKERFILE=$(query_manifest dockerfile $REPOSITORY)
 
+init_submodules $REPOSITORY
+
 cd $(query_manifest buildDir $REPOSITORY)
 
 LAST_SUCCESSFUL_COMMIT=$(last_successful_commit $REPOSITORY)

.circleci/cond_spot_run_tests

@@ -5,6 +5,8 @@ set -o pipefail
 REPOSITORY=$1
 shift
 
+init_submodules $REPOSITORY
+
 cd $(query_manifest projectDir $REPOSITORY)
 
 mkdir -p /tmp/test-logs

.circleci/config.yml

@@ -49,6 +49,7 @@ checkout: &checkout
       git remote add origin $CIRCLE_REPOSITORY_URL
       git fetch --depth 50 --filter=blob:none origin $CIRCLE_SHA1
       git checkout FETCH_HEAD
+      git submodule update --init --recursive
 
 # This build step checks out the code from the benchmark-archive repository. The key is saved in CircleCi environment in base64 format.
 # Initially it just fetches the latest version.

.circleci/init_submodules

@@ -0,0 +1,18 @@
+#!/bin/bash
+# For a given repository, init any required submodules.
+set -euo pipefail
+
+REPOSITORY=$1
+
+BUILD_DIR=$(query_manifest buildDir $REPOSITORY)
+if [ -d "$BUILD_DIR" ]; then
+  echo "Initialising any submodules under $REPOSITORY buildDir: $BUILD_DIR"
+  git submodule update --init --recursive $BUILD_DIR && exit || echo "No submodules under buildDir"
+fi
+
+SUBMODULE_PATH=$(query_manifest submodulePath $REPOSITORY)
+if [ -n "$SUBMODULE_PATH" ]; then
+  # TODO: Needs to actually init all dependency submodules as well.
+  echo "Initialising submodule: $SUBMODULE_PATH"
+  git submodule update --init --recursive $SUBMODULE_PATH
+fi

.circleci/remote_build/remote_build

@@ -15,6 +15,7 @@ git config remote.origin.promisor true
 git config remote.origin.partialclonefilter blob:none
 git fetch --depth 50 origin $COMMIT_HASH
 git checkout FETCH_HEAD
+git submodule update --init
 
 echo "Git operations completed..."
 
@@ -24,4 +25,4 @@ echo "Calling setup env..."
 source ./setup_env "$COMMIT_HASH" "$COMMIT_TAG" "$JOB_NAME" "$GIT_REPOSITORY_URL"
 cd ..
 echo "Calling build..."
-build $@
+build $@

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "foundation"]
 	path = foundation
 	url = git@github.com:AztecProtocol/foundation.git
+[submodule "cpp/src/msgpack-c"]
+	path = cpp/src/msgpack-c
+	url = https://github.com/AztecProtocol/msgpack-c

cpp/src/CMakeLists.txt

@@ -27,7 +27,7 @@ if(CMAKE_CXX_COMPILER_ID MATCHES "GNU")
     add_compile_options(-fconstexpr-ops-limit=100000000)
 endif()
 
-include_directories(${CMAKE_CURRENT_SOURCE_DIR})
+include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/msgpack-c/include)
 
 # I feel this should be limited to ecc, however it's currently used in headers that go across libraries,
 # and there currently isn't an easy way to inherit the DDISABLE_SHENANIGANS parameter.
@@ -90,6 +90,7 @@ if(WASM)
         $<TARGET_OBJECTS:stdlib_pedersen_commitment_objects>
         $<TARGET_OBJECTS:stdlib_blake2s_objects>
         $<TARGET_OBJECTS:stdlib_blake3s_objects>
+        $<TARGET_OBJECTS:stdlib_keccak_objects>
         $<TARGET_OBJECTS:stdlib_sha256_objects>
         $<TARGET_OBJECTS:stdlib_aes128_objects>
         $<TARGET_OBJECTS:stdlib_merkle_tree_objects>
@@ -192,6 +193,7 @@ if(WASM)
         $<TARGET_OBJECTS:stdlib_pedersen_commitment_objects>
         $<TARGET_OBJECTS:stdlib_blake2s_objects>
         $<TARGET_OBJECTS:stdlib_blake3s_objects>
+        $<TARGET_OBJECTS:stdlib_keccak_objects>
         $<TARGET_OBJECTS:stdlib_sha256_objects>
         $<TARGET_OBJECTS:stdlib_aes128_objects>
         $<TARGET_OBJECTS:stdlib_merkle_tree_objects>
@@ -227,6 +229,7 @@ else()
         $<TARGET_OBJECTS:stdlib_pedersen_commitment_objects>
         $<TARGET_OBJECTS:stdlib_blake2s_objects>
         $<TARGET_OBJECTS:stdlib_blake3s_objects>
+        $<TARGET_OBJECTS:stdlib_keccak_objects>
         $<TARGET_OBJECTS:stdlib_sha256_objects>
         $<TARGET_OBJECTS:stdlib_aes128_objects>
         $<TARGET_OBJECTS:stdlib_merkle_tree_objects>

cpp/src/barretenberg/common/log.hpp

@@ -18,23 +18,10 @@
 
 namespace {
 
-inline void format_chain(std::ostream&) {}
-
-template <typename T> void format_chain(std::ostream& os, T const& first)
-{
-    os << first;
-}
-
-template <typename T, typename... Args> void format_chain(std::ostream& os, T const& first, Args const&... args)
-{
-    os << first;
-    format_chain(os, args...);
-}
-
 template <typename... Args> std::string format(Args... args)
 {
     std::ostringstream os;
-    format_chain(os, args...);
+    ((os << args), ...);
     return os.str();
 }
 

cpp/src/barretenberg/common/msgpack.hpp

@@ -0,0 +1,108 @@
+#pragma once
+/* Minimal header for msgpack. This should be included, *not* msgpack.hpp directly.
+That should be treated as an implementation-only header.
+
+## Overview
+
+The Msgpack module allows for efficient serialization and deserialization of data structures. It can be applied to map-like objects, array-like objects, and custom serialization/deserialization logic.
+
+## Binding objects
+
+Marking structs/classes with their fields for msgpack allows you to pack and unpack the class.
+
+1. All objects bound should have a default constructor
+2. Objects can be tightly packed as binary (see field_impl.hpp), array-like, or map-like. See below
+3. You should list all fields of a class in the below methods, or use the custom method.
+
+### Map-like Objects
+
+To make objects serializable as a map-like format, define the `msgpack` method in your class as follows:
+
+```cpp
+void msgpack(auto ar) {
+    ar(NVP(composer_type, circuit_size, num_public_inputs, commitments, contains_recursive_proof, recursive_proof_public_input_indices));
+}
+or
+MSGPACK(composer_type, circuit_size, num_public_inputs, commitments, contains_recursive_proof, recursive_proof_public_input_indices);
+```
+
+This approach assumes 1. all members are default constructible 2. you give it all members 3. all members are writable references
+
+This method maps the object's properties (e.g., `composer_type`, `circuit_size`, etc.) to their respective keys in the serialized data.
+
+### Array-like Objects
+
+For array-like objects, define the `msgpack` method similarly:
+
+```cpp
+void msgpack(auto ar) {
+    ar(NVP(composer_type, circuit_size, num_public_inputs, commitments, contains_recursive_proof, recursive_proof_public_input_indices));
+}
+```
+
+This will serialize the object's properties as an array, rather than a map.
+
+### Custom Serialization and Deserialization
+
+For custom serialization and deserialization, define `msgpack_pack` and `msgpack_unpack` methods in your class:
+
+```cpp
+// For serialization
+template <class Params> void field<Params>::msgpack_pack(auto& packer) const
+{
+    auto adjusted = from_montgomery_form();
+    uint64_t bin_data[4] = {
+        htonll(adjusted.data[3]), htonll(adjusted.data[2]), htonll(adjusted.data[1]), htonll(adjusted.data[0])
+    };
+    packer.pack_bin(sizeof(bin_data));
+    packer.pack_bin_body((const char*)bin_data, sizeof(bin_data));
+}
+
+// For deserialization
+template <class Params> void field<Params>::msgpack_unpack(auto o)
+{
+    msgpack::read_bin64(o, data, 4);
+    uint64_t reversed[] = {data[3], data[2], data[1], data[0]};
+    for (int i = 0; i < 4; i++) {
+        data[i] = reversed[i];
+    }
+    *this = to_montgomery_form();
+}
+```
+
+These methods allow you to implement custom logic for the serialization and deserialization processes.
+
+
+## Packing/Unpacking
+
+Only when actually using msgpack to write or read data, include "barretenberg/common/msgpack_impl.hpp".
+You can then use msgpack library features to serialize and deserialize C++ objects.
+
+e.g. packing
+```
+    // Create a buffer to store the encoded data
+    msgpack::sbuffer buffer;
+    msgpack::pack(buffer, obj);
+
+    uint8_t* output = (uint8_t*)aligned_alloc(64, buffer.size());
+    memcpy(output, buffer.data(), buffer.size());
+    // Convert the buffer data to a string and return it
+    return { output, buffer.size() };
+```
+
+e.g. unpacking
+
+```
+    msgpack::unpack((const char*)encoded_data, encoded_data_size).get().convert(*value);
+```
+
+*/
+#include <type_traits>
+#include "msgpack/name_value_pair_macro.hpp"
+#include "msgpack/concepts.hpp"
+
+// Helper for above documented syntax
+#define MSGPACK(...) \
+    void msgpack(auto pack_fn) { \
+        pack_fn(NVP(__VA_ARGS__)); \
+    }

cpp/src/barretenberg/common/msgpack/concepts.hpp

@@ -0,0 +1,11 @@
+#pragma once
+
+struct DoNothing {
+    void operator()(auto...){}
+};
+namespace msgpack {
+template<typename T>
+concept HasMsgPack = requires(T t, DoNothing nop) { t.msgpack(nop); };
+template<typename T>
+concept HasMsgPackPack = requires(T t, DoNothing nop) { t.msgpack_pack(nop); };
+}

cpp/src/barretenberg/common/msgpack/name_value_pair_macro.hpp

@@ -0,0 +1,34 @@
+#pragma once
+
+/* start of #define NVP:
+ * expands to name-value pairs (NVPs), e.g. NVP(x,y,z) -> "x", x, "y", y", "z", z
+ * used in msgpack serialization. */
+
+// hacky counting of variadic macro params:
+#define VA_NARGS_IMPL(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, N, ...) N
+#define VA_NARGS(...) VA_NARGS_IMPL(__VA_ARGS__, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1)
+
+// name-value pair expansion for variables
+// used in msgpack map expansion
+// n<=3 case
+#define _NVP1(x) #x, x
+#define _NVP2(x, y) #x, x, #y, y
+#define _NVP3(x, y, z) #x, x, #y, y, #z, z
+// n>3 cases
+#define _NVP4(x, ...) _NVP1(x), _NVP3(__VA_ARGS__)
+#define _NVP5(x, ...) _NVP1(x), _NVP4(__VA_ARGS__)
+#define _NVP6(x, ...) _NVP1(x), _NVP5(__VA_ARGS__)
+#define _NVP7(x, ...) _NVP1(x), _NVP6(__VA_ARGS__)
+#define _NVP8(x, ...) _NVP1(x), _NVP7(__VA_ARGS__)
+#define _NVP9(x, ...) _NVP1(x), _NVP8(__VA_ARGS__)
+#define _NVP10(x, ...) _NVP1(x), _NVP9(__VA_ARGS__)
+#define _NVP11(x, ...) _NVP1(x), _NVP10(__VA_ARGS__)
+#define _NVP12(x, ...) _NVP1(x), _NVP11(__VA_ARGS__)
+#define _NVP13(x, ...) _NVP1(x), _NVP12(__VA_ARGS__)
+#define _NVP14(x, ...) _NVP1(x), _NVP13(__VA_ARGS__)
+#define _NVP15(x, ...) _NVP1(x), _NVP14(__VA_ARGS__)
+
+#define CONCAT(a, b) a ## b
+#define _NVP_N(n) CONCAT(_NVP, n)
+#define NVP(...) _NVP_N(VA_NARGS(__VA_ARGS__))(__VA_ARGS__)
+// end of #define NVP

cpp/src/barretenberg/common/msgpack/struct_map_impl.hpp

@@ -0,0 +1,72 @@
+#pragma once
+// Note: heavy header due to serialization logic, don't include if msgpack.hpp will do
+#include <string>
+#include <iostream>
+#include <iomanip>
+#include <sstream>
+#include <cassert>
+#define MSGPACK_NO_BOOST
+#include <msgpack.hpp>
+#include "concepts.hpp"
+
+namespace msgpack {
+    template <typename Tuple, std::size_t... Is> auto drop_keys_impl(Tuple&& tuple, std::index_sequence<Is...>)
+    {
+        // Expand 0 to n/2 to 1 to n+1 (increments of 2)
+        // Use it to filter the tuple
+        return std::tie(std::get<Is * 2 + 1>(std::forward<Tuple>(tuple))...);
+    }
+
+    template <typename... Args> auto drop_keys(std::tuple<Args...>&& tuple)
+    {
+        static_assert(sizeof...(Args) % 2 == 0, "Tuple must contain an even number of elements");
+        // Compile time sequence of integers from 0 to n/2
+        auto compile_time_0_to_n_div_2 = std::make_index_sequence<sizeof...(Args) / 2>{};
+        return drop_keys_impl(tuple, compile_time_0_to_n_div_2);
+    }
+} // namespace msgpack
+
+namespace msgpack {
+    template <typename T, typename... Args>
+    concept MsgpackConstructible = requires(T object, Args... args) {
+        T{args...};
+    };
+}
+
+namespace msgpack::adaptor {
+// reads structs with msgpack() method from a JSON-like dictionary
+template <HasMsgPack T> struct convert<T> {
+    msgpack::object const& operator()(msgpack::object const& o, T& v) const
+    {
+        static_assert(std::is_default_constructible_v<T>, "MSGPACK requires default-constructible types");
+        v.msgpack([&](auto&... args) {
+            auto static_checker = [&](auto&... value_args) {
+                static_assert(msgpack::MsgpackConstructible<T, decltype(value_args)...>,
+                    "MSGPACK requires a constructor that can take the types listed in MSGPACK. "
+                    "Type or arg count mismatch, or member initializer constructor not available.");
+            };
+            std::apply(static_checker, drop_keys(std::tie(args...)));
+            msgpack::type::define_map<decltype(args)...>{args...}.msgpack_unpack(o);
+        });
+        return o;
+    }
+};
+
+// converts structs with msgpack() method from a JSON-like dictionary
+template <HasMsgPack T> struct pack<T> {
+    template <typename Stream> packer<Stream>& operator()(msgpack::packer<Stream>& o, T const& v) const
+    {
+        static_assert(std::is_default_constructible_v<T>, "T requires a default-constructor for MSGPACK");
+        const_cast<T&>(v).msgpack([&](auto&... args) {
+            auto static_checker = [&](auto &... value_args) {
+                static_assert(msgpack::MsgpackConstructible<T, decltype(value_args)...>,
+                              "T requires a constructor that can take the types listed in MSGPACK. "
+                              "Type or arg count mismatch, or member initializer constructor not available.");
+            };
+            std::apply(static_checker, drop_keys(std::tie(args...)));
+            msgpack::type::define_map<decltype(args)...>{args...}.msgpack_pack(o); });
+        return o;
+    }
+};
+
+} // namespace msgpack::adaptor

cpp/src/barretenberg/common/msgpack/variant_impl.hpp

@@ -0,0 +1,16 @@
+#pragma once
+// Note: Meant to only be included in compilation units that need msgpack
+#define MSGPACK_NO_BOOST
+#include <msgpack.hpp>
+#include <variant>
+
+namespace msgpack::adaptor {
+// writes std::variant to msgpack format (TODO should we read std::variant?)
+template<typename... T>
+struct pack<std::variant< T...>> {
+    auto &operator()(auto& o, std::variant<T...> const &variant) const {
+        std::visit([&o](auto &&arg) { msgpack::pack(o, arg); }, variant);
+        return o;
+    }
+};
+}

cpp/src/barretenberg/common/msgpack_impl.hpp

@@ -0,0 +1,4 @@
+#pragma once
+// Note: Meant to only be included in compilation units that need msgpack
+#include "msgpack/struct_map_impl.hpp"
+#include "msgpack/variant_impl.hpp"

cpp/src/barretenberg/common/serialize.hpp

@@ -40,6 +40,7 @@
 #ifndef __i386__
 __extension__ using uint128_t = unsigned __int128;
 #endif
+
 namespace serialize {
 // Basic integer read / write, to / from raw buffers.
 // Pointers to buffers are advanced by length of type.