feat(DRAFT): Protobuf bindings in Barretenberg

barretenberg/cpp/.gitignore

@@ -13,4 +13,5 @@ go*.tar.gz
 barretenberg_modules.dot
 barretenberg_modules.png
 src/barretenberg/bb/config.hpp
+src/barretenberg/dsl/acir_format/proto
 bench-out

barretenberg/cpp/CMakeLists.txt

@@ -161,6 +161,7 @@ include(cmake/benchmark.cmake)
 include(cmake/module.cmake)
 include(cmake/msgpack.cmake)
 include(cmake/lmdb.cmake)
+include(cmake/protobuf.cmake)
 
 # We do not need to bloat barretenberg.wasm with gzip functionality in a browser context as the browser can do this
 if (NOT WASM)

barretenberg/cpp/bootstrap.sh

@@ -143,6 +143,9 @@ export -f build_native build_darwin build_nodejs_module build_wasm build_wasm_th
 
 function build {
   echo_header "bb cpp build"
+
+  ./scripts/codegen_dsl.sh
+
   builds=(
     build_native
     build_nodejs_module

barretenberg/cpp/cmake/protobuf.cmake

@@ -0,0 +1,43 @@
+# The protobuf compiler should be installed on the builder machine;
+# the job here is to find where its C libraries are located, e.g.
+# under /usr/include/google/protobuf, so that we can include their headers.
+# See docs at https://cmake.org/cmake/help/latest/module/FindProtobuf.html
+# See source at https://fossies.org/linux/cmake/Modules/FindProtobuf.cmake or /usr/share/cmake-3.28/Modules/FindProtobuf.cmake
+
+
+FetchContent_Declare(protobuf
+  GIT_REPOSITORY https://github.com/protocolbuffers/protobuf.git
+  GIT_TAG        v29.3
+  SOURCE_SUBDIR  cmake
+  FIND_PACKAGE_ARGS NAMES protobuf
+)
+FetchContent_MakeAvailable(protobuf)
+
+include(FindProtobuf)
+
+#list(APPEND CMAKE_PREFIX_PATH "/usr/local/lib/cmake/protobuf")
+#set(Protobuf_DIR "/usr/local/lib/cmake/protobuf")
+
+# TODO: This errors saying it cannot find `Protobuf_INCLUDE_DIR`, unless we build protobuf from source.
+# find_package(Protobuf REQUIRED)
+find_package(Protobuf CONFIG REQUIRED)
+
+message("   --> PROTOBUF Found: ${Protobuf_FOUND}")
+message("   --> PROTOBUF VERSION: ${Protobuf_VERSION}")
+message("   --> PROTOBUF LIB: ${PROTOBUF_LIBRARIES}")
+message("   --> PROTOBUF INCLUDE: ${PROTOBUF_INCLUDE_DIRS}")
+
+# set(Protobuf_INCLUDE_DIR "/usr/include/google/protobuf")
+# set(Protobuf_INCLUDE_DIRS "/usr/include/google/protobuf")
+include_directories(${Protobuf_INCLUDE_DIRS})
+
+#include_directories("/usr/local/include")
+#include_directories("/usr/local/include/absl")
+#include_directories("/usr/include")
+
+
+# Add this to what needs to use protobuf:
+# target_link_libraries(<bin or lib> ${Protobuf_LIBRARIES})
+
+# TODO: We should be able to generate .pb.h and .pb.cc files from the .proto files
+# using the `protobuf_generate_cpp` command, instead of using `cpp/scripts/codegen_dsl.sh`

barretenberg/cpp/scripts/codegen_dsl.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -eu
+
+cd $(dirname $0)/..
+
+make -C src/barretenberg/dsl --no-print-directory

barretenberg/cpp/src/barretenberg/dsl/Makefile

@@ -0,0 +1,30 @@
+PROTO_SRC_DIR := ../../../../../noir/noir-repo/acvm-repo/acir/src/proto
+PROTO_DST_DIR := ./acir_format/proto
+
+# Using acir/program.proto instead of the top level program.proto so that we don't deserialize the Brillig part.
+PROTO_SCHEMAS += \
+	acir/program.proto \
+	acir/witness.proto \
+	acir/circuit.proto \
+	acir/native.proto
+
+PROTO_CPP := $(PROTO_SCHEMAS:.proto=.pb.h)
+PROTO_DST := $(patsubst %, $(PROTO_DST_DIR)/%, $(PROTO_CPP))
+
+.PHONY: all
+all: $(PROTO_DST)
+
+# Compile an ACIR protobuf schema to C++
+$(PROTO_DST_DIR)/%.pb.h: $(PROTO_SRC_DIR)/%.proto $(PROTO_DST_DIR) | .protoc
+	protoc --cpp_out $(PROTO_DST_DIR) -I $(PROTO_SRC_DIR) $<
+	sed -i 's/#include "acir\//#include "barretenberg\/dsl\/acir_format\/proto\/acir\//g' $(PROTO_DST_DIR)/$*.pb.*
+
+$(PROTO_DST_DIR):
+	mkdir -p $@
+
+.PHONY: .protoc
+.protoc:
+	@if [ -z "$$(which protoc)" ]; then \
+		echo "Please install the protobuf compiler"; \
+		exit 1; \
+	fi

barretenberg/cpp/src/barretenberg/dsl/README.md

@@ -6,11 +6,17 @@ This package adds support to use [ACIR](https://github.com/noir-lang/noir/tree/m
 
 There are two types of breaking serialization changes. One that alters the internal ACIR structure passed to barretenberg, and one that changes how we serialize the buffer passed to barretenberg.
 
+### `serde`
+
+We use `serde` to generate C++ code the domain classes used in Noir. These can be serialized using `bincode`, however this for format is generally
+not backwards or forwards compatible, which is why we use `protobuf` as well (see further below).
+
 1. Internal Structure Change
 
 Go to the ACVM `acir` crate and re-run the [serde reflection test](../../../../../noir/noir-repo/acvm-repo/acir/src/lib.rs#L51). Remember to comment out the hash check to write the updated C++ serde file. Copy that file into the `dsl` package's [serde folder](./acir_format/serde/) where you will see an `acir.hpp` file.
 
 You will have to update a couple things in the new `acir.hpp`:
+
 - Replace all `throw serde::deserialization_error` with `throw_or_abort`
 - The top-level struct (such as `Program`) will still use its own namespace for its internal fields. This extra `Program::` can be removed from the top-level `struct Program { .. }` object.
 
@@ -28,7 +34,49 @@ In the context of Aztec we need to regenerate all the artifacts in [noir-project
 
 The Aztec artifacts can be individually regenerated as well using `yarn test -u`.
 You can also run the command on the relevant workspaces, which at the time are the following:
-```
+
+```shell
 yarn workspace @aztec/stdlib test -u
 yarn workspace @aztec/protocol-contracts test -u
 ```
+
+### `protobuf`
+
+To combat the problems with breaking changes described under `serde` above, we can use [protobuf](https://protobuf.dev) to serialize the ACIR program
+and witness stack. The schemas are [defined](https://github.com/noir-lang/noir/tree/master/acvm-repo/acir/src/proto) in the Noir repository,
+and the C++ code is generated for them here by running the following command:
+
+```shell
+barretenberg/cpp/scripts/codegen_dsl.sh
+```
+
+The resulting `.pb.c` and `.pb.h` files are written to `./acir_format/proto/acir` but aren't checked into source control.
+
+For example:
+
+```console
+% barretenberg/cpp/scripts/codegen_dsl.sh
+mkdir -p acir_format/proto
+protoc --cpp_out ./acir_format/proto -I ../../../../../noir/noir-repo/acvm-repo/acir/src/proto ../../../../../noir/noir-repo/acvm-repo/acir/src/proto/acir/program.proto
+protoc --cpp_out ./acir_format/proto -I ../../../../../noir/noir-repo/acvm-repo/acir/src/proto ../../../../../noir/noir-repo/acvm-repo/acir/src/proto/acir/witness.proto
+protoc --cpp_out ./acir_format/proto -I ../../../../../noir/noir-repo/acvm-repo/acir/src/proto ../../../../../noir/noir-repo/acvm-repo/acir/src/proto/acir/circuit.proto
+protoc --cpp_out ./acir_format/proto -I ../../../../../noir/noir-repo/acvm-repo/acir/src/proto ../../../../../noir/noir-repo/acvm-repo/acir/src/proto/acir/native.proto
+% ls barretenberg/cpp/src/barretenberg/dsl/acir_format/proto/acir
+circuit.pb.cc  circuit.pb.h  native.pb.cc  native.pb.h  program.pb.cc  program.pb.h  witness.pb.cc  witness.pb.h
+```
+
+#### Installing Protobuf
+
+We need to install protobuf from source to work well with `cmake`:
+
+```
+# instead of `apt install libprotobuf-dev protobuf-compiler`
+# https://github.com/protocolbuffers/protobuf/blob/v29.3/cmake/README.md
+# using the same version as the Rust build
+git clone --branch v29.3 https://github.com/protocolbuffers/protobuf.git
+cd protobuf
+git submodule update --init --recursive
+cmake .
+cmake --build . --parallel 10
+sudo cmake --install .
+```

barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_to_constraint_buf.hpp

@@ -1,5 +1,6 @@
 #pragma once
 #include "acir_format.hpp"
+#include "convert/proto_to_serde.hpp"
 #include "serde/index.hpp"
 
 namespace acir_format {
@@ -25,4 +26,4 @@ AcirProgramStack get_acir_program_stack(std::string const& bytecode_path,
                                         std::string const& witness_path,
                                         uint32_t honk_recursion);
 #endif
-} // namespace acir_format
+} // namespace acir_format

barretenberg/cpp/src/barretenberg/dsl/acir_format/convert/proto_to_serde.hpp

@@ -0,0 +1,5 @@
+#include "../proto/acir/circuit.pb.h"
+#include "../proto/acir/native.pb.h"
+#include "../proto/acir/program.pb.h"
+#include "../proto/acir/witness.pb.h"
+#include "../serde/index.hpp"

bootstrap.sh

@@ -24,7 +24,7 @@ function encourage_dev_container {
 # Developers should probably use the dev container in /build-images to ensure the smoothest experience.
 function check_toolchains {
   # Check for various required utilities.
-  for util in jq parallel awk git curl; do
+  for util in jq parallel awk git curl protoc; do
     if ! command -v $util > /dev/null; then
       encourage_dev_container
       echo "Utility $util not found."