diff --git a/barretenberg/cpp/.gitignore b/barretenberg/cpp/.gitignore index 89bbad4d6b0f..7ce106f42be3 100644 --- a/barretenberg/cpp/.gitignore +++ b/barretenberg/cpp/.gitignore @@ -13,4 +13,5 @@ go*.tar.gz barretenberg_modules.dot barretenberg_modules.png src/barretenberg/bb/config.hpp +src/barretenberg/dsl/acir_format/proto bench-out diff --git a/barretenberg/cpp/CMakeLists.txt b/barretenberg/cpp/CMakeLists.txt index 109269b3f578..e5bd81fa6e0c 100644 --- a/barretenberg/cpp/CMakeLists.txt +++ b/barretenberg/cpp/CMakeLists.txt @@ -161,6 +161,7 @@ include(cmake/benchmark.cmake) include(cmake/module.cmake) include(cmake/msgpack.cmake) include(cmake/lmdb.cmake) +include(cmake/protobuf.cmake) # We do not need to bloat barretenberg.wasm with gzip functionality in a browser context as the browser can do this if (NOT WASM) diff --git a/barretenberg/cpp/bootstrap.sh b/barretenberg/cpp/bootstrap.sh index d0286daccd77..807793a71bf2 100755 --- a/barretenberg/cpp/bootstrap.sh +++ b/barretenberg/cpp/bootstrap.sh @@ -143,6 +143,9 @@ export -f build_native build_darwin build_nodejs_module build_wasm build_wasm_th function build { echo_header "bb cpp build" + + ./scripts/codegen_dsl.sh + builds=( build_native build_nodejs_module diff --git a/barretenberg/cpp/cmake/protobuf.cmake b/barretenberg/cpp/cmake/protobuf.cmake new file mode 100644 index 000000000000..5ea6b9a3f848 --- /dev/null +++ b/barretenberg/cpp/cmake/protobuf.cmake @@ -0,0 +1,43 @@ +# The protobuf compiler should be installed on the builder machine; +# the job here is to find where its C libraries are located, e.g. +# under /usr/include/google/protobuf, so that we can include their headers. +# See docs at https://cmake.org/cmake/help/latest/module/FindProtobuf.html +# See source at https://fossies.org/linux/cmake/Modules/FindProtobuf.cmake or /usr/share/cmake-3.28/Modules/FindProtobuf.cmake + + +FetchContent_Declare(protobuf + GIT_REPOSITORY https://github.com/protocolbuffers/protobuf.git + GIT_TAG v29.3 + SOURCE_SUBDIR cmake + FIND_PACKAGE_ARGS NAMES protobuf +) +FetchContent_MakeAvailable(protobuf) + +include(FindProtobuf) + +#list(APPEND CMAKE_PREFIX_PATH "/usr/local/lib/cmake/protobuf") +#set(Protobuf_DIR "/usr/local/lib/cmake/protobuf") + +# TODO: This errors saying it cannot find `Protobuf_INCLUDE_DIR`, unless we build protobuf from source. +# find_package(Protobuf REQUIRED) +find_package(Protobuf CONFIG REQUIRED) + +message(" --> PROTOBUF Found: ${Protobuf_FOUND}") +message(" --> PROTOBUF VERSION: ${Protobuf_VERSION}") +message(" --> PROTOBUF LIB: ${PROTOBUF_LIBRARIES}") +message(" --> PROTOBUF INCLUDE: ${PROTOBUF_INCLUDE_DIRS}") + +# set(Protobuf_INCLUDE_DIR "/usr/include/google/protobuf") +# set(Protobuf_INCLUDE_DIRS "/usr/include/google/protobuf") +include_directories(${Protobuf_INCLUDE_DIRS}) + +#include_directories("/usr/local/include") +#include_directories("/usr/local/include/absl") +#include_directories("/usr/include") + + +# Add this to what needs to use protobuf: +# target_link_libraries( ${Protobuf_LIBRARIES}) + +# TODO: We should be able to generate .pb.h and .pb.cc files from the .proto files +# using the `protobuf_generate_cpp` command, instead of using `cpp/scripts/codegen_dsl.sh` diff --git a/barretenberg/cpp/scripts/codegen_dsl.sh b/barretenberg/cpp/scripts/codegen_dsl.sh new file mode 100755 index 000000000000..d683f447dce2 --- /dev/null +++ b/barretenberg/cpp/scripts/codegen_dsl.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -eu + +cd $(dirname $0)/.. + +make -C src/barretenberg/dsl --no-print-directory diff --git a/barretenberg/cpp/src/barretenberg/dsl/Makefile b/barretenberg/cpp/src/barretenberg/dsl/Makefile new file mode 100644 index 000000000000..825ef5c179bf --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/dsl/Makefile @@ -0,0 +1,30 @@ +PROTO_SRC_DIR := ../../../../../noir/noir-repo/acvm-repo/acir/src/proto +PROTO_DST_DIR := ./acir_format/proto + +# Using acir/program.proto instead of the top level program.proto so that we don't deserialize the Brillig part. +PROTO_SCHEMAS += \ + acir/program.proto \ + acir/witness.proto \ + acir/circuit.proto \ + acir/native.proto + +PROTO_CPP := $(PROTO_SCHEMAS:.proto=.pb.h) +PROTO_DST := $(patsubst %, $(PROTO_DST_DIR)/%, $(PROTO_CPP)) + +.PHONY: all +all: $(PROTO_DST) + +# Compile an ACIR protobuf schema to C++ +$(PROTO_DST_DIR)/%.pb.h: $(PROTO_SRC_DIR)/%.proto $(PROTO_DST_DIR) | .protoc + protoc --cpp_out $(PROTO_DST_DIR) -I $(PROTO_SRC_DIR) $< + sed -i 's/#include "acir\//#include "barretenberg\/dsl\/acir_format\/proto\/acir\//g' $(PROTO_DST_DIR)/$*.pb.* + +$(PROTO_DST_DIR): + mkdir -p $@ + +.PHONY: .protoc +.protoc: + @if [ -z "$$(which protoc)" ]; then \ + echo "Please install the protobuf compiler"; \ + exit 1; \ + fi diff --git a/barretenberg/cpp/src/barretenberg/dsl/README.md b/barretenberg/cpp/src/barretenberg/dsl/README.md index 61fc0d4d05ae..209d16f2b5e1 100644 --- a/barretenberg/cpp/src/barretenberg/dsl/README.md +++ b/barretenberg/cpp/src/barretenberg/dsl/README.md @@ -6,11 +6,17 @@ This package adds support to use [ACIR](https://github.com/noir-lang/noir/tree/m There are two types of breaking serialization changes. One that alters the internal ACIR structure passed to barretenberg, and one that changes how we serialize the buffer passed to barretenberg. +### `serde` + +We use `serde` to generate C++ code the domain classes used in Noir. These can be serialized using `bincode`, however this for format is generally +not backwards or forwards compatible, which is why we use `protobuf` as well (see further below). + 1. Internal Structure Change Go to the ACVM `acir` crate and re-run the [serde reflection test](../../../../../noir/noir-repo/acvm-repo/acir/src/lib.rs#L51). Remember to comment out the hash check to write the updated C++ serde file. Copy that file into the `dsl` package's [serde folder](./acir_format/serde/) where you will see an `acir.hpp` file. You will have to update a couple things in the new `acir.hpp`: + - Replace all `throw serde::deserialization_error` with `throw_or_abort` - The top-level struct (such as `Program`) will still use its own namespace for its internal fields. This extra `Program::` can be removed from the top-level `struct Program { .. }` object. @@ -28,7 +34,49 @@ In the context of Aztec we need to regenerate all the artifacts in [noir-project The Aztec artifacts can be individually regenerated as well using `yarn test -u`. You can also run the command on the relevant workspaces, which at the time are the following: -``` + +```shell yarn workspace @aztec/stdlib test -u yarn workspace @aztec/protocol-contracts test -u ``` + +### `protobuf` + +To combat the problems with breaking changes described under `serde` above, we can use [protobuf](https://protobuf.dev) to serialize the ACIR program +and witness stack. The schemas are [defined](https://github.com/noir-lang/noir/tree/master/acvm-repo/acir/src/proto) in the Noir repository, +and the C++ code is generated for them here by running the following command: + +```shell +barretenberg/cpp/scripts/codegen_dsl.sh +``` + +The resulting `.pb.c` and `.pb.h` files are written to `./acir_format/proto/acir` but aren't checked into source control. + +For example: + +```console +% barretenberg/cpp/scripts/codegen_dsl.sh +mkdir -p acir_format/proto +protoc --cpp_out ./acir_format/proto -I ../../../../../noir/noir-repo/acvm-repo/acir/src/proto ../../../../../noir/noir-repo/acvm-repo/acir/src/proto/acir/program.proto +protoc --cpp_out ./acir_format/proto -I ../../../../../noir/noir-repo/acvm-repo/acir/src/proto ../../../../../noir/noir-repo/acvm-repo/acir/src/proto/acir/witness.proto +protoc --cpp_out ./acir_format/proto -I ../../../../../noir/noir-repo/acvm-repo/acir/src/proto ../../../../../noir/noir-repo/acvm-repo/acir/src/proto/acir/circuit.proto +protoc --cpp_out ./acir_format/proto -I ../../../../../noir/noir-repo/acvm-repo/acir/src/proto ../../../../../noir/noir-repo/acvm-repo/acir/src/proto/acir/native.proto +% ls barretenberg/cpp/src/barretenberg/dsl/acir_format/proto/acir +circuit.pb.cc circuit.pb.h native.pb.cc native.pb.h program.pb.cc program.pb.h witness.pb.cc witness.pb.h +``` + +#### Installing Protobuf + +We need to install protobuf from source to work well with `cmake`: + +``` +# instead of `apt install libprotobuf-dev protobuf-compiler` +# https://github.com/protocolbuffers/protobuf/blob/v29.3/cmake/README.md +# using the same version as the Rust build +git clone --branch v29.3 https://github.com/protocolbuffers/protobuf.git +cd protobuf +git submodule update --init --recursive +cmake . +cmake --build . --parallel 10 +sudo cmake --install . +``` diff --git a/barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_to_constraint_buf.hpp b/barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_to_constraint_buf.hpp index 4560c8245445..b916d3bd8609 100644 --- a/barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_to_constraint_buf.hpp +++ b/barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_to_constraint_buf.hpp @@ -1,5 +1,6 @@ #pragma once #include "acir_format.hpp" +#include "convert/proto_to_serde.hpp" #include "serde/index.hpp" namespace acir_format { @@ -25,4 +26,4 @@ AcirProgramStack get_acir_program_stack(std::string const& bytecode_path, std::string const& witness_path, uint32_t honk_recursion); #endif -} // namespace acir_format \ No newline at end of file +} // namespace acir_format diff --git a/barretenberg/cpp/src/barretenberg/dsl/acir_format/convert/proto_to_serde.cpp b/barretenberg/cpp/src/barretenberg/dsl/acir_format/convert/proto_to_serde.cpp new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/barretenberg/cpp/src/barretenberg/dsl/acir_format/convert/proto_to_serde.hpp b/barretenberg/cpp/src/barretenberg/dsl/acir_format/convert/proto_to_serde.hpp new file mode 100644 index 000000000000..16fc17b6ca5d --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/dsl/acir_format/convert/proto_to_serde.hpp @@ -0,0 +1,5 @@ +#include "../proto/acir/circuit.pb.h" +#include "../proto/acir/native.pb.h" +#include "../proto/acir/program.pb.h" +#include "../proto/acir/witness.pb.h" +#include "../serde/index.hpp" diff --git a/bootstrap.sh b/bootstrap.sh index 99ddacf810e9..177b96706603 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -24,7 +24,7 @@ function encourage_dev_container { # Developers should probably use the dev container in /build-images to ensure the smoothest experience. function check_toolchains { # Check for various required utilities. - for util in jq parallel awk git curl; do + for util in jq parallel awk git curl protoc; do if ! command -v $util > /dev/null; then encourage_dev_container echo "Utility $util not found."