diff --git a/barretenberg/README.md b/barretenberg/README.md index c809fb6a9f66..ece95771370f 100644 --- a/barretenberg/README.md +++ b/barretenberg/README.md @@ -477,7 +477,7 @@ To do a single benchmark you can do e.g. `IVC_BENCH=amm-add-liquidity ./bootstrap.sh bench_ivc origin/master~3` If one doesn't provide the commit, it generates these IVC inputs on the fly (depends on yarn-project having been bootstrapped). -To use these inputs manually, just abort after input download and run ClientIVC proving with --input runtime_stack on those inputs (stored in `yarn-project/end-to-end/example-app-ivc-inputs-out`). +To use these inputs manually, just abort after input download and run ClientIVC proving on those inputs (stored in `yarn-project/end-to-end/example-app-ivc-inputs-out`). #### Using Tracy to Profile Memory/CPU/Gate Counts @@ -532,7 +532,7 @@ By default, the barretenberg.wasm.gz that is used by bb.js (aka barretenberg/ts) One can get stack traces working from WASM by running root level ./bootstrap.sh (or otherwise building what you need) and then doing: ``` cmake --build --preset wasm-threads --target barretenberg-debug.wasm.gz -mv build-wasm-threads/barretenberg-debug.wasm.gz build-wasm-threads/barretenberg.wasm.gz +cp build-wasm-threads/bin/barretenberg-debug.wasm.gz ../ts/dest/node/barretenberg_wasm/barretenberg-threads.wasm.gz ``` This will mean that any yarn-project or barretenberg/ts tests that run will get stack traces with function names. @@ -541,5 +541,5 @@ To get more detailed information use the following (NOTE: takes >10 minutes!): ``` cmake --preset wasm-threads-dbg cmake --build --preset wasm-threads-dbg --target barretenberg-debug.wasm.gz -mv build-wasm-threads-dbg/barretenberg-debug.wasm.gz build-wasm-threads/barretenberg.wasm.gz +cp build-wasm-threads-dbg/bin/barretenberg-debug.wasm.gz ../ts/dest/node/barretenberg_wasm/barretenberg-threads.wasm.gz ``` diff --git a/barretenberg/acir_tests/bootstrap.sh b/barretenberg/acir_tests/bootstrap.sh index 03816b07306c..b15cfba94721 100755 --- a/barretenberg/acir_tests/bootstrap.sh +++ b/barretenberg/acir_tests/bootstrap.sh @@ -215,9 +215,6 @@ function run_benchmark { # TODO(https://github.com/AztecProtocol/barretenberg/issues/1254): More complete testing, including failure tests function bench { - # TODO(https://github.com/AztecProtocol/barretenberg/issues/1265) fix acir benchmarking - # LOG_FILE=bench-acir.jsonl ./bench_acir_tests.sh - export HARDWARE_CONCURRENCY=16 rm -rf bench-out && mkdir -p bench-out diff --git a/barretenberg/acir_tests/flows/prove_then_verify.sh b/barretenberg/acir_tests/flows/prove_then_verify.sh index 6d738105e5da..76585ee9753b 100755 --- a/barretenberg/acir_tests/flows/prove_then_verify.sh +++ b/barretenberg/acir_tests/flows/prove_then_verify.sh @@ -6,23 +6,6 @@ BFLAG="-b ./target/program.json" FLAGS="-c $CRS_PATH ${VERBOSE:+-v}" [ "${RECURSIVE}" = "true" ] && FLAGS+=" --recursive" -# TODO: Use this when client ivc support write_vk. Currently it keeps its own flow. -# case ${SYS:-} in -# "") -# prove_cmd=prove -# verify_cmd=verify -# ;; -# "client_ivc") -# prove_cmd=prove -# verify_cmd=verify -# flags+=" --scheme client_ivc --input_type ${INPUT_TYPE:-single_circuit}" -# ;; -# *) -# prove_cmd=prove_$SYS -# verify_cmd=verify_$SYS -# ;; -# esac - # Test we can perform the proof/verify flow. # This ensures we test independent pk construction through real/garbage witness data paths. # We use process substitution pipes to avoid temporary files, which need cleanup, and can collide with parallelism. diff --git a/barretenberg/acir_tests/flows/prove_then_verify_client_ivc.sh b/barretenberg/acir_tests/flows/prove_then_verify_client_ivc.sh index 5b27b828a81c..957867df46f6 100755 --- a/barretenberg/acir_tests/flows/prove_then_verify_client_ivc.sh +++ b/barretenberg/acir_tests/flows/prove_then_verify_client_ivc.sh @@ -1,17 +1,8 @@ #!/usr/bin/env bash # Create intermediate state in a directory. Uses a temp dir to ensure parallel safe and cleans up on exit. +# TODO this is unused set -eux -INFLAG=${INPUT_TYPE:---input_type runtime_stack} - -if [ "$INFLAG" = "--input_type runtime_stack" ]; then - BFLAG=target/acir.msgpack - WFLAG=target/witness.msgpack -else - BFLAG=target/program.json - WFLAG=target/witness.gz -fi - CRS_PATH=${CRS_PATH:-$HOME/.bb-crs} BIN=$(realpath ${BIN:-../cpp/build/bin/bb}) @@ -23,6 +14,6 @@ trap "rm -rf $outdir" EXIT flags="--scheme client_ivc -c $CRS_PATH ${VERBOSE:+-v}" parallel ::: \ - "$BIN prove $flags -b $BFLAG -w $WFLAG $INFLAG --output_format proof -o $outdir" \ - "$BIN write_vk $flags -b $BFLAG $INFLAG --verifier_type ivc -o $outdir" + "$BIN prove $flags -i target/ivc-inputs.msgpack $INFLAG --output_format proof -o $outdir" \ + "$BIN write_vk $flags -i target/ivc-inputs.msgpack $INFLAG --verifier_type ivc -o $outdir" $BIN verify $flags -p $outdir/proof -k $outdir/vk diff --git a/barretenberg/acir_tests/flows/sol_honk.sh b/barretenberg/acir_tests/flows/sol_honk.sh index eeec6940b08c..7d58daf57a64 100755 --- a/barretenberg/acir_tests/flows/sol_honk.sh +++ b/barretenberg/acir_tests/flows/sol_honk.sh @@ -4,7 +4,7 @@ set -eu VFLAG=${VERBOSE:+-v} BFLAG="-b ./target/program.json" FLAGS="-c $CRS_PATH $VFLAG --scheme ultra_honk" -PROVE_FLAGS="$FLAGS $BFLAG --oracle_hash keccak --output_format bytes_and_fields --write_vk --input_type single_circuit" +PROVE_FLAGS="$FLAGS $BFLAG --oracle_hash keccak --output_format bytes_and_fields --write_vk" VERIFY_FLAGS="$FLAGS --oracle_hash keccak" outdir=$(mktemp -d) diff --git a/barretenberg/cpp/.clangd b/barretenberg/cpp/.clangd index c38612ce84f4..596fea83da79 100644 --- a/barretenberg/cpp/.clangd +++ b/barretenberg/cpp/.clangd @@ -1,5 +1,7 @@ CompileFlags: # Tweak the parse settings - Remove: -fconstexpr-ops-limit=* -Wfatal-errors + Remove: + - "-fconstexpr-ops-limit=*" + - "-Wfatal-errors" --- # Applies all barretenberg source files If: @@ -51,6 +53,8 @@ Diagnostics: - readability-container-data-pointer # Many hits; potential for false positives. - cppcoreguidelines-pro-type-member-init + # We do use this, so no point banning it. The name is a warning enough. + - cppcoreguidelines-pro-type-reinterpret-cast # As cryptographers, we often think of bools as 0/1 values; would bloat code in some places. - modernize-use-bool-literals # Triggers on every TYPED_TEST @@ -71,6 +75,12 @@ Diagnostics: - modernize-use-auto # Not worth rewriting. Redundant types are often clearer. - modernize-return-braced-init-list + # Not worth rewriting. Also some gotchas with mac not accepting emplaces of C++20 default constructors. + - modernize-use-emplace + # We only really use macros where a constexpr would NOT be able to replace its functionality. + - cppcoreguidelines-macro-usage + # Opinionated, we sometimes just want to use a default-derived constructor. + - modernize-use-designated-initializers --- # this divider is necessary # Disable some checks for Google Test/Bench diff --git a/barretenberg/cpp/CMakePresets.json b/barretenberg/cpp/CMakePresets.json index 6be54bbb347b..ab5f82d88ba7 100644 --- a/barretenberg/cpp/CMakePresets.json +++ b/barretenberg/cpp/CMakePresets.json @@ -471,7 +471,7 @@ }, { "name": "wasm-threads-assert", - "displayName": "Build for WASM with multithreading and and asserts", + "displayName": "Build for WASM with multithreading and asserts", "binaryDir": "build-wasm-threads-assert", "description": "Build with wasi-sdk with asserts", "inherits": "wasm-threads", diff --git a/barretenberg/cpp/bootstrap.sh b/barretenberg/cpp/bootstrap.sh index df812084737f..0a38b9c5850e 100755 --- a/barretenberg/cpp/bootstrap.sh +++ b/barretenberg/cpp/bootstrap.sh @@ -220,10 +220,9 @@ function bench { --benchmark_filter="construct_proof_ultrahonk_power_of_2/20$" } function ultra_honk_wasm { - wasmtime run --env HARDWARE_CONCURRENCY --env IGNITION_CRS_PATH --env GRUMPKIN_CRS_PATH -Wthreads=y -Sthreads=y --dir=. \ - ./build-wasm-threads/bin/ultra_honk_bench \ - --benchmark_out=./bench-out/ultra_honk_wasm.json \ - --benchmark_filter="construct_proof_ultrahonk_power_of_2/20$" + scripts/wasmtime.sh ./build-wasm-threads/bin/ultra_honk_bench \ + --benchmark_out=./bench-out/ultra_honk_wasm.json \ + --benchmark_filter="construct_proof_ultrahonk_power_of_2/20$" } # Client IVC @@ -243,10 +242,9 @@ function bench { --benchmark_filter="ClientIVCBench/Full/6$" } function client_ivc_wasm { - wasmtime run --env HARDWARE_CONCURRENCY --env IGNITION_CRS_PATH --env GRUMPKIN_CRS_PATH -Wthreads=y -Sthreads=y --dir=. \ - ./build-wasm-threads/bin/client_ivc_bench \ - --benchmark_out=./bench-out/client_ivc_wasm.json \ - --benchmark_filter="ClientIVCBench/Full/6$" + scripts/wasmtime.sh ./build-wasm-threads/bin/client_ivc_bench \ + --benchmark_out=./bench-out/client_ivc_wasm.json \ + --benchmark_filter="ClientIVCBench/Full/6$" } function run_benchmark { @@ -300,7 +298,7 @@ case "$cmd" in # Takes an optional master commit to download them from. Otherwise, downloads from latest master commit. git fetch origin master - build the benchmarked benches + # build the benchmarked benches parallel --line-buffered --tag -v denoise ::: \ "build_preset $native_preset --target bb_cli_bench" \ "build_preset wasm-threads --target bb_cli_bench" diff --git a/barretenberg/cpp/scripts/ci_benchmark_ivc_flows.sh b/barretenberg/cpp/scripts/ci_benchmark_ivc_flows.sh index 404e67c4bb57..1caf36c1faaa 100755 --- a/barretenberg/cpp/scripts/ci_benchmark_ivc_flows.sh +++ b/barretenberg/cpp/scripts/ci_benchmark_ivc_flows.sh @@ -4,21 +4,23 @@ source $(git rev-parse --show-toplevel)/ci3/source if [[ $# -ne 2 ]]; then - echo "Usage: $0 " + echo "Usage: $0 " exit 1 fi -export input_folder="$1" -output_folder="$2" - cd .. +export input_folder="$1" +benchmark_output="$2" echo_header "bb ivc flow bench" export HARDWARE_CONCURRENCY=16 export IGNITION_CRS_PATH=./srs_db/ignition export GRUMPKIN_CRS_PATH=./srs_db/grumpkin +export native_preset=${NATIVE_PRESET:-clang16-assert} +export native_build_dir=$(scripts/cmake/preset-build-dir $native_preset) -rm -rf bench-out/ivc && mkdir -p bench-out/ivc +mkdir -p bench-out +rm -rf bench-out/ivc-* function verify_ivc_flow { local flow="$1" @@ -28,10 +30,10 @@ function verify_ivc_flow { # TODO(AD): Checking which one would be good, but there isn't too much that can go wrong here. set +e echo_stderr "Private verify." - ./build/bin/bb verify --scheme client_ivc -p "$proof" -k ../../yarn-project/bb-prover/artifacts/private-civc-vk 1>&2 + "./$native_build_dir/bin/bb" verify --scheme client_ivc -p "$proof" -k ../../yarn-project/bb-prover/artifacts/private-civc-vk 1>&2 local private_result=$? echo_stderr "Private verify: $private_result." - ./build/bin/bb verify --scheme client_ivc -p "$proof" -k ../../yarn-project/bb-prover/artifacts/public-civc-vk 1>&2 + "./$native_build_dir/bin/bb" verify --scheme client_ivc -p "$proof" -k ../../yarn-project/bb-prover/artifacts/public-civc-vk 1>&2 local public_result=$? echo_stderr "Public verify: $public_result." if [[ $private_result -eq $public_result ]]; then @@ -44,86 +46,71 @@ function verify_ivc_flow { fi } -function client_ivc_flow_native { - set -eu - local flow=$1 - local flow_folder="$input_folder/$flow" - local start=$(date +%s%N) - mkdir -p "bench-out/$flow-proof-files" - export MEMUSAGE_OUT="bench-out/$flow-proof-files/peak-memory-native-mb.txt" - - function bb_cli_bench_native { - export MAIN_ARGS="$*" - memusage ./build/bin/bb_cli_bench \ - --benchmark_out=bench-out/$flow-proof-files/op-counts.json \ - --benchmark_out_format=json || { - echo "bb_cli_bench failed with args: $*" +function run_bb_cli_bench { + local runtime="$1" + local output="$2" + local args="$3" + export MAIN_ARGS="$args" + + if [[ "$runtime" == "native" ]]; then + memusage "./$native_build_dir/bin/bb_cli_bench" \ + --benchmark_out=$output/op-counts.json \ + --benchmark_out_format=json || { + echo "bb_cli_bench native failed with args: $args" exit 1 } - } - - bb_cli_bench_native prove -o "bench-out/$flow-proof-files" -b "$flow_folder/acir.msgpack" -w "$flow_folder/witnesses.msgpack" --scheme client_ivc --input_type runtime_stack - local end=$(date +%s%N) - local elapsed_ns=$(( end - start )) - local elapsed_ms=$(( elapsed_ns / 1000000 )) - local memory_taken_mb=$(cat "$MEMUSAGE_OUT") - echo "$flow (native) has proven in $((elapsed_ms / 1000))s and peak memory of ${memory_taken_mb}MB." - dump_fail "verify_ivc_flow $flow bench-out/$flow-proof-files/proof" - echo "$flow (native) has verified." - cat > "./bench-out/ivc/$flow-ivc-native.json" < "./bench-out/ivc/$flow-ivc-wasm.json" < "$output/benchmarks.json" < $output_folder/ivc-bench.json + ./bench-out/*/benchmarks.json \ + > $benchmark_output/ivc-bench.json diff --git a/barretenberg/cpp/scripts/cmake/preset-build-dir b/barretenberg/cpp/scripts/cmake/preset-build-dir new file mode 100755 index 000000000000..4cb2fa57122d --- /dev/null +++ b/barretenberg/cpp/scripts/cmake/preset-build-dir @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# This script is used to find the binary directory for a given CMake preset. +# It reads the CMakepresets.json file and traverses the inheritance tree to find the binary directory. + +source $(git rev-parse --show-toplevel)/ci3/source + +preset="${1:-}" + +[[ -z "$preset" ]] && { echo "Usage: $0 "; exit 1; } + +declare -A preset_to_dir preset_to_inherits + +# Read a map of name => inherits, name => binaryDir +while IFS=$'\t' read -r name inherits binaryDir; do + preset_to_inherits["$name"]="$inherits" + preset_to_dir["$name"]="$binaryDir" +done < <( + jq -r ' + .configurePresets[]? + | [ + .name, + .inherits // "", + .binaryDir // "" + ] + | @tsv + ' ../../CMakePresets.json +) + +function get_binary_dir { + local name="$1" + [[ -n "${preset_to_dir[$name]:-}" ]] && [[ -n "${preset_to_dir[$name]}" ]] && { echo "${preset_to_dir[$name]}"; return; } + for parent in ${preset_to_inherits[$name]}; do + result="$(get_binary_dir "$parent")" + [[ -n "$result" ]] && { preset_to_dir["$name"]="$result"; echo "$result"; return; } + done + echo "Error: no binaryDir found for preset '$name' or its parents." >&2 + exit 1 +} + +result="$(get_binary_dir "$preset")" +echo "$result" diff --git a/barretenberg/cpp/scripts/generate_civc_vks.sh b/barretenberg/cpp/scripts/generate_civc_vks.sh index 4f6ee2224171..6115b12ef2ae 100755 --- a/barretenberg/cpp/scripts/generate_civc_vks.sh +++ b/barretenberg/cpp/scripts/generate_civc_vks.sh @@ -19,7 +19,7 @@ export GRUMPKIN_CRS_PATH=./srs_db/grumpkin function write_vk { set -eu local flow_folder="$input_folder/$1" - ./build/bin/bb write_vk --scheme client_ivc -v -b "$flow_folder/acir.msgpack" --input_type runtime_stack --verifier_type ivc -o "$flow_folder" + ./build/bin/bb write_vk --scheme client_ivc -v --ivc_inputs_path "$flow_folder/ivc-inputs.msgpack" --verifier_type ivc -o "$flow_folder" } export -f write_vk diff --git a/barretenberg/cpp/scripts/google-bench/summarize-op-counts b/barretenberg/cpp/scripts/google-bench/summarize-op-counts index a5ba823722a9..3eef1a489bdf 100755 --- a/barretenberg/cpp/scripts/google-bench/summarize-op-counts +++ b/barretenberg/cpp/scripts/google-bench/summarize-op-counts @@ -3,15 +3,21 @@ set -eu op_count_json="$1" +num_lines="${2:-10}" if [ -z "$op_count_json" ]; then - echo "Usage: $0 " + echo "Usage: $0 [number of lines]" exit 1 fi +# Get the time in seconds, truncating to 3 decimal places +jq -r '.benchmarks[0].real_time / 1e3' $op_count_json | + awk '{printf "%.9f\n", $1}' | + sed 's/^\./0./' | + xargs -I{} echo -e "{}\tTOTAL(s)" + jq -r '.benchmarks[0] | to_entries[] | select(.key | endswith("(t)")) | {op: (.key | rtrimstr("(t)")), seconds: (.value / 1e9)} | - [.op, .seconds] | @tsv' $op_count_json | - sort -k2 -nr | - head -10 | - awk '{printf "%-50s %10.2f seconds\n", $1, $2}' + [.seconds,.op] | @tsv' $op_count_json | + sort -k1 -nr | + head -$num_lines diff --git a/barretenberg/cpp/scripts/wasmtime.sh b/barretenberg/cpp/scripts/wasmtime.sh index 806aedf26318..23a33412147b 100755 --- a/barretenberg/cpp/scripts/wasmtime.sh +++ b/barretenberg/cpp/scripts/wasmtime.sh @@ -1,3 +1,17 @@ #!/usr/bin/env bash +# Helper for passing environment variables to wasm and common config. +# Allows accessing ~/.bb-crs and ./ (more can be added as parameters to this script). set -eu -exec wasmtime run -Wthreads=y -Sthreads=y --env IGNITION_CRS_PATH --env GRUMPKIN_CRS_PATH --env HARDWARE_CONCURRENCY=16 --env WASM_BACKTRACE_DETAILS=1 --env HOME --env MAIN_ARGS "$@" + +exec wasmtime run \ + -Wthreads=y \ + -Sthreads=y \ + --env HARDWARE_CONCURRENCY \ + --env WASM_BACKTRACE_DETAILS=1 \ + --env HOME \ + ${MAIN_ARGS:+--env MAIN_ARGS} \ + --env IGNITION_CRS_PATH \ + --env GRUMPKIN_CRS_PATH \ + --dir=$HOME/.bb-crs \ + --dir=. \ + "$@" diff --git a/barretenberg/cpp/src/barretenberg/acir_formal_proofs/acir_loader.cpp b/barretenberg/cpp/src/barretenberg/acir_formal_proofs/acir_loader.cpp index 883ecba688e8..8dad40e6c5b3 100644 --- a/barretenberg/cpp/src/barretenberg/acir_formal_proofs/acir_loader.cpp +++ b/barretenberg/cpp/src/barretenberg/acir_formal_proofs/acir_loader.cpp @@ -1,9 +1,9 @@ #include "acir_loader.hpp" #include "barretenberg/dsl/acir_format/acir_format.hpp" #include "barretenberg/dsl/acir_format/acir_to_constraint_buf.hpp" +#include "barretenberg/serialize/msgpack_impl.hpp #include "barretenberg/smt_verification/circuit/ultra_circuit.hpp" #include "barretenberg/smt_verification/terms/term.hpp" -#include "msgpack/v3/sbuffer_decl.hpp" #include #include #include @@ -72,4 +72,4 @@ smt_circuit::UltraCircuit AcirToSmtLoader::get_integer_smt_circuit(smt_solver::S { smt_circuit::CircuitSchema circuit_info = smt_circuit_schema::unpack_from_buffer(this->circuit_buf); return smt_circuit::UltraCircuit(circuit_info, solver, smt_terms::TermType::ITerm); -} \ No newline at end of file +} diff --git a/barretenberg/cpp/src/barretenberg/acir_formal_proofs/acir_loader.hpp b/barretenberg/cpp/src/barretenberg/acir_formal_proofs/acir_loader.hpp index c05bcb9af775..f5c6ccf23e1f 100644 --- a/barretenberg/cpp/src/barretenberg/acir_formal_proofs/acir_loader.hpp +++ b/barretenberg/cpp/src/barretenberg/acir_formal_proofs/acir_loader.hpp @@ -1,7 +1,7 @@ #pragma once #include "barretenberg/dsl/acir_format/acir_format.hpp" +#include "barretenberg/serialize/msgpack.hpp" #include "barretenberg/smt_verification/circuit/ultra_circuit.hpp" -#include "msgpack/v3/sbuffer_decl.hpp" #include #include #include @@ -99,4 +99,4 @@ class AcirToSmtLoader { std::vector acir_program_buf; ///< Buffer containing the raw ACIR program data read from file acir_format::AcirFormat constraint_system; ///< The parsed constraint system from the ACIR program msgpack::sbuffer circuit_buf; ///< Buffer for circuit serialization using MessagePack -}; \ No newline at end of file +}; diff --git a/barretenberg/cpp/src/barretenberg/api/api.hpp b/barretenberg/cpp/src/barretenberg/api/api.hpp index a78a1c2d967b..e554755429fb 100644 --- a/barretenberg/cpp/src/barretenberg/api/api.hpp +++ b/barretenberg/cpp/src/barretenberg/api/api.hpp @@ -17,7 +17,6 @@ class API { uint32_t honk_recursion{ 0 }; // flag that differentiates between different recursion modes; deprecated? bool ipa_accumulation{ false }; // indicate whether the command is doing IPA proof aggregation std::string scheme; // the proving system or IVC scheme - std::string input_type; // is the input bytecode a single circuit or a stack of circuits? std::string oracle_hash_type; // which hash function does the prover use as a random oracle? std::string output_format; // output bytes, fields, both, or a msgpack buffer of fields std::string verifier_type; // is a verification key for use a single circuit verifier (e.g. a SNARK or folding @@ -37,7 +36,6 @@ class API { << " honk_recursion: " << flags.honk_recursion << "\n" << " ipa_accumulation: " << flags.ipa_accumulation << "\n" << " scheme: " << flags.scheme << "\n" - << " input_type: " << flags.input_type << "\n" << " oracle_hash_type: " << flags.oracle_hash_type << "\n" << " output_format: " << flags.output_format << "\n" << " verifier_type: " << flags.verifier_type << "\n" @@ -53,21 +51,11 @@ class API { const std::filesystem::path& bytecode_path, const std::filesystem::path& witness_path) = 0; - virtual void prove(const Flags& flags, - const std::filesystem::path& bytecode_path, - const std::filesystem::path& witness_path, - const std::filesystem::path& output_dir) = 0; - virtual bool verify(const Flags& flags, const std::filesystem::path& public_inputs_path, const std::filesystem::path& proof_path, const std::filesystem::path& vk_path) = 0; - // TODO(https://github.com/AztecProtocol/barretenberg/issues/1255): deprecate or include as flag to `prove` - virtual bool prove_and_verify(const Flags& flags, - const std::filesystem::path& bytecode_path, - const std::filesystem::path& witness_path) = 0; - virtual void write_vk(const Flags& flags, const std::filesystem::path& bytecode_path, const std::filesystem::path& output_path) = 0; diff --git a/barretenberg/cpp/src/barretenberg/api/api_avm.cpp b/barretenberg/cpp/src/barretenberg/api/api_avm.cpp index ab1df1fece22..f7efeb435d47 100644 --- a/barretenberg/cpp/src/barretenberg/api/api_avm.cpp +++ b/barretenberg/cpp/src/barretenberg/api/api_avm.cpp @@ -46,7 +46,7 @@ void avm_prove(const std::filesystem::path&, const std::filesystem::path&, const vinfo("num of pub inputs: ", static_cast(vk_as_fields[1])); const auto to_json = [](const std::vector& data) { - return format("[", join(map(data, [](auto fr) { return format("\"", fr, "\""); })), "]"); + return format("[", join(transform::map(data, [](auto fr) { return format("\"", fr, "\""); })), "]"); }; std::string vk_json = to_json(vk_as_fields); const auto proof_path = output_path / "proof"; diff --git a/barretenberg/cpp/src/barretenberg/api/api_client_ivc.cpp b/barretenberg/cpp/src/barretenberg/api/api_client_ivc.cpp index 1c131ba77f88..ff64ee5af076 100644 --- a/barretenberg/cpp/src/barretenberg/api/api_client_ivc.cpp +++ b/barretenberg/cpp/src/barretenberg/api/api_client_ivc.cpp @@ -4,12 +4,13 @@ #include "barretenberg/api/log.hpp" #include "barretenberg/api/write_prover_output.hpp" #include "barretenberg/client_ivc/mock_circuit_producer.hpp" +#include "barretenberg/client_ivc/private_execution_steps.hpp" #include "barretenberg/common/map.hpp" #include "barretenberg/common/throw_or_abort.hpp" #include "barretenberg/common/try_catch_shim.hpp" #include "barretenberg/dsl/acir_format/acir_to_constraint_buf.hpp" #include "barretenberg/dsl/acir_format/ivc_recursion_constraint.hpp" -#include "libdeflate.h" +#include "barretenberg/serialize/msgpack.hpp" #include namespace bb { @@ -19,7 +20,7 @@ acir_format::WitnessVector witness_map_to_witness_vector(std::map decompress(const void* bytes, size_t size) -{ - std::vector content; - // initial size guess - content.resize(1024ULL * 128ULL); - for (;;) { - auto decompressor = std::unique_ptr{ - libdeflate_alloc_decompressor(), libdeflate_free_decompressor - }; - size_t actual_size = 0; - libdeflate_result decompress_result = libdeflate_gzip_decompress( - decompressor.get(), bytes, size, std::data(content), std::size(content), &actual_size); - if (decompress_result == LIBDEFLATE_INSUFFICIENT_SPACE) { - // need a bigger buffer - content.resize(content.size() * 2); - continue; - } - if (decompress_result == LIBDEFLATE_BAD_DATA) { - THROW std::invalid_argument("bad gzip data in bb main"); - } - content.resize(actual_size); - break; - } - return content; -} - -template std::shared_ptr read_to_shared_ptr(const std::filesystem::path& path) -{ - return std::make_shared(from_buffer(read_file(path))); -}; - -template T unpack_from_file(const std::filesystem::path& filename) -{ - std::ifstream fin; - fin.open(filename, std::ios::ate | std::ios::binary); - if (!fin.is_open()) { - THROW std::invalid_argument("file not found"); - } - if (fin.tellg() == -1) { - THROW std::invalid_argument("something went wrong"); - } - - size_t fsize = static_cast(fin.tellg()); - fin.seekg(0, std::ios_base::beg); - - T result; - char* encoded_data = new char[fsize]; - fin.read(encoded_data, static_cast(fsize)); - msgpack::unpack(encoded_data, fsize).get().convert(result); - return result; -} - /** * @brief Compute and write to file a MegaHonk VK for a circuit to be accumulated in the IVC * @note This method differes from write_vk_honk in that it handles kernel circuits which require special @@ -133,22 +82,22 @@ void write_standalone_vk(const std::string& output_data_type, write(to_write, output_data_type, "vk", output_path); } -size_t get_num_public_inputs_in_final_circuit(const std::filesystem::path& bytecode_path) +size_t get_num_public_inputs_in_final_circuit(const std::filesystem::path& input_path) { using namespace acir_format; - - const std::string bincode = unpack_from_file>(bytecode_path).back(); - const std::vector bincode_buf = decompress(bincode.data(), bincode.size()); - const AcirFormat constraints = circuit_buf_to_acir_format(bincode_buf, /*honk_recursion=*/0); + auto steps = PrivateExecutionStepRaw::load_and_decompress(input_path); + const PrivateExecutionStepRaw& last_step = steps.back(); + std::vector bytecode_buf(last_step.bytecode.begin(), last_step.bytecode.end()); + const AcirFormat constraints = circuit_buf_to_acir_format(bytecode_buf, /*honk_recursion=*/0); return constraints.public_inputs.size(); } -void write_vk_for_ivc(const std::string& bytecode_path, const std::filesystem::path& output_dir) +void write_vk_for_ivc(const std::string& input_path, const std::filesystem::path& output_dir) { init_bn254_crs(1 << CONST_PG_LOG_N); init_grumpkin_crs(1 << CONST_ECCVM_LOG_N); - const size_t num_public_inputs_in_final_circuit = get_num_public_inputs_in_final_circuit(bytecode_path); + const size_t num_public_inputs_in_final_circuit = get_num_public_inputs_in_final_circuit(input_path); info("num_public_inputs_in_final_circuit: ", num_public_inputs_in_final_circuit); // MAGIC_NUMBER is bb::PAIRING_POINT_ACCUMULATOR_SIZE or bb::PROPAGATED_DATABUS_COMMITMENTS_SIZE static constexpr size_t MAGIC_NUMBER = 16; @@ -180,83 +129,18 @@ void write_vk_for_ivc(const std::string& bytecode_path, const std::filesystem::p } } -std::vector _build_folding_stack(const std::string& input_type, - const std::filesystem::path& bytecode_path, - const std::filesystem::path& witness_path) -{ - using namespace acir_format; - - std::vector folding_stack; - - // TODO(https://github.com/AztecProtocol/barretenberg/issues/1162): Efficiently unify ACIR stack parsing - // really a single circuit IS a compiletime stack but we want the input type distinction since it is meaningful - // for vk writing (maybe this is not ideal?) - if (input_type == "single_circuit" || input_type == "compiletime_stack") { - auto program_stack = acir_format::get_acir_program_stack(bytecode_path, witness_path, /*honk_recursion=*/0); - // Accumulate the entire program stack into the IVC - while (!program_stack.empty()) { - auto stack_item = program_stack.back(); - folding_stack.push_back(AcirProgram{ stack_item.constraints, stack_item.witness }); - program_stack.pop_back(); - } - } - - if (input_type == "runtime_stack") { - std::vector gzipped_bincodes = unpack_from_file>(bytecode_path); - std::vector witness_data = unpack_from_file>(witness_path); - for (auto [bincode, wit] : zip_view(gzipped_bincodes, witness_data)) { - // TODO(#7371) there is a lot of copying going on in bincode, we should make sure this writes as a - // buffer in the future - std::vector constraint_buf = decompress(bincode.data(), bincode.size()); // NOLINT - std::vector witness_buf = decompress(wit.data(), wit.size()); // NOLINT - - AcirFormat constraints = circuit_buf_to_acir_format(constraint_buf, /*honk_recursion=*/0); - WitnessVector witness = witness_buf_to_witness_data(witness_buf); - - folding_stack.push_back(AcirProgram{ constraints, witness }); - } - } - - return folding_stack; -} - -std::shared_ptr _accumulate(std::vector& folding_stack) -{ - using Builder = MegaCircuitBuilder; - using Program = acir_format::AcirProgram; - using namespace acir_format; - - TraceSettings trace_settings{ AZTEC_TRACE_STRUCTURE }; - auto ivc = std::make_shared(trace_settings); - - const ProgramMetadata metadata{ ivc }; - - // Accumulate the entire program stack into the IVC - for (Program& program : folding_stack) { - // Construct a bberg circuit from the acir representation then accumulate it into the IVC - Builder circuit = acir_format::create_circuit(program, metadata); - - // Do one step of ivc accumulator or, if there is only one circuit in the stack, prove that circuit. In this - // case, no work is added to the Goblin opqueue, but VM proofs for trivials inputs are produced. - ivc->accumulate(circuit); - } - - return ivc; -} - void ClientIVCAPI::prove(const Flags& flags, - const std::filesystem::path& bytecode_path, - const std::filesystem::path& witness_path, + const std::filesystem::path& input_path, const std::filesystem::path& output_dir) { // TODO(https://github.com/AztecProtocol/barretenberg/issues/1163) set these dynamically init_bn254_crs(1 << CONST_PG_LOG_N); init_grumpkin_crs(1 << CONST_ECCVM_LOG_N); - std::vector folding_stack = - _build_folding_stack(flags.input_type, bytecode_path, witness_path); + PrivateExecutionSteps steps; + steps.parse(PrivateExecutionStepRaw::load_and_decompress(input_path)); - std::shared_ptr ivc = _accumulate(folding_stack); + std::shared_ptr ivc = steps.accumulate(); ClientIVC::Proof proof = ivc->prove(); // We verify this proof. Another bb call to verify has the overhead of loading the SRS, @@ -266,8 +150,8 @@ void ClientIVCAPI::prove(const Flags& flags, THROW std::runtime_error("Failed to verify the private (ClientIVC) transaction proof!"); } - // We'd like to use the `write` function that UltraHonkAPI uses, but there are missing functions for creating string - // representations of vks that don't feel worth implementing + // We'd like to use the `write` function that UltraHonkAPI uses, but there are missing functions for creating + // std::string representations of vks that don't feel worth implementing const bool output_to_stdout = output_dir == "-"; const auto write_proof = [&]() { @@ -285,7 +169,7 @@ void ClientIVCAPI::prove(const Flags& flags, if (flags.write_vk) { vinfo("writing ClientIVC vk in directory ", output_dir); - write_vk_for_ivc(bytecode_path, output_dir); + write_vk_for_ivc(input_path, output_dir); } } @@ -308,21 +192,16 @@ bool ClientIVCAPI::verify([[maybe_unused]] const Flags& flags, return verified; } -bool ClientIVCAPI::prove_and_verify(const Flags& flags, - const std::filesystem::path& bytecode_path, - const std::filesystem::path& witness_path) +bool ClientIVCAPI::prove_and_verify(const std::filesystem::path& input_path) { - if (!(flags.input_type == "compiletime_stack" || flags.input_type == "runtime_stack")) { - throw_or_abort("No input_type or input_type not supported"); - } - // TODO(https://github.com/AztecProtocol/barretenberg/issues/1163) set these dynamically init_bn254_crs(1 << CONST_PG_LOG_N); init_grumpkin_crs(1 << CONST_ECCVM_LOG_N); - std::vector folding_stack = - _build_folding_stack(flags.input_type, bytecode_path, witness_path); - std::shared_ptr ivc = _accumulate(folding_stack); + PrivateExecutionSteps steps; + steps.parse(PrivateExecutionStepRaw::load_and_decompress(input_path)); + + std::shared_ptr ivc = steps.accumulate(); const bool verified = ivc->prove_and_verify(); return verified; } @@ -340,13 +219,16 @@ void ClientIVCAPI::write_solidity_verifier([[maybe_unused]] const Flags& flags, throw_or_abort("API function contract not implemented"); } +void ClientIVCAPI::write_ivc_vk(const std::filesystem::path& input_path, const std::filesystem::path& output_path) +{ + write_vk_for_ivc(input_path, output_path); +} + void ClientIVCAPI::write_vk(const Flags& flags, const std::filesystem::path& bytecode_path, const std::filesystem::path& output_path) { - if (flags.verifier_type == "ivc") { - write_vk_for_ivc(bytecode_path, output_path); - } else if (flags.verifier_type == "standalone") { + if (flags.verifier_type == "standalone") { write_standalone_vk(flags.output_format, bytecode_path, output_path); } else { const std::string msg = std::string("Can't write vk for verifier type ") + flags.verifier_type; @@ -364,7 +246,7 @@ bool ClientIVCAPI::check([[maybe_unused]] const Flags& flags, void gate_count_for_ivc(const std::string& bytecode_path, bool include_gates_per_opcode) { - // All circuit reports will be built into the string below + // All circuit reports will be built into the std::string below std::string functions_string = "{\"functions\": [\n "; // TODO(https://github.com/AztecProtocol/barretenberg/issues/1181): Use enum for honk_recursion. auto constraint_systems = get_constraint_systems(bytecode_path, /*honk_recursion=*/0); diff --git a/barretenberg/cpp/src/barretenberg/api/api_client_ivc.hpp b/barretenberg/cpp/src/barretenberg/api/api_client_ivc.hpp index 77673a48a9e9..0af9d318d963 100644 --- a/barretenberg/cpp/src/barretenberg/api/api_client_ivc.hpp +++ b/barretenberg/cpp/src/barretenberg/api/api_client_ivc.hpp @@ -2,6 +2,7 @@ #include "barretenberg/api/acir_format_getters.hpp" #include "barretenberg/api/api.hpp" #include +#include #include #include @@ -10,19 +11,14 @@ namespace bb { class ClientIVCAPI : public API { public: - void prove(const Flags& flags, - const std::filesystem::path& bytecode_path, - const std::filesystem::path& witness_path, - const std::filesystem::path& output_dir) override; + void prove(const Flags& flags, const std::filesystem::path& input_path, const std::filesystem::path& output_dir); bool verify(const Flags& flags, const std::filesystem::path& public_inputs_path, const std::filesystem::path& proof_path, const std::filesystem::path& vk_path) override; - bool prove_and_verify(const Flags& flags, - const std::filesystem::path& bytecode_path, - const std::filesystem::path& witness_path) override; + bool prove_and_verify(const std::filesystem::path& input_path); void gates(const Flags& flags, const std::filesystem::path& bytecode_path) override; @@ -30,6 +26,10 @@ class ClientIVCAPI : public API { const std::filesystem::path& output_path, const std::filesystem::path& vk_path) override; + // write a vk for verifying a folding stack based on the number of public inputs of the final circuit + void write_ivc_vk(const std::filesystem::path& input_path, const std::filesystem::path& output_path); + + // write a vk for a circuit void write_vk(const Flags& flags, const std::filesystem::path& bytecode_path, const std::filesystem::path& output_path) override; @@ -47,5 +47,4 @@ acir_format::WitnessVector witness_map_to_witness_vector(std::map decompress(const void* bytes, size_t size); -void write_vk_for_ivc(const bool output_fields, const std::string& bytecode_path, const std::string& output_path); } // namespace bb diff --git a/barretenberg/cpp/src/barretenberg/api/api_ultra_honk.hpp b/barretenberg/cpp/src/barretenberg/api/api_ultra_honk.hpp index 02ba7a95bdf3..d0ea6a5f044c 100644 --- a/barretenberg/cpp/src/barretenberg/api/api_ultra_honk.hpp +++ b/barretenberg/cpp/src/barretenberg/api/api_ultra_honk.hpp @@ -17,7 +17,7 @@ class UltraHonkAPI : public API { void prove(const Flags& flags, const std::filesystem::path& bytecode_path, const std::filesystem::path& witness_path, - const std::filesystem::path& output_dir) override; + const std::filesystem::path& output_dir); bool verify(const Flags& flags, const std::filesystem::path& public_inputs_path, @@ -26,7 +26,7 @@ class UltraHonkAPI : public API { bool prove_and_verify(const Flags& flags, const std::filesystem::path& bytecode_path, - const std::filesystem::path& witness_path) override; + const std::filesystem::path& witness_path); void write_vk(const Flags& flags, const std::filesystem::path& bytecode_path, diff --git a/barretenberg/cpp/src/barretenberg/api/api_ultra_plonk.cpp b/barretenberg/cpp/src/barretenberg/api/api_ultra_plonk.cpp index 73d3f00d2f06..e23cd6b85d85 100644 --- a/barretenberg/cpp/src/barretenberg/api/api_ultra_plonk.cpp +++ b/barretenberg/cpp/src/barretenberg/api/api_ultra_plonk.cpp @@ -24,7 +24,7 @@ acir_proofs::AcirComposer verifier_init() std::string to_json(const std::vector& data) { - return format("[", join(map(data, [](auto fr) { return format("\"", fr, "\""); })), "]"); + return format("[", join(transform::map(data, [](auto fr) { return format("\"", fr, "\""); })), "]"); } std::string vk_to_json(std::vector const& data) @@ -33,7 +33,7 @@ std::string vk_to_json(std::vector const& data) std::vector rotated(data.begin(), data.end() - 1); rotated.insert(rotated.begin(), data.at(data.size() - 1)); - return format("[", join(map(rotated, [](auto fr) { return format("\"", fr, "\""); })), "]"); + return format("[", join(transform::map(rotated, [](auto fr) { return format("\"", fr, "\""); })), "]"); } /** @@ -48,6 +48,7 @@ std::string vk_to_json(std::vector const& data) * @param output_path Path to write the proof to * @param recursive Whether to use recursive proof generation or non-recursive */ + void prove_ultra_plonk(const std::string& bytecode_path, const std::string& witness_path, const std::string& output_path, diff --git a/barretenberg/cpp/src/barretenberg/api/file_io.hpp b/barretenberg/cpp/src/barretenberg/api/file_io.hpp index b341cbfc7145..fc0c7db584be 100644 --- a/barretenberg/cpp/src/barretenberg/api/file_io.hpp +++ b/barretenberg/cpp/src/barretenberg/api/file_io.hpp @@ -10,6 +10,7 @@ #include #include +namespace bb { inline size_t get_file_size(std::string const& filename) { // Open the file in binary mode and move to the end. @@ -64,7 +65,7 @@ inline void write_file(const std::string& filename, std::vector const& size_t total_written = 0; size_t data_size = data.size(); while (total_written < data_size) { - ssize_t written = write(fd, data.data() + total_written, data_size - total_written); + ssize_t written = ::write(fd, data.data() + total_written, data_size - total_written); if (written == -1) { close(fd); THROW std::runtime_error("Failed to write to file descriptor: " + filename); @@ -77,7 +78,8 @@ inline void write_file(const std::string& filename, std::vector const& if (!file) { THROW std::runtime_error("Failed to open data file for writing: " + filename); } - file.write((char*)data.data(), (std::streamsize)data.size()); + file.write((char*)data.data(), static_cast(data.size())); file.close(); } } +} // namespace bb diff --git a/barretenberg/cpp/src/barretenberg/api/prove_tube.cpp b/barretenberg/cpp/src/barretenberg/api/prove_tube.cpp index 19f075ea0ac4..135049a8d831 100644 --- a/barretenberg/cpp/src/barretenberg/api/prove_tube.cpp +++ b/barretenberg/cpp/src/barretenberg/api/prove_tube.cpp @@ -74,7 +74,7 @@ void prove_tube(const std::string& output_path, const std::string& vk_path) if (data.empty()) { return std::string("[]"); } - return format("[", join(map(data, [](auto fr) { return format("\"", fr, "\""); })), "]"); + return format("[", join(transform::map(data, [](auto fr) { return format("\"", fr, "\""); })), "]"); }; auto public_inputs_data = to_json(public_inputs_and_proof.public_inputs); auto proof_data = to_json(public_inputs_and_proof.proof); diff --git a/barretenberg/cpp/src/barretenberg/api/write_prover_output.hpp b/barretenberg/cpp/src/barretenberg/api/write_prover_output.hpp index 400bab52ad73..e4494e939d07 100644 --- a/barretenberg/cpp/src/barretenberg/api/write_prover_output.hpp +++ b/barretenberg/cpp/src/barretenberg/api/write_prover_output.hpp @@ -27,7 +27,7 @@ void write(const ProverOutput& prover_output, if (data.empty()) { return std::string("[]"); } - return format("[", join(map(data, [](auto fr) { return format("\"", fr, "\""); })), "]"); + return format("[", join(transform::map(data, [](auto fr) { return format("\"", fr, "\""); })), "]"); }; const auto write_bytes = [&](const ObjectToWrite& obj) { diff --git a/barretenberg/cpp/src/barretenberg/bb/cli.cpp b/barretenberg/cpp/src/barretenberg/bb/cli.cpp index 9a28fe519580..60a14e56e6a0 100644 --- a/barretenberg/cpp/src/barretenberg/bb/cli.cpp +++ b/barretenberg/cpp/src/barretenberg/bb/cli.cpp @@ -116,6 +116,7 @@ int parse_and_run_cli_command(int argc, char* argv[]) // Some paths, with defaults, that may or may not be set by commands std::filesystem::path bytecode_path{ "./target/program.json" }; std::filesystem::path witness_path{ "./target/witness.gz" }; + std::filesystem::path ivc_inputs_path; std::filesystem::path output_path{ "./out" }; // sometimes a directory where things will be written, sometimes the path of a file to be written @@ -202,17 +203,6 @@ int parse_and_run_cli_command(int argc, char* argv[]) return subcommand->add_flag("--write_vk", flags.write_vk, "Write the provided circuit's verification key"); }; - const auto add_input_type_option = [&](CLI::App* subcommand) { - auto* input_type_option = - subcommand - ->add_option("--input_type", - flags.input_type, - "Is the input a single circuit, a compile-time stack or a run-time stack?") - ->check(CLI::IsMember({ "single_circuit", "compiletime_stack", "runtime_stack" }).name("is_member")) - ->default_val("single_circuit"); - return input_type_option; - }; - const auto add_ipa_accumulation_flag = [&](CLI::App* subcommand) { return subcommand->add_flag( "--ipa_accumulation", flags.ipa_accumulation, "Accumulate/Aggregate IPA (Inner Product Argument) claims"); @@ -237,6 +227,12 @@ int parse_and_run_cli_command(int argc, char* argv[]) /* ->check(CLI::ExistingFile) OR stdin indicator - */; }; + const auto add_ivc_inputs_path_options = [&](CLI::App* subcommand) { + subcommand->add_option( + "--ivc_inputs_path", ivc_inputs_path, "For IVC, path to input stack with bytecode and witnesses.") + /* ->check(CLI::ExistingFile) OR stdin indicator - */; + }; + const auto add_public_inputs_path_option = [&](CLI::App* subcommand) { return subcommand->add_option( "--public_inputs_path, -i", public_inputs_path, "Path to public inputs.") /* ->check(CLI::ExistingFile) */; @@ -325,6 +321,7 @@ int parse_and_run_cli_command(int argc, char* argv[]) add_bytecode_path_option(prove); add_witness_path_option(prove); add_output_path_option(prove, output_path); + add_ivc_inputs_path_options(prove); add_verbose_flag(prove); add_debug_flag(prove); @@ -332,7 +329,6 @@ int parse_and_run_cli_command(int argc, char* argv[]) add_oracle_hash_option(prove); add_output_format_option(prove); add_write_vk_flag(prove); - add_input_type_option(prove); add_zk_option(prove); add_init_kzg_accumulator_option(prove); add_ipa_accumulation_flag(prove); @@ -353,11 +349,11 @@ int parse_and_run_cli_command(int argc, char* argv[]) add_scheme_option(write_vk); add_bytecode_path_option(write_vk); add_output_path_option(write_vk, output_path); + add_ivc_inputs_path_options(write_vk); add_verbose_flag(write_vk); add_debug_flag(write_vk); add_output_format_option(write_vk); - add_input_type_option(write_vk); add_crs_path_option(write_vk); add_init_kzg_accumulator_option(write_vk); add_oracle_hash_option(write_vk); @@ -698,8 +694,9 @@ int parse_and_run_cli_command(int argc, char* argv[]) print_subcommand_options(deepest); } - // prob this construction is too much - const auto execute_command = [&](API& api) { + // TODO(AD): it is inflexible that CIVC shares an API command (prove) with UH this way. The base API class is a + // poor fit. It would be better to have a separate handling for each scheme with subcommands to prove. + const auto execute_non_prove_command = [&](API& api) { if (check->parsed()) { api.check(flags, bytecode_path, witness_path); return 0; @@ -708,10 +705,6 @@ int parse_and_run_cli_command(int argc, char* argv[]) api.gates(flags, bytecode_path); return 0; } - if (prove->parsed()) { - api.prove(flags, bytecode_path, witness_path, output_path); - return 0; - } if (write_vk->parsed()) { api.write_vk(flags, bytecode_path, output_path); return 0; @@ -808,10 +801,30 @@ int parse_and_run_cli_command(int argc, char* argv[]) // NEW STANDARD API else if (flags.scheme == "client_ivc") { ClientIVCAPI api; - return execute_command(api); + if (prove->parsed()) { + if (ivc_inputs_path.empty()) { + throw_or_abort("The prove commands for ClientIVC expect --ivc_inputs_path " + ""); + } + api.prove(flags, ivc_inputs_path, output_path); + return 0; + } + if (write_vk->parsed() && flags.verifier_type == "ivc") { + if (ivc_inputs_path.empty()) { + throw_or_abort("The write_vk command for --verifier_type ivc expects --ivc_inputs_path " + ""); + } + api.write_ivc_vk(ivc_inputs_path, output_path); + return 0; + } + return execute_non_prove_command(api); } else if (flags.scheme == "ultra_honk") { UltraHonkAPI api; - return execute_command(api); + if (prove->parsed()) { + api.prove(flags, bytecode_path, witness_path, output_path); + return 0; + } + return execute_non_prove_command(api); } else { throw_or_abort("No match for API command"); return 1; diff --git a/barretenberg/cpp/src/barretenberg/benchmark/bb_cli_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/bb_cli_bench/CMakeLists.txt index cc5dba6fb117..0d70012d0e01 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/bb_cli_bench/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/benchmark/bb_cli_bench/CMakeLists.txt @@ -1,3 +1,3 @@ -# if (NOT(FUZZING) AND NOT(WASM)) +if (NOT(FUZZING)) barretenberg_module(bb_cli_bench bb-cli-lib) -# endif() +endif() diff --git a/barretenberg/cpp/src/barretenberg/client_ivc/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/client_ivc/CMakeLists.txt index 4cabbc72fbcc..18eb032d515a 100644 --- a/barretenberg/cpp/src/barretenberg/client_ivc/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/client_ivc/CMakeLists.txt @@ -1 +1 @@ -barretenberg_module(client_ivc stdlib_protogalaxy_verifier stdlib_goblin_verifier) \ No newline at end of file +barretenberg_module(client_ivc stdlib_protogalaxy_verifier stdlib_goblin_verifier dsl) diff --git a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.cpp b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.cpp index 88f5dd8cf1fd..53d198be2434 100644 --- a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.cpp +++ b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.cpp @@ -1,8 +1,24 @@ #include "barretenberg/client_ivc/client_ivc.hpp" +#include "barretenberg/common/op_count.hpp" +#include "barretenberg/serialize/msgpack_impl.hpp" #include "barretenberg/ultra_honk/oink_prover.hpp" namespace bb { +// Constructor +ClientIVC::ClientIVC(TraceSettings trace_settings) + : trace_usage_tracker(trace_settings) + , trace_settings(trace_settings) + , goblin(bn254_commitment_key) +{ + // Allocate BN254 commitment key based on the max dyadic Mega structured trace size and translator circuit size. + // https://github.com/AztecProtocol/barretenberg/issues/1319): Account for Translator only when it's necessary + size_t commitment_key_size = + std::max(trace_settings.dyadic_size(), 1UL << TranslatorFlavor::CONST_TRANSLATOR_LOG_N); + info("BN254 commitment key size: ", commitment_key_size); + bn254_commitment_key = std::make_shared>(commitment_key_size); +} + /** * @brief Instantiate a stdlib verification queue for use in the kernel completion logic * @details Construct a stdlib proof/verification_key for each entry in the native verification queue. By default, both @@ -170,7 +186,10 @@ void ClientIVC::accumulate(ClientCircuit& circuit, trace_usage_tracker.update(circuit); // Set the verification key from precomputed if available, else compute it - honk_vk = precomputed_vk ? precomputed_vk : std::make_shared(proving_key->proving_key); + { + PROFILE_THIS_NAME("ClientIVC::accumulate create MegaVerificationKey"); + honk_vk = precomputed_vk ? precomputed_vk : std::make_shared(proving_key->proving_key); + } if (mock_vk) { honk_vk->set_metadata(proving_key->proving_key); vinfo("set honk vk metadata"); @@ -384,4 +403,81 @@ std::vector> ClientIVC::precompute_ return vkeys; } +// Proof methods +size_t ClientIVC::Proof::size() const +{ + return mega_proof.size() + goblin_proof.size(); +} + +msgpack::sbuffer ClientIVC::Proof::to_msgpack_buffer() const +{ + msgpack::sbuffer buffer; + msgpack::pack(buffer, *this); + return buffer; +} + +uint8_t* ClientIVC::Proof::to_msgpack_heap_buffer() const +{ + msgpack::sbuffer buffer = to_msgpack_buffer(); + + std::vector buf(buffer.data(), buffer.data() + buffer.size()); + return to_heap_buffer(buf); +} + +ClientIVC::Proof ClientIVC::Proof::from_msgpack_buffer(uint8_t const*& buffer) +{ + auto uint8_buffer = from_buffer>(buffer); + + msgpack::sbuffer sbuf; + sbuf.write(reinterpret_cast(uint8_buffer.data()), uint8_buffer.size()); + + return from_msgpack_buffer(sbuf); +} + +ClientIVC::Proof ClientIVC::Proof::from_msgpack_buffer(const msgpack::sbuffer& buffer) +{ + msgpack::object_handle oh = msgpack::unpack(buffer.data(), buffer.size()); + msgpack::object obj = oh.get(); + Proof proof; + obj.convert(proof); + return proof; +} + +void ClientIVC::Proof::to_file_msgpack(const std::string& filename) const +{ + msgpack::sbuffer buffer = to_msgpack_buffer(); + std::ofstream ofs(filename, std::ios::binary); + if (!ofs.is_open()) { + throw_or_abort("Failed to open file for writing."); + } + ofs.write(buffer.data(), static_cast(buffer.size())); + ofs.close(); +} + +ClientIVC::Proof ClientIVC::Proof::from_file_msgpack(const std::string& filename) +{ + std::ifstream ifs(filename, std::ios::binary); + if (!ifs.is_open()) { + throw_or_abort("Failed to open file for reading."); + } + + ifs.seekg(0, std::ios::end); + size_t file_size = static_cast(ifs.tellg()); + ifs.seekg(0, std::ios::beg); + + std::vector buffer(file_size); + ifs.read(buffer.data(), static_cast(file_size)); + ifs.close(); + msgpack::sbuffer msgpack_buffer; + msgpack_buffer.write(buffer.data(), file_size); + + return Proof::from_msgpack_buffer(msgpack_buffer); +} + +// VerificationKey construction +ClientIVC::VerificationKey ClientIVC::get_vk() const +{ + return { honk_vk, std::make_shared(), std::make_shared() }; +} + } // namespace bb diff --git a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.hpp b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.hpp index 638a3d16fcc9..aad35eb6a00d 100644 --- a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.hpp +++ b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.hpp @@ -75,16 +75,11 @@ class ClientIVC { HonkProof mega_proof; GoblinProof goblin_proof; - size_t size() const { return mega_proof.size() + goblin_proof.size(); } + size_t size() const; // TODO(https://github.com/AztecProtocol/barretenberg/issues/1299): The following msgpack methods are generic // and should leverage some kind of shared msgpack utility. - msgpack::sbuffer to_msgpack_buffer() const - { - msgpack::sbuffer buffer; - msgpack::pack(buffer, *this); - return buffer; - } + msgpack::sbuffer to_msgpack_buffer() const; /** * @brief Very quirky method to convert a msgpack buffer to a "heap" buffer @@ -93,13 +88,7 @@ class ClientIVC { * * @return uint8_t* Double size-prefixed msgpack buffer */ - uint8_t* to_msgpack_heap_buffer() const - { - msgpack::sbuffer buffer = to_msgpack_buffer(); - - std::vector buf(buffer.data(), buffer.data() + buffer.size()); - return to_heap_buffer(buf); - } + uint8_t* to_msgpack_heap_buffer() const; class DeserializationError : public std::runtime_error { public: @@ -108,55 +97,11 @@ class ClientIVC { {} }; - static Proof from_msgpack_buffer(uint8_t const*& buffer) - { - auto uint8_buffer = from_buffer>(buffer); - - msgpack::sbuffer sbuf; - sbuf.write(reinterpret_cast(uint8_buffer.data()), uint8_buffer.size()); - - return from_msgpack_buffer(sbuf); - } - - static Proof from_msgpack_buffer(const msgpack::sbuffer& buffer) - { - msgpack::object_handle oh = msgpack::unpack(buffer.data(), buffer.size()); - msgpack::object obj = oh.get(); - Proof proof; - obj.convert(proof); - return proof; - } - - void to_file_msgpack(const std::string& filename) const - { - msgpack::sbuffer buffer = to_msgpack_buffer(); - std::ofstream ofs(filename, std::ios::binary); - if (!ofs.is_open()) { - throw_or_abort("Failed to open file for writing."); - } - ofs.write(buffer.data(), static_cast(buffer.size())); - ofs.close(); - } - - static Proof from_file_msgpack(const std::string& filename) - { - std::ifstream ifs(filename, std::ios::binary); - if (!ifs.is_open()) { - throw_or_abort("Failed to open file for reading."); - } - - ifs.seekg(0, std::ios::end); - size_t file_size = static_cast(ifs.tellg()); - ifs.seekg(0, std::ios::beg); - - std::vector buffer(file_size); - ifs.read(buffer.data(), static_cast(file_size)); - ifs.close(); - msgpack::sbuffer msgpack_buffer; - msgpack_buffer.write(buffer.data(), file_size); - - return Proof::from_msgpack_buffer(msgpack_buffer); - } + static Proof from_msgpack_buffer(uint8_t const*& buffer); + static Proof from_msgpack_buffer(const msgpack::sbuffer& buffer); + + void to_file_msgpack(const std::string& filename) const; + static Proof from_file_msgpack(const std::string& filename); MSGPACK_FIELDS(mega_proof, goblin_proof); }; @@ -220,18 +165,7 @@ class ClientIVC { bool initialized = false; // Is the IVC accumulator initialized - ClientIVC(TraceSettings trace_settings = {}) - : trace_usage_tracker(trace_settings) - , trace_settings(trace_settings) - , goblin(bn254_commitment_key) - { - // Allocate BN254 commitment key based on the max dyadic Mega structured trace size and translator circuit size. - // https://github.com/AztecProtocol/barretenberg/issues/1319): Account for Translator only when it's necessary - size_t commitment_key_size = - std::max(trace_settings.dyadic_size(), 1UL << TranslatorFlavor::CONST_TRANSLATOR_LOG_N); - info("BN254 commitment key size: ", commitment_key_size); - bn254_commitment_key = std::make_shared>(commitment_key_size); - } + ClientIVC(TraceSettings trace_settings = {}); void instantiate_stdlib_verification_queue( ClientCircuit& circuit, const std::vector>& input_keys = {}); @@ -270,9 +204,7 @@ class ClientIVC { std::vector> precompute_folding_verification_keys( std::vector circuits); - VerificationKey get_vk() const - { - return { honk_vk, std::make_shared(), std::make_shared() }; - } + VerificationKey get_vk() const; }; + } // namespace bb diff --git a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp index 0e5d0dbac8da..9af3ca6a2ced 100644 --- a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp +++ b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp @@ -4,6 +4,7 @@ #include "barretenberg/goblin/goblin.hpp" #include "barretenberg/goblin/mock_circuits.hpp" #include "barretenberg/protogalaxy/folding_test_utils.hpp" +#include "barretenberg/serialize/msgpack_impl.hpp" #include "barretenberg/stdlib_circuit_builders/mega_circuit_builder.hpp" #include "barretenberg/stdlib_circuit_builders/ultra_circuit_builder.hpp" #include diff --git a/barretenberg/cpp/src/barretenberg/client_ivc/private_execution_steps.cpp b/barretenberg/cpp/src/barretenberg/client_ivc/private_execution_steps.cpp new file mode 100644 index 000000000000..94476343f324 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/client_ivc/private_execution_steps.cpp @@ -0,0 +1,126 @@ +#include "private_execution_steps.hpp" +#include "barretenberg/common/serialize.hpp" +#include "barretenberg/dsl/acir_format/acir_to_constraint_buf.hpp" +#include + +namespace bb { + +std::string decompress(const void* bytes, size_t size) +{ + std::string content; + // initial size guess + content.resize(1024ULL * 128ULL); + for (;;) { + auto decompressor = std::unique_ptr{ + libdeflate_alloc_decompressor(), libdeflate_free_decompressor + }; + size_t actual_size = 0; + libdeflate_result decompress_result = + libdeflate_gzip_decompress(decompressor.get(), bytes, size, content.data(), content.size(), &actual_size); + if (decompress_result == LIBDEFLATE_INSUFFICIENT_SPACE) { + // need a bigger buffer + content.resize(content.size() * 2); + continue; + } + if (decompress_result == LIBDEFLATE_BAD_DATA) { + THROW std::invalid_argument("bad gzip data in bb main"); + } + content.resize(actual_size); + break; + } + return content; +} + +template T unpack_from_file(const std::filesystem::path& filename) +{ + std::ifstream fin; + fin.open(filename, std::ios::ate | std::ios::binary); + if (!fin.is_open()) { + THROW std::invalid_argument("file not found"); + } + if (fin.tellg() == -1) { + THROW std::invalid_argument("something went wrong"); + } + + size_t fsize = static_cast(fin.tellg()); + fin.seekg(0, std::ios_base::beg); + + T result; + std::string encoded_data(fsize, '\0'); + fin.read(encoded_data.data(), static_cast(fsize)); + msgpack::unpack(encoded_data.data(), fsize).get().convert(result); + return result; +} + +// TODO(#7371) we should not have so many levels of serialization here. +std::vector PrivateExecutionStepRaw::load_and_decompress( + const std::filesystem::path& input_path) +{ + auto raw_steps = unpack_from_file>(input_path); + for (PrivateExecutionStepRaw& step : raw_steps) { + step.bytecode = decompress(step.bytecode.data(), step.bytecode.size()); + step.witness = decompress(step.witness.data(), step.witness.size()); + } + return raw_steps; +} + +std::vector PrivateExecutionStepRaw::parse_uncompressed(const std::vector& buf) +{ + std::vector raw_steps; + // Read with msgpack + msgpack::unpack(reinterpret_cast(buf.data()), buf.size()).get().convert(raw_steps); + // Unlike load_and_decompress, we don't need to decompress the bytecode and witness fields + return raw_steps; +} + +void PrivateExecutionSteps::parse(const std::vector& steps) +{ + for (const PrivateExecutionStepRaw& step : steps) { + // TODO(#7371) there is a lot of copying going on in bincode. We need the generated bincode code to + // use spans instead of vectors. + std::vector bytecode_buf(step.bytecode.begin(), step.bytecode.end()); + std::vector witness_buf(step.witness.begin(), step.witness.end()); + acir_format::AcirFormat constraints = + acir_format::circuit_buf_to_acir_format(bytecode_buf, /*honk_recursion=*/0); + acir_format::WitnessVector witness = acir_format::witness_buf_to_witness_data(witness_buf); + + folding_stack.push_back({ constraints, witness }); + if (step.vk.empty()) { + // For backwards compatibility, but it affects performance and correctness. + precomputed_vks.emplace_back(); + } else { + auto vk = from_buffer>(step.vk); + precomputed_vks.push_back(vk); + } + function_names.push_back(step.function_name); + } +} + +std::shared_ptr PrivateExecutionSteps::accumulate() +{ + TraceSettings trace_settings{ AZTEC_TRACE_STRUCTURE }; + auto ivc = std::make_shared(trace_settings); + + const acir_format::ProgramMetadata metadata{ ivc }; + + for (auto& vk : precomputed_vks) { + if (vk == nullptr) { + info("DEPRECATED: No VK was provided for at least one client IVC step and it will be computed. This is " + "slower and insecure."); + break; + } + } + // Accumulate the entire program stack into the IVC + for (auto [program, precomputed_vk, function_name] : zip_view(folding_stack, precomputed_vks, function_names)) { + // Construct a bberg circuit from the acir representation then accumulate it into the IVC + auto circuit = acir_format::create_circuit(program, metadata); + + info("ClientIVC: accumulating " + function_name); + // Do one step of ivc accumulator or, if there is only one circuit in the stack, prove that circuit. In this + // case, no work is added to the Goblin opqueue, but VM proofs for trivials inputs are produced. + ivc->accumulate(circuit, precomputed_vk); + } + + return ivc; +} +} // namespace bb diff --git a/barretenberg/cpp/src/barretenberg/client_ivc/private_execution_steps.hpp b/barretenberg/cpp/src/barretenberg/client_ivc/private_execution_steps.hpp new file mode 100644 index 000000000000..c316be9b1293 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/client_ivc/private_execution_steps.hpp @@ -0,0 +1,49 @@ +#include "barretenberg/dsl/acir_format/acir_format.hpp" +#include "barretenberg/serialize/msgpack.hpp" +#include +#include +#include +#include + +namespace bb { + +/** + * @brief This is the msgpack encoding of the objects returned by the following typescript: + * const stepToStruct = (step: PrivateExecutionStep) => { + * return { + * bytecode: step.bytecode, + * witness: serializeWitness(step.witness), + * vk: step.vk, + * functionName: step.functionName + * }; + * }; + * await fs.writeFile(path, encode(executionSteps.map(stepToStruct))); + * See format notes below. + */ +struct PrivateExecutionStepRaw { + // Represents bincode. + std::string bytecode; + // Represents bincoded witness data. + std::string witness; + // Represents a legacy-serialized vk. + std::string vk; + // Represents the function name. + std::string function_name; + + // Unrolled from MSGPACK_FIELDS for custom name for function_name. + void msgpack(auto pack_fn) { pack_fn(NVP(bytecode, witness, vk), "functionName", function_name); }; + static std::vector load_and_decompress(const std::filesystem::path& input_path); + static std::vector parse_uncompressed(const std::vector& buf); +}; + +// TODO(https://github.com/AztecProtocol/barretenberg/issues/1162) this should have a common code path with +// the WASM folding stack code. +struct PrivateExecutionSteps { + std::vector folding_stack; + std::vector function_names; + std::vector> precomputed_vks; + + std::shared_ptr accumulate(); + void parse(const std::vector& steps); +}; +} // namespace bb diff --git a/barretenberg/cpp/src/barretenberg/common/log.cpp b/barretenberg/cpp/src/barretenberg/common/log.cpp index 60c9679cedb8..29b80a3b4a04 100644 --- a/barretenberg/cpp/src/barretenberg/common/log.cpp +++ b/barretenberg/cpp/src/barretenberg/common/log.cpp @@ -9,4 +9,4 @@ bool verbose_logging = true; #endif // Used for `debug` in log.hpp. -bool debug_logging = false; \ No newline at end of file +bool debug_logging = false; diff --git a/barretenberg/cpp/src/barretenberg/common/map.hpp b/barretenberg/cpp/src/barretenberg/common/map.hpp index 1118131e6fe7..b205c8f1fe15 100644 --- a/barretenberg/cpp/src/barretenberg/common/map.hpp +++ b/barretenberg/cpp/src/barretenberg/common/map.hpp @@ -3,6 +3,7 @@ #include #include +namespace bb::transform { /* * Generic map function for mapping a containers element to another type. */ @@ -61,3 +62,4 @@ template Cont filter(Cont const& in, F op) std::remove_if(copy.begin(), copy.end(), op); return copy; } +} // namespace bb::transform diff --git a/barretenberg/cpp/src/barretenberg/common/wasm_export.hpp b/barretenberg/cpp/src/barretenberg/common/wasm_export.hpp index b9ba4efc139d..cab57a90e1f8 100644 --- a/barretenberg/cpp/src/barretenberg/common/wasm_export.hpp +++ b/barretenberg/cpp/src/barretenberg/common/wasm_export.hpp @@ -1,4 +1,5 @@ #pragma once +#include #ifdef __clang__ #define WASM_EXPORT extern "C" __attribute__((visibility("default"))) __attribute__((annotate("wasm_export"))) #define ASYNC_WASM_EXPORT \ @@ -13,4 +14,6 @@ #define WASM_IMPORT(name) extern "C" __attribute__((import_module("env"), import_name(name))) #else #define WASM_IMPORT(name) extern "C" -#endif \ No newline at end of file +#endif + +using uint8_vec_vec_in_buf = uint8_t const*; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp index 8e49cf005c56..61cfd1194c6f 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp @@ -8,7 +8,7 @@ #include "barretenberg/lmdblib/lmdb_store_base.hpp" #include "barretenberg/numeric/uint128/uint128.hpp" #include "barretenberg/numeric/uint256/uint256.hpp" -#include "barretenberg/serialize/msgpack.hpp" +#include "barretenberg/serialize/msgpack_impl.hpp" #include "lmdb_tree_store.hpp" #include #include @@ -326,4 +326,4 @@ void LMDBTreeStore::write_node(const fr& nodeHash, const NodePayload& nodeData, tx.put_value(key, encoded, *_nodeDatabase); } -} // namespace bb::crypto::merkle_tree \ No newline at end of file +} // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp index eaa06f3e18fd..1d4b80f2e1be 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp @@ -10,7 +10,7 @@ #include "barretenberg/lmdblib/lmdb_read_transaction.hpp" #include "barretenberg/lmdblib/lmdb_store_base.hpp" #include "barretenberg/lmdblib/lmdb_write_transaction.hpp" -#include "barretenberg/serialize/msgpack.hpp" +#include "barretenberg/serialize/msgpack_impl.hpp" #include "barretenberg/world_state/types.hpp" #include "lmdb.h" #include diff --git a/barretenberg/cpp/src/barretenberg/crypto/poseidon2/c_bind.hpp b/barretenberg/cpp/src/barretenberg/crypto/poseidon2/c_bind.hpp index 724ab507252d..b6f55fe4c49c 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/poseidon2/c_bind.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/poseidon2/c_bind.hpp @@ -10,4 +10,5 @@ using namespace bb; WASM_EXPORT void poseidon2_hash(fr::vec_in_buf inputs_buffer, fr::out_buf output); WASM_EXPORT void poseidon2_hashes(fr::vec_in_buf inputs_buffer, fr::out_buf output); WASM_EXPORT void poseidon2_permutation(fr::vec_in_buf inputs_buffer, fr::vec_out_buf output); -} \ No newline at end of file +WASM_EXPORT void poseidon2_hash_accumulate(fr::vec_in_buf inputs_buffer, fr::out_buf output); +} diff --git a/barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_format.test.cpp b/barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_format.test.cpp index 4ab8575c627b..04305825b4a3 100644 --- a/barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_format.test.cpp +++ b/barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_format.test.cpp @@ -8,7 +8,6 @@ #include "barretenberg/op_queue/ecc_op_queue.hpp" #include "barretenberg/plonk/composer/standard_composer.hpp" #include "barretenberg/plonk/composer/ultra_composer.hpp" -#include "barretenberg/plonk/proof_system/types/proof.hpp" #include "barretenberg/serialize/test_helper.hpp" #include "ecdsa_secp256k1.hpp" diff --git a/barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_to_constraint_buf.cpp b/barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_to_constraint_buf.cpp index 5ccb1dd2a7c9..0c2f10e0bc23 100644 --- a/barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_to_constraint_buf.cpp +++ b/barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_to_constraint_buf.cpp @@ -596,10 +596,10 @@ void handle_blackbox_func_call(Acir::Opcode::BlackBoxFuncCall const& arg, } } else if constexpr (std::is_same_v) { af.aes128_constraints.push_back(AES128Constraint{ - .inputs = map(arg.inputs, [](auto& e) { return parse_input(e); }), - .iv = map(arg.iv, [](auto& e) { return parse_input(e); }), - .key = map(arg.key, [](auto& e) { return parse_input(e); }), - .outputs = map(arg.outputs, [](auto& e) { return e.value; }), + .inputs = transform::map(arg.inputs, [](auto& e) { return parse_input(e); }), + .iv = transform::map(arg.iv, [](auto& e) { return parse_input(e); }), + .key = transform::map(arg.key, [](auto& e) { return parse_input(e); }), + .outputs = transform::map(arg.outputs, [](auto& e) { return e.value; }), }); for (auto& output : af.aes128_constraints.back().outputs) { af.constrained_witness.insert(output); @@ -607,9 +607,9 @@ void handle_blackbox_func_call(Acir::Opcode::BlackBoxFuncCall const& arg, af.original_opcode_indices.aes128_constraints.push_back(opcode_index); } else if constexpr (std::is_same_v) { af.sha256_compression.push_back(Sha256Compression{ - .inputs = map(arg.inputs, [](auto& e) { return parse_input(e); }), - .hash_values = map(arg.hash_values, [](auto& e) { return parse_input(e); }), - .result = map(arg.outputs, [](auto& e) { return e.value; }), + .inputs = transform::map(arg.inputs, [](auto& e) { return parse_input(e); }), + .hash_values = transform::map(arg.hash_values, [](auto& e) { return parse_input(e); }), + .result = transform::map(arg.outputs, [](auto& e) { return e.value; }), }); for (auto& output : af.sha256_compression.back().result) { af.constrained_witness.insert(output); @@ -617,14 +617,14 @@ void handle_blackbox_func_call(Acir::Opcode::BlackBoxFuncCall const& arg, af.original_opcode_indices.sha256_compression.push_back(opcode_index); } else if constexpr (std::is_same_v) { af.blake2s_constraints.push_back(Blake2sConstraint{ - .inputs = map(arg.inputs, - [](auto& e) { - return Blake2sInput{ - .blackbox_input = parse_input(e), - .num_bits = e.num_bits, - }; - }), - .result = map(arg.outputs, [](auto& e) { return e.value; }), + .inputs = transform::map(arg.inputs, + [](auto& e) { + return Blake2sInput{ + .blackbox_input = parse_input(e), + .num_bits = e.num_bits, + }; + }), + .result = transform::map(arg.outputs, [](auto& e) { return e.value; }), }); for (auto& output : af.blake2s_constraints.back().result) { af.constrained_witness.insert(output); @@ -632,14 +632,14 @@ void handle_blackbox_func_call(Acir::Opcode::BlackBoxFuncCall const& arg, af.original_opcode_indices.blake2s_constraints.push_back(opcode_index); } else if constexpr (std::is_same_v) { af.blake3_constraints.push_back(Blake3Constraint{ - .inputs = map(arg.inputs, - [](auto& e) { - return Blake3Input{ - .blackbox_input = parse_input(e), - .num_bits = e.num_bits, - }; - }), - .result = map(arg.outputs, [](auto& e) { return e.value; }), + .inputs = transform::map(arg.inputs, + [](auto& e) { + return Blake3Input{ + .blackbox_input = parse_input(e), + .num_bits = e.num_bits, + }; + }), + .result = transform::map(arg.outputs, [](auto& e) { return e.value; }), }); for (auto& output : af.blake3_constraints.back().result) { af.constrained_witness.insert(output); @@ -648,10 +648,13 @@ void handle_blackbox_func_call(Acir::Opcode::BlackBoxFuncCall const& arg, } else if constexpr (std::is_same_v) { af.ecdsa_k1_constraints.push_back(EcdsaSecp256k1Constraint{ .hashed_message = - map(arg.hashed_message, [](auto& e) { return get_witness_from_function_input(e); }), - .signature = map(arg.signature, [](auto& e) { return get_witness_from_function_input(e); }), - .pub_x_indices = map(arg.public_key_x, [](auto& e) { return get_witness_from_function_input(e); }), - .pub_y_indices = map(arg.public_key_y, [](auto& e) { return get_witness_from_function_input(e); }), + transform::map(arg.hashed_message, [](auto& e) { return get_witness_from_function_input(e); }), + .signature = + transform::map(arg.signature, [](auto& e) { return get_witness_from_function_input(e); }), + .pub_x_indices = + transform::map(arg.public_key_x, [](auto& e) { return get_witness_from_function_input(e); }), + .pub_y_indices = + transform::map(arg.public_key_y, [](auto& e) { return get_witness_from_function_input(e); }), .result = arg.output.value, }); af.constrained_witness.insert(af.ecdsa_k1_constraints.back().result); @@ -659,18 +662,21 @@ void handle_blackbox_func_call(Acir::Opcode::BlackBoxFuncCall const& arg, } else if constexpr (std::is_same_v) { af.ecdsa_r1_constraints.push_back(EcdsaSecp256r1Constraint{ .hashed_message = - map(arg.hashed_message, [](auto& e) { return get_witness_from_function_input(e); }), - .pub_x_indices = map(arg.public_key_x, [](auto& e) { return get_witness_from_function_input(e); }), - .pub_y_indices = map(arg.public_key_y, [](auto& e) { return get_witness_from_function_input(e); }), + transform::map(arg.hashed_message, [](auto& e) { return get_witness_from_function_input(e); }), + .pub_x_indices = + transform::map(arg.public_key_x, [](auto& e) { return get_witness_from_function_input(e); }), + .pub_y_indices = + transform::map(arg.public_key_y, [](auto& e) { return get_witness_from_function_input(e); }), .result = arg.output.value, - .signature = map(arg.signature, [](auto& e) { return get_witness_from_function_input(e); }), + .signature = + transform::map(arg.signature, [](auto& e) { return get_witness_from_function_input(e); }), }); af.constrained_witness.insert(af.ecdsa_r1_constraints.back().result); af.original_opcode_indices.ecdsa_r1_constraints.push_back(opcode_index); } else if constexpr (std::is_same_v) { af.multi_scalar_mul_constraints.push_back(MultiScalarMul{ - .points = map(arg.points, [](auto& e) { return parse_input(e); }), - .scalars = map(arg.scalars, [](auto& e) { return parse_input(e); }), + .points = transform::map(arg.points, [](auto& e) { return parse_input(e); }), + .scalars = transform::map(arg.scalars, [](auto& e) { return parse_input(e); }), .out_point_x = arg.outputs[0].value, .out_point_y = arg.outputs[1].value, .out_point_is_infinite = arg.outputs[2].value, @@ -704,8 +710,8 @@ void handle_blackbox_func_call(Acir::Opcode::BlackBoxFuncCall const& arg, af.original_opcode_indices.ec_add_constraints.push_back(opcode_index); } else if constexpr (std::is_same_v) { af.keccak_permutations.push_back(Keccakf1600{ - .state = map(arg.inputs, [](auto& e) { return parse_input(e); }), - .result = map(arg.outputs, [](auto& e) { return e.value; }), + .state = transform::map(arg.inputs, [](auto& e) { return parse_input(e); }), + .result = transform::map(arg.outputs, [](auto& e) { return e.value; }), }); for (auto& output : af.keccak_permutations.back().result) { af.constrained_witness.insert(output); @@ -729,9 +735,11 @@ void handle_blackbox_func_call(Acir::Opcode::BlackBoxFuncCall const& arg, } auto c = RecursionConstraint{ - .key = map(arg.verification_key, [](auto& e) { return get_witness_from_function_input(e); }), - .proof = map(arg.proof, [](auto& e) { return get_witness_from_function_input(e); }), - .public_inputs = map(arg.public_inputs, [](auto& e) { return get_witness_from_function_input(e); }), + .key = transform::map(arg.verification_key, + [](auto& e) { return get_witness_from_function_input(e); }), + .proof = transform::map(arg.proof, [](auto& e) { return get_witness_from_function_input(e); }), + .public_inputs = + transform::map(arg.public_inputs, [](auto& e) { return get_witness_from_function_input(e); }), .key_hash = input_key, .proof_type = proof_type_in, }; @@ -763,15 +771,15 @@ void handle_blackbox_func_call(Acir::Opcode::BlackBoxFuncCall const& arg, } } else if constexpr (std::is_same_v) { af.bigint_from_le_bytes_constraints.push_back(BigIntFromLeBytes{ - .inputs = map(arg.inputs, [](auto& e) { return get_witness_from_function_input(e); }), - .modulus = map(arg.modulus, [](auto& e) -> uint32_t { return e; }), + .inputs = transform::map(arg.inputs, [](auto& e) { return get_witness_from_function_input(e); }), + .modulus = transform::map(arg.modulus, [](auto& e) -> uint32_t { return e; }), .result = arg.output, }); af.original_opcode_indices.bigint_from_le_bytes_constraints.push_back(opcode_index); } else if constexpr (std::is_same_v) { af.bigint_to_le_bytes_constraints.push_back(BigIntToLeBytes{ .input = arg.input, - .result = map(arg.outputs, [](auto& e) { return e.value; }), + .result = transform::map(arg.outputs, [](auto& e) { return e.value; }), }); for (auto& output : af.bigint_to_le_bytes_constraints.back().result) { af.constrained_witness.insert(output); @@ -811,8 +819,8 @@ void handle_blackbox_func_call(Acir::Opcode::BlackBoxFuncCall const& arg, af.original_opcode_indices.bigint_operations.push_back(opcode_index); } else if constexpr (std::is_same_v) { af.poseidon2_constraints.push_back(Poseidon2Constraint{ - .state = map(arg.inputs, [](auto& e) { return parse_input(e); }), - .result = map(arg.outputs, [](auto& e) { return e.value; }), + .state = transform::map(arg.inputs, [](auto& e) { return parse_input(e); }), + .result = transform::map(arg.outputs, [](auto& e) { return e.value; }), .len = arg.len, }); for (auto& output : af.poseidon2_constraints.back().result) { @@ -886,8 +894,8 @@ AcirFormat circuit_serde_to_acir_format(Acir::Circuit const& circuit, uint32_t h // `varnum` is the true number of variables, thus we add one to the index which starts at zero af.varnum = circuit.current_witness_index + 1; af.num_acir_opcodes = static_cast(circuit.opcodes.size()); - af.public_inputs = join({ map(circuit.public_parameters.value, [](auto e) { return e.value; }), - map(circuit.return_values.value, [](auto e) { return e.value; }) }); + af.public_inputs = join({ transform::map(circuit.public_parameters.value, [](auto e) { return e.value; }), + transform::map(circuit.return_values.value, [](auto e) { return e.value; }) }); // Map to a pair of: BlockConstraint, and list of opcodes associated with that BlockConstraint // NOTE: We want to deterministically visit this map, so unordered_map should not be used. std::map>> block_id_to_block_constraint; diff --git a/barretenberg/cpp/src/barretenberg/dsl/acir_format/proof_surgeon.hpp b/barretenberg/cpp/src/barretenberg/dsl/acir_format/proof_surgeon.hpp index 2e4a887fdec4..692a5a653adc 100644 --- a/barretenberg/cpp/src/barretenberg/dsl/acir_format/proof_surgeon.hpp +++ b/barretenberg/cpp/src/barretenberg/dsl/acir_format/proof_surgeon.hpp @@ -17,7 +17,8 @@ class ProofSurgeon { // construct a string of the form "[, , ...]" static std::string to_json(const std::vector& data) { - return format("[", bb::join(map(data, [](auto fr) { return format("\"", fr, "\""); }), ", "), "]"); + return format( + "[", bb::join(bb::transform::map(data, [](auto fr) { return format("\"", fr, "\""); }), ", "), "]"); } public: diff --git a/barretenberg/cpp/src/barretenberg/dsl/acir_format/serde/acir.hpp b/barretenberg/cpp/src/barretenberg/dsl/acir_format/serde/acir.hpp index 564cf86de06f..f87016f64d02 100644 --- a/barretenberg/cpp/src/barretenberg/dsl/acir_format/serde/acir.hpp +++ b/barretenberg/cpp/src/barretenberg/dsl/acir_format/serde/acir.hpp @@ -1,7 +1,7 @@ #pragma once +#include "barretenberg/serialize/msgpack_impl.hpp" #include "bincode.hpp" -#include "msgpack.hpp" #include "serde.hpp" namespace Acir { diff --git a/barretenberg/cpp/src/barretenberg/dsl/acir_format/sha256_constraint.cpp b/barretenberg/cpp/src/barretenberg/dsl/acir_format/sha256_constraint.cpp index 80fd8a18867e..7ac05a1ba818 100644 --- a/barretenberg/cpp/src/barretenberg/dsl/acir_format/sha256_constraint.cpp +++ b/barretenberg/cpp/src/barretenberg/dsl/acir_format/sha256_constraint.cpp @@ -1,7 +1,7 @@ #include "sha256_constraint.hpp" +#include "barretenberg/serialize/msgpack_impl.hpp" #include "barretenberg/stdlib/hash/sha256/sha256.hpp" #include "barretenberg/stdlib/hash/sha256/sha256_plookup.hpp" -#include "msgpack/v3/unpack_decl.hpp" #include "round.hpp" namespace acir_format { diff --git a/barretenberg/cpp/src/barretenberg/dsl/acir_proofs/c_bind.cpp b/barretenberg/cpp/src/barretenberg/dsl/acir_proofs/c_bind.cpp index e2ac38475cc6..325e31aa4658 100644 --- a/barretenberg/cpp/src/barretenberg/dsl/acir_proofs/c_bind.cpp +++ b/barretenberg/cpp/src/barretenberg/dsl/acir_proofs/c_bind.cpp @@ -2,14 +2,17 @@ #include "../acir_format/acir_to_constraint_buf.hpp" #include "acir_composer.hpp" #include "barretenberg/client_ivc/client_ivc.hpp" +#include "barretenberg/client_ivc/private_execution_steps.hpp" #include "barretenberg/common/mem.hpp" #include "barretenberg/common/net.hpp" #include "barretenberg/common/serialize.hpp" #include "barretenberg/common/slab_allocator.hpp" +#include "barretenberg/common/zip_view.hpp" #include "barretenberg/dsl/acir_format/acir_format.hpp" #include "barretenberg/dsl/acir_format/ivc_recursion_constraint.hpp" #include "barretenberg/plonk/proof_system/proving_key/serialize.hpp" #include "barretenberg/plonk/proof_system/verification_key/verification_key.hpp" +#include "barretenberg/plonk_honk_shared/execution_trace/mega_execution_trace.hpp" #include "barretenberg/serialize/msgpack.hpp" #include "honk_contract.hpp" #include @@ -119,7 +122,7 @@ WASM_EXPORT void acir_init_verification_key(in_ptr acir_composer_ptr) WASM_EXPORT void acir_get_verification_key(in_ptr acir_composer_ptr, uint8_t** out) { - auto acir_composer = reinterpret_cast(*acir_composer_ptr); + auto* acir_composer = reinterpret_cast(*acir_composer_ptr); auto vk = acir_composer->init_verification_key(); // We flatten to a vector first, as that's how we treat it on the calling side. *out = to_heap_buffer(to_buffer(*vk)); @@ -130,7 +133,7 @@ WASM_EXPORT void acir_get_proving_key(in_ptr acir_composer_ptr, bool const* recursive, uint8_t** out) { - auto acir_composer = reinterpret_cast(*acir_composer_ptr); + auto* acir_composer = reinterpret_cast(*acir_composer_ptr); auto constraint_system = acir_format::circuit_buf_to_acir_format(from_buffer>(acir_vec), /*honk_recursion=*/0); acir_composer->create_finalized_circuit(constraint_system, *recursive); @@ -141,14 +144,14 @@ WASM_EXPORT void acir_get_proving_key(in_ptr acir_composer_ptr, WASM_EXPORT void acir_verify_proof(in_ptr acir_composer_ptr, uint8_t const* proof_buf, bool* result) { - auto acir_composer = reinterpret_cast(*acir_composer_ptr); + auto* acir_composer = reinterpret_cast(*acir_composer_ptr); auto proof = from_buffer>(proof_buf); *result = acir_composer->verify_proof(proof); } WASM_EXPORT void acir_get_solidity_verifier(in_ptr acir_composer_ptr, out_str_buf out) { - auto acir_composer = reinterpret_cast(*acir_composer_ptr); + auto* acir_composer = reinterpret_cast(*acir_composer_ptr); auto str = acir_composer->get_solidity_verifier(); *out = to_heap_buffer(str); } @@ -158,7 +161,7 @@ WASM_EXPORT void acir_serialize_proof_into_fields(in_ptr acir_composer_ptr, uint32_t const* num_inner_public_inputs, fr::vec_out_buf out) { - auto acir_composer = reinterpret_cast(*acir_composer_ptr); + auto* acir_composer = reinterpret_cast(*acir_composer_ptr); auto proof = from_buffer>(proof_buf); auto proof_as_fields = acir_composer->serialize_proof_into_fields(proof, ntohl(*num_inner_public_inputs)); @@ -169,7 +172,7 @@ WASM_EXPORT void acir_serialize_verification_key_into_fields(in_ptr acir_compose fr::vec_out_buf out_vkey, fr::out_buf out_key_hash) { - auto acir_composer = reinterpret_cast(*acir_composer_ptr); + auto* acir_composer = reinterpret_cast(*acir_composer_ptr); auto vkey_as_fields = acir_composer->serialize_verification_key_into_fields(); auto vk_hash = vkey_as_fields.back(); @@ -179,92 +182,14 @@ WASM_EXPORT void acir_serialize_verification_key_into_fields(in_ptr acir_compose write(out_key_hash, vk_hash); } -WASM_EXPORT void acir_prove_and_verify_aztec_client(uint8_t const* acir_stack, - uint8_t const* witness_stack, - bool* verified) -{ - using Program = acir_format::AcirProgram; - - std::vector> witnesses = from_buffer>>(witness_stack); - std::vector> acirs = from_buffer>>(acir_stack); - std::vector folding_stack; - - for (auto [bincode, wit] : zip_view(acirs, witnesses)) { - acir_format::WitnessVector witness = acir_format::witness_buf_to_witness_data(wit); - acir_format::AcirFormat constraints = acir_format::circuit_buf_to_acir_format(bincode, /*honk_recursion=*/0); - folding_stack.push_back(Program{ constraints, witness }); - } - // TODO(#7371) dedupe this with the rest of the similar code - TraceSettings trace_settings{ AZTEC_TRACE_STRUCTURE }; - auto ivc = std::make_shared(trace_settings); - - const acir_format::ProgramMetadata metadata{ ivc }; - - // Accumulate the entire program stack into the IVC - bool is_kernel = false; - auto start = std::chrono::steady_clock::now(); - for (Program& program : folding_stack) { - // Construct a bberg circuit from the acir representation then accumulate it into the IVC - vinfo("constructing circuit..."); - auto circuit = acir_format::create_circuit(program, metadata); - - // Set the internal is_kernel flag based on the local mechanism only if it has not already been set to true - if (!circuit.databus_propagation_data.is_kernel) { - circuit.databus_propagation_data.is_kernel = is_kernel; - } - is_kernel = !is_kernel; - - vinfo("done constructing circuit. calling ivc.accumulate..."); - ivc->accumulate(circuit); - vinfo("done accumulating."); - } - auto end = std::chrono::steady_clock::now(); - auto diff = std::chrono::duration_cast(end - start); - vinfo("time to construct and accumulate all circuits: ", diff.count()); - - vinfo("calling ivc.prove_and_verify..."); - bool result = ivc->prove_and_verify(); - info("verified?: ", result); - - end = std::chrono::steady_clock::now(); - diff = std::chrono::duration_cast(end - start); - vinfo("time to construct, accumulate, prove and verify all circuits: ", diff.count()); - - *verified = result; -} - -WASM_EXPORT void acir_prove_aztec_client(uint8_t const* acir_stack, - uint8_t const* witness_stack, - uint8_t** out_proof, - uint8_t** out_vk) +WASM_EXPORT void acir_prove_aztec_client(uint8_t const* ivc_inputs_buf, uint8_t** out_proof, uint8_t** out_vk) { - using Program = acir_format::AcirProgram; - - std::vector> witnesses = from_buffer>>(witness_stack); - std::vector> acirs = from_buffer>>(acir_stack); - std::vector folding_stack; - - for (auto [bincode, wit] : zip_view(acirs, witnesses)) { - acir_format::WitnessVector witness = acir_format::witness_buf_to_witness_data(wit); - acir_format::AcirFormat constraints = acir_format::circuit_buf_to_acir_format(bincode, /*honk_recursion=*/0); - folding_stack.push_back(Program{ constraints, witness }); - } - TraceSettings trace_settings{ AZTEC_TRACE_STRUCTURE }; - auto ivc = std::make_shared(trace_settings); - - const acir_format::ProgramMetadata metadata{ ivc }; - + auto ivc_inputs_vec = from_buffer>(ivc_inputs_buf); // Accumulate the entire program stack into the IVC auto start = std::chrono::steady_clock::now(); - for (Program& program : folding_stack) { - // Construct a bberg circuit from the acir representation then accumulate it into the IVC - vinfo("constructing circuit..."); - auto circuit = acir_format::create_circuit(program, metadata); - - vinfo("done constructing circuit. calling ivc.accumulate..."); - ivc->accumulate(circuit); - vinfo("done accumulating."); - } + PrivateExecutionSteps steps; + steps.parse(PrivateExecutionStepRaw::parse_uncompressed(ivc_inputs_vec)); + std::shared_ptr ivc = steps.accumulate(); auto end = std::chrono::steady_clock::now(); auto diff = std::chrono::duration_cast(end - start); vinfo("time to construct and accumulate all circuits: ", diff.count()); @@ -272,6 +197,7 @@ WASM_EXPORT void acir_prove_aztec_client(uint8_t const* acir_stack, vinfo("calling ivc.prove ..."); ClientIVC::Proof proof = ivc->prove(); end = std::chrono::steady_clock::now(); + diff = std::chrono::duration_cast(end - start); vinfo("time to construct, accumulate, prove all circuits: ", diff.count()); @@ -436,16 +362,21 @@ WASM_EXPORT void acir_vk_as_fields_mega_honk(uint8_t const* vk_buf, fr::vec_out_ *out_vkey = to_heap_buffer(vkey_as_fields); } -WASM_EXPORT void acir_gates_aztec_client(uint8_t const* acir_stack, uint8_t** out) +WASM_EXPORT void acir_gates_aztec_client(uint8_t const* ivc_inputs_buf, uint8_t** out) { - - std::vector> acirs = from_buffer>>(acir_stack); + auto ivc_inputs_vec = from_buffer>(ivc_inputs_buf); + // Note: we parse a stack, but only 'bytecode' needs to be set. + auto raw_steps = PrivateExecutionStepRaw::parse_uncompressed(ivc_inputs_vec); std::vector totals; TraceSettings trace_settings{ AZTEC_TRACE_STRUCTURE }; - for (auto& bincode : acirs) { + auto ivc = std::make_shared(trace_settings); + const acir_format::ProgramMetadata metadata{ ivc }; + + for (const PrivateExecutionStepRaw& step : raw_steps) { + std::vector bytecode_vec(step.bytecode.begin(), step.bytecode.end()); const acir_format::AcirFormat constraint_system = - acir_format::circuit_buf_to_acir_format(bincode, /*honk_recursion=*/0); + acir_format::circuit_buf_to_acir_format(bytecode_vec, /*honk_recursion=*/0); // Create an acir program from the constraint system acir_format::AcirProgram program{ constraint_system }; @@ -459,7 +390,6 @@ WASM_EXPORT void acir_gates_aztec_client(uint8_t const* acir_stack, uint8_t** ou builder.finalize_circuit(/*ensure_nonzero=*/true); totals.push_back(static_cast(builder.num_gates)); } - auto totalsBytes = to_buffer(totals); - *out = to_heap_buffer(totals); + *out = to_heap_buffer(to_buffer(totals)); } diff --git a/barretenberg/cpp/src/barretenberg/dsl/acir_proofs/c_bind.hpp b/barretenberg/cpp/src/barretenberg/dsl/acir_proofs/c_bind.hpp index ac6303fc9405..1aa5f5743569 100644 --- a/barretenberg/cpp/src/barretenberg/dsl/acir_proofs/c_bind.hpp +++ b/barretenberg/cpp/src/barretenberg/dsl/acir_proofs/c_bind.hpp @@ -47,14 +47,7 @@ WASM_EXPORT void acir_prove_and_verify_mega_honk(uint8_t const* constraint_syste uint8_t const* witness_buf, bool* result); -WASM_EXPORT void acir_prove_and_verify_aztec_client(uint8_t const* constraint_system_buf, - uint8_t const* witness_buf, - bool* result); - -WASM_EXPORT void acir_prove_aztec_client(uint8_t const* acir_stack, - uint8_t const* witness_stack, - uint8_t** out_proof, - uint8_t** out_vk); +WASM_EXPORT void acir_prove_aztec_client(uint8_t const* ivc_inputs_buf, uint8_t** out_proof, uint8_t** out_vk); WASM_EXPORT void acir_verify_aztec_client(uint8_t const* proof_buf, uint8_t const* vk_buf, bool* result); @@ -72,7 +65,7 @@ WASM_EXPORT void acir_get_proving_key(in_ptr acir_composer_ptr, WASM_EXPORT void acir_verify_proof(in_ptr acir_composer_ptr, uint8_t const* proof_buf, bool* result); WASM_EXPORT void acir_get_solidity_verifier(in_ptr acir_composer_ptr, out_str_buf out); -WASM_EXPORT void acir_honk_solidity_verifier(uint8_t const* proof_buf, uint8_t const* vk_buf, uint8_t** out); +WASM_EXPORT void acir_honk_solidity_verifier(uint8_t const* proof_buf, uint8_t const* vk_buf, out_str_buf out); WASM_EXPORT void acir_serialize_proof_into_fields(in_ptr acir_composer_ptr, uint8_t const* proof_buf, @@ -98,4 +91,4 @@ WASM_EXPORT void acir_vk_as_fields_ultra_honk(uint8_t const* vk_buf, fr::vec_out WASM_EXPORT void acir_vk_as_fields_mega_honk(uint8_t const* vk_buf, fr::vec_out_buf out_vkey); -WASM_EXPORT void acir_gates_aztec_client(uint8_t const* acir_stack, uint8_t** out); \ No newline at end of file +WASM_EXPORT void acir_gates_aztec_client(uint8_t const* ivc_inputs_buf, uint8_t** out); diff --git a/barretenberg/cpp/src/barretenberg/ecc/fields/field_impl_generic.hpp b/barretenberg/cpp/src/barretenberg/ecc/fields/field_impl_generic.hpp index 12caafe2f5aa..4489395f286e 100644 --- a/barretenberg/cpp/src/barretenberg/ecc/fields/field_impl_generic.hpp +++ b/barretenberg/cpp/src/barretenberg/ecc/fields/field_impl_generic.hpp @@ -1008,4 +1008,4 @@ template constexpr struct field::wide_array field::mul_512(const } // NOLINTEND(readability-implicit-bool-conversion) -} // namespace bb \ No newline at end of file +} // namespace bb diff --git a/barretenberg/cpp/src/barretenberg/goblin/goblin.cpp b/barretenberg/cpp/src/barretenberg/goblin/goblin.cpp index 15af11ca2da3..de2cf5f6abaf 100644 --- a/barretenberg/cpp/src/barretenberg/goblin/goblin.cpp +++ b/barretenberg/cpp/src/barretenberg/goblin/goblin.cpp @@ -15,6 +15,11 @@ Goblin::Goblin(const std::shared_ptr>& bn254_commitm Goblin::MergeProof Goblin::prove_merge(MegaBuilder& circuit_builder) { PROFILE_THIS_NAME("Goblin::merge"); + // TODO(https://github.com/AztecProtocol/barretenberg/issues/993): Some circuits (particularly on the first call + // to accumulate) may not have any goblin ecc ops prior to the call to merge(), so the commitment to the new + // contribution (C_t_shift) in the merge prover will be the point at infinity. (Note: Some dummy ops are added + // in 'add_gates_to_ensure...' but not until proving_key construction which comes later). See issue for ideas + // about how to resolve. if (circuit_builder.blocks.ecc_op.size() == 0) { MockCircuits::construct_goblin_ecc_op_circuit(circuit_builder); } diff --git a/barretenberg/cpp/src/barretenberg/goblin/goblin.hpp b/barretenberg/cpp/src/barretenberg/goblin/goblin.hpp index f516f1e9624a..dfa8be49fd4b 100644 --- a/barretenberg/cpp/src/barretenberg/goblin/goblin.hpp +++ b/barretenberg/cpp/src/barretenberg/goblin/goblin.hpp @@ -48,14 +48,40 @@ class Goblin { Goblin(const std::shared_ptr>& bn254_commitment_key = nullptr); + /** + * @brief Construct a merge proof for the goblin ECC ops in the provided circuit + * + * @param circuit_builder + */ MergeProof prove_merge(MegaBuilder& circuit_builder); + /** + * @brief Construct an ECCVM proof and the translation polynomial evaluations + */ void prove_eccvm(); + /** + * @brief Construct a translator proof + * + */ void prove_translator(); + /** + * @brief Constuct a full Goblin proof (ECCVM, Translator, merge) + * @details The merge proof is assumed to already have been constucted in the last accumulate step. It is simply + * moved into the final proof here. + * + * @return Proof + */ GoblinProof prove(MergeProof merge_proof_in = {}); + /** + * @brief Verify a full Goblin proof (ECCVM, Translator, merge) + * + * @param proof + * @return true + * @return false + */ static bool verify(const GoblinProof& proof); }; diff --git a/barretenberg/cpp/src/barretenberg/messaging/dispatcher.hpp b/barretenberg/cpp/src/barretenberg/messaging/dispatcher.hpp index f6ec73c9318c..e4f7234da3e2 100644 --- a/barretenberg/cpp/src/barretenberg/messaging/dispatcher.hpp +++ b/barretenberg/cpp/src/barretenberg/messaging/dispatcher.hpp @@ -1,7 +1,7 @@ #pragma once #include "barretenberg/messaging/header.hpp" -#include "barretenberg/serialize/cbind.hpp" +#include "barretenberg/serialize/msgpack_impl.hpp" #include #include #include diff --git a/barretenberg/cpp/src/barretenberg/messaging/stream_parser.hpp b/barretenberg/cpp/src/barretenberg/messaging/stream_parser.hpp index 9f5006dced1b..2c6641b3e9c8 100644 --- a/barretenberg/cpp/src/barretenberg/messaging/stream_parser.hpp +++ b/barretenberg/cpp/src/barretenberg/messaging/stream_parser.hpp @@ -1,7 +1,7 @@ #pragma once #include "barretenberg/messaging/header.hpp" -#include "barretenberg/serialize/cbind.hpp" +#include "barretenberg/serialize/msgpack_impl.hpp" #include "msgpack/v3/object_fwd_decl.hpp" #include #include diff --git a/barretenberg/cpp/src/barretenberg/nodejs_module/util/async_op.hpp b/barretenberg/cpp/src/barretenberg/nodejs_module/util/async_op.hpp index d5204a0c4d29..50eb35a52644 100644 --- a/barretenberg/cpp/src/barretenberg/nodejs_module/util/async_op.hpp +++ b/barretenberg/cpp/src/barretenberg/nodejs_module/util/async_op.hpp @@ -1,6 +1,6 @@ #pragma once -#include "barretenberg/serialize/cbind.hpp" +#include "barretenberg/serialize/msgpack_impl.hpp" #include #include #include diff --git a/barretenberg/cpp/src/barretenberg/nodejs_module/world_state/world_state.cpp b/barretenberg/cpp/src/barretenberg/nodejs_module/world_state/world_state.cpp index 7af648f0d2cd..d4a133130161 100644 --- a/barretenberg/cpp/src/barretenberg/nodejs_module/world_state/world_state.cpp +++ b/barretenberg/cpp/src/barretenberg/nodejs_module/world_state/world_state.cpp @@ -21,10 +21,9 @@ #include "barretenberg/nodejs_module/util/async_op.hpp" #include "barretenberg/nodejs_module/world_state/world_state.hpp" #include "barretenberg/nodejs_module/world_state/world_state_message.hpp" +#include "barretenberg/serialize/msgpack.hpp" #include "barretenberg/world_state/fork.hpp" #include "barretenberg/world_state/types.hpp" -#include "msgpack/v3/pack_decl.hpp" -#include "msgpack/v3/sbuffer_decl.hpp" #include "napi.h" using namespace bb::nodejs; diff --git a/barretenberg/cpp/src/barretenberg/plonk/proof_system/widgets/random_widgets/plookup_widget_impl.hpp b/barretenberg/cpp/src/barretenberg/plonk/proof_system/widgets/random_widgets/plookup_widget_impl.hpp index 3913120ee60d..48c821108b1d 100644 --- a/barretenberg/cpp/src/barretenberg/plonk/proof_system/widgets/random_widgets/plookup_widget_impl.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk/proof_system/widgets/random_widgets/plookup_widget_impl.hpp @@ -162,7 +162,7 @@ void ProverPlookupWidget::compute_gra key->polynomial_store.get("table_value_3_lagrange").data(), key->polynomial_store.get("table_value_4_lagrange").data(), }; - auto lagrange_base_tables = map(lagrange_base_tables_ptr, [](auto& e) { return e.get(); }); + auto lagrange_base_tables = transform::map(lagrange_base_tables_ptr, [](auto& e) { return e.get(); }); auto lookup_selector = key->polynomial_store.get("table_type_lagrange"); auto lookup_index_selector = key->polynomial_store.get("q_3_lagrange"); @@ -172,7 +172,7 @@ void ProverPlookupWidget::compute_gra key->polynomial_store.get("w_2_lagrange").data(), key->polynomial_store.get("w_3_lagrange").data(), }; - auto lagrange_base_wires = map(lagrange_base_wires_ptr, [](auto& e) { return e.get(); }); + auto lagrange_base_wires = transform::map(lagrange_base_wires_ptr, [](auto& e) { return e.get(); }); const fr beta_constant = beta + fr(1); // (1 + β) const fr gamma_beta_constant = gamma * beta_constant; // γ(1 + β) @@ -432,7 +432,7 @@ bb::fr ProverPlookupWidget::compute_q key->polynomial_store.get("w_2_fft").data(), key->polynomial_store.get("w_3_fft").data(), }; - auto wire_ffts = map(wire_ffts_ptr, [](auto& e) { return e.get(); }); + auto wire_ffts = transform::map(wire_ffts_ptr, [](auto& e) { return e.get(); }); auto s_fft = key->polynomial_store.get("s_fft"); @@ -442,7 +442,7 @@ bb::fr ProverPlookupWidget::compute_q key->polynomial_store.get("table_value_3_fft").data(), key->polynomial_store.get("table_value_4_fft").data(), }; - auto table_ffts = map(table_ffts_ptr, [](auto& e) { return e.get(); }); + auto table_ffts = transform::map(table_ffts_ptr, [](auto& e) { return e.get(); }); auto column_1_step_size = key->polynomial_store.get("q_2_fft"); auto column_2_step_size = key->polynomial_store.get("q_m_fft"); diff --git a/barretenberg/cpp/src/barretenberg/serialize/msgpack.hpp b/barretenberg/cpp/src/barretenberg/serialize/msgpack.hpp index 642affd82f17..b1b427ac7062 100644 --- a/barretenberg/cpp/src/barretenberg/serialize/msgpack.hpp +++ b/barretenberg/cpp/src/barretenberg/serialize/msgpack.hpp @@ -106,8 +106,10 @@ to the object itself, do break up the above to keep a reference to the handle, f } ``` */ +// For sbuffer forward declaration: #include "msgpack_impl/concepts.hpp" #include "msgpack_impl/name_value_pair_macro.hpp" +#include #include // Helper for above documented syntax diff --git a/barretenberg/cpp/src/barretenberg/serialize/cbind.hpp b/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl.hpp similarity index 94% rename from barretenberg/cpp/src/barretenberg/serialize/cbind.hpp rename to barretenberg/cpp/src/barretenberg/serialize/msgpack_impl.hpp index e2b717008fb5..307e5f746a38 100644 --- a/barretenberg/cpp/src/barretenberg/serialize/cbind.hpp +++ b/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl.hpp @@ -1,10 +1,11 @@ #pragma once -// Meant to be the main header included by translation units that use msgpack. +// Meant to be the main header included by *.cpp files* that use msgpack. // Note: heavy header due to serialization logic, don't include if msgpack.hpp will do // CBinding helpers that take a function or a lambda and // - bind the input as a coded msgpack array of all the arguments (using template metamagic) // - bind the return value to an out buffer, where the caller must free the memory +#define MSGPACK_NO_BOOST #include "barretenberg/common/try_catch_shim.hpp" #include "msgpack_impl/check_memory_span.hpp" #include "msgpack_impl/concepts.hpp" @@ -29,7 +30,7 @@ inline std::pair msgpack_encode_buffer(auto&& obj) msgpack::sbuffer buffer; msgpack::pack(buffer, obj); - uint8_t* output = (uint8_t*)aligned_alloc(64, buffer.size()); + uint8_t* output = static_cast(aligned_alloc(64, buffer.size())); memcpy(output, buffer.data(), buffer.size()); // Convert the buffer data to a string and return it return { output, buffer.size() }; @@ -60,7 +61,7 @@ inline void msgpack_cbind_impl(auto func, // The function to be ap // msgpack::unpack takes a buffer and its size, and returns an object_handle. // Calling .get() on that handle yields an object, and calling .convert on that // object converts it into the given type, in this case, the parameter tuple for func. - msgpack::unpack((const char*)input_in, input_len_in).get().convert(params); + msgpack::unpack(reinterpret_cast(input_in), input_len_in).get().convert(params); // Apply the function to the parameters, then encode the result into a MessagePack buffer. // std::apply takes a function and a tuple, and applies the function to the tuple's elements. @@ -80,7 +81,7 @@ inline void msgpack_cbind_schema_impl(auto func, uint8_t** output_out, size_t* o // Object representation of the cbind auto cbind_obj = get_func_traits(); std::string schema = msgpack_schema_to_string(cbind_obj); - *output_out = (uint8_t*)aligned_alloc(64, schema.size() + 1); + *output_out = static_cast(aligned_alloc(64, schema.size() + 1)); memcpy(*output_out, schema.c_str(), schema.size() + 1); *output_len_out = schema.size(); } diff --git a/barretenberg/cpp/src/barretenberg/serialize/msgpack_schema.test.cpp b/barretenberg/cpp/src/barretenberg/serialize/msgpack_schema.test.cpp index 33a8454dac31..90b0078c0c58 100644 --- a/barretenberg/cpp/src/barretenberg/serialize/msgpack_schema.test.cpp +++ b/barretenberg/cpp/src/barretenberg/serialize/msgpack_schema.test.cpp @@ -1,5 +1,5 @@ -#include "barretenberg/serialize/cbind.hpp" #include "barretenberg/serialize/msgpack.hpp" +#include "barretenberg/serialize/msgpack_impl.hpp" #include diff --git a/barretenberg/cpp/src/barretenberg/serialize/test_helper.hpp b/barretenberg/cpp/src/barretenberg/serialize/test_helper.hpp index 8c6efc418ed8..456e703e016c 100644 --- a/barretenberg/cpp/src/barretenberg/serialize/test_helper.hpp +++ b/barretenberg/cpp/src/barretenberg/serialize/test_helper.hpp @@ -1,5 +1,5 @@ #pragma once -#include "cbind.hpp" +#include "msgpack_impl.hpp" #include /*** diff --git a/barretenberg/cpp/src/barretenberg/smt_verification/circuit/circuit_schema.hpp b/barretenberg/cpp/src/barretenberg/smt_verification/circuit/circuit_schema.hpp index 099e60c1c732..9579bcdb1642 100644 --- a/barretenberg/cpp/src/barretenberg/smt_verification/circuit/circuit_schema.hpp +++ b/barretenberg/cpp/src/barretenberg/smt_verification/circuit/circuit_schema.hpp @@ -1,7 +1,6 @@ #pragma once #include -#include "barretenberg/serialize/cbind.hpp" #include "barretenberg/serialize/msgpack.hpp" namespace smt_circuit_schema { diff --git a/barretenberg/cpp/src/barretenberg/smt_verification/smt_polynomials.test.cpp b/barretenberg/cpp/src/barretenberg/smt_verification/smt_polynomials.test.cpp index f8e74061d442..de52c4d016e1 100644 --- a/barretenberg/cpp/src/barretenberg/smt_verification/smt_polynomials.test.cpp +++ b/barretenberg/cpp/src/barretenberg/smt_verification/smt_polynomials.test.cpp @@ -4,7 +4,7 @@ #include #include -#include "barretenberg/serialize/cbind.hpp" +#include "barretenberg/serialize/msgpack_impl.hpp #include "barretenberg/stdlib/primitives/field/field.hpp" #include "barretenberg/stdlib_circuit_builders/standard_circuit_builder.hpp" @@ -109,4 +109,4 @@ TEST(polynomial_evaluation, private) ASSERT_FALSE(res); info("Gates: ", circuit.get_num_gates()); info("Result: ", s.getResult()); -} \ No newline at end of file +} diff --git a/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/circuit_builder_base.hpp b/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/circuit_builder_base.hpp index c63b8005a9a5..217a88bfcaf4 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/circuit_builder_base.hpp +++ b/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/circuit_builder_base.hpp @@ -4,8 +4,8 @@ #include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp" #include "barretenberg/plonk_honk_shared/execution_trace/gate_data.hpp" #include "barretenberg/plonk_honk_shared/types/aggregation_object_type.hpp" +#include "barretenberg/serialize/msgpack.hpp" #include "barretenberg/stdlib_circuit_builders/public_component_key.hpp" -#include #include #include diff --git a/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/circuit_builder_base_impl.hpp b/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/circuit_builder_base_impl.hpp index 2adc7defe50a..5af8773c99c7 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/circuit_builder_base_impl.hpp +++ b/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/circuit_builder_base_impl.hpp @@ -1,5 +1,5 @@ #pragma once -#include "barretenberg/serialize/cbind.hpp" +#include "barretenberg/serialize/msgpack_impl.hpp" #include "circuit_builder_base.hpp" namespace bb { diff --git a/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/standard_circuit_builder.cpp b/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/standard_circuit_builder.cpp index 03effb888410..e9b658fd4cd5 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/standard_circuit_builder.cpp +++ b/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/standard_circuit_builder.cpp @@ -4,8 +4,7 @@ #include #include -#include "barretenberg/serialize/cbind.hpp" -#include "barretenberg/serialize/msgpack.hpp" +#include "barretenberg/serialize/msgpack_impl.hpp" namespace bb { diff --git a/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/ultra_circuit_builder.cpp b/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/ultra_circuit_builder.cpp index 5b2e9db05b4b..f477ca849b58 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/ultra_circuit_builder.cpp +++ b/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/ultra_circuit_builder.cpp @@ -7,7 +7,8 @@ */ #include "ultra_circuit_builder.hpp" #include "barretenberg/crypto/poseidon2/poseidon2_params.hpp" -#include +#include "barretenberg/plonk/proof_system/constants.hpp" +#include "barretenberg/serialize/msgpack_impl.hpp" #include #include #include diff --git a/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/ultra_circuit_builder.hpp b/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/ultra_circuit_builder.hpp index e926842b4aa2..327b5c6865c5 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/ultra_circuit_builder.hpp +++ b/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/ultra_circuit_builder.hpp @@ -12,7 +12,6 @@ #include #include -#include "barretenberg/serialize/cbind.hpp" #include "barretenberg/serialize/msgpack.hpp" namespace bb { diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp index 0d84107a5b51..fd46f8e2792e 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp @@ -12,6 +12,7 @@ #include "barretenberg/stdlib_circuit_builders/ultra_rollup_flavor.hpp" #include "barretenberg/stdlib_circuit_builders/ultra_zk_flavor.hpp" #include "barretenberg/trace_to_polynomials/trace_to_polynomials.hpp" +#include namespace bb { /** diff --git a/barretenberg/exports.json b/barretenberg/exports.json index ec246bb837b2..3e7eb0b1195d 100644 --- a/barretenberg/exports.json +++ b/barretenberg/exports.json @@ -128,23 +128,23 @@ "isAsync": false }, { - "functionName": "blake2s", + "functionName": "poseidon2_hash_accumulate", "inArgs": [ { - "name": "data", - "type": "const uint8_t *" + "name": "inputs_buffer", + "type": "fr::vec_in_buf" } ], "outArgs": [ { - "name": "r", - "type": "out_buf32" + "name": "output", + "type": "fr::out_buf" } ], "isAsync": false }, { - "functionName": "blake2s_to_field_", + "functionName": "blake2s", "inArgs": [ { "name": "data", @@ -154,214 +154,23 @@ "outArgs": [ { "name": "r", - "type": "fr::out_buf" - } - ], - "isAsync": false - }, - { - "functionName": "schnorr_compute_public_key", - "inArgs": [ - { - "name": "private_key", - "type": "fr::in_buf" - } - ], - "outArgs": [ - { - "name": "public_key_buf", - "type": "affine_element::out_buf" - } - ], - "isAsync": false - }, - { - "functionName": "schnorr_negate_public_key", - "inArgs": [ - { - "name": "public_key_buffer", - "type": "affine_element::in_buf" - } - ], - "outArgs": [ - { - "name": "output", - "type": "affine_element::out_buf" - } - ], - "isAsync": false - }, - { - "functionName": "schnorr_construct_signature", - "inArgs": [ - { - "name": "message", - "type": "const uint8_t *" - }, - { - "name": "private_key", - "type": "fr::in_buf" - } - ], - "outArgs": [ - { - "name": "s", "type": "out_buf32" - }, - { - "name": "e", - "type": "out_buf32" - } - ], - "isAsync": false - }, - { - "functionName": "schnorr_verify_signature", - "inArgs": [ - { - "name": "message", - "type": "const uint8_t *" - }, - { - "name": "pub_key", - "type": "affine_element::in_buf" - }, - { - "name": "sig_s", - "type": "in_buf32" - }, - { - "name": "sig_e", - "type": "in_buf32" - } - ], - "outArgs": [ - { - "name": "result", - "type": "bool *" - } - ], - "isAsync": false - }, - { - "functionName": "schnorr_multisig_create_multisig_public_key", - "inArgs": [ - { - "name": "private_key", - "type": "fq::in_buf" - } - ], - "outArgs": [ - { - "name": "multisig_pubkey_buf", - "type": "multisig::MultiSigPublicKey::out_buf" - } - ], - "isAsync": false - }, - { - "functionName": "schnorr_multisig_validate_and_combine_signer_pubkeys", - "inArgs": [ - { - "name": "signer_pubkey_buf", - "type": "multisig::MultiSigPublicKey::vec_in_buf" - } - ], - "outArgs": [ - { - "name": "combined_key_buf", - "type": "affine_element::out_buf" - }, - { - "name": "success", - "type": "bool *" - } - ], - "isAsync": false - }, - { - "functionName": "schnorr_multisig_construct_signature_round_1", - "inArgs": [], - "outArgs": [ - { - "name": "round_one_public_output_buf", - "type": "multisig::RoundOnePublicOutput::out_buf" - }, - { - "name": "round_one_private_output_buf", - "type": "multisig::RoundOnePrivateOutput::out_buf" } ], "isAsync": false }, { - "functionName": "schnorr_multisig_construct_signature_round_2", - "inArgs": [ - { - "name": "message", - "type": "const uint8_t *" - }, - { - "name": "private_key", - "type": "fq::in_buf" - }, - { - "name": "signer_round_one_private_buf", - "type": "multisig::RoundOnePrivateOutput::in_buf" - }, - { - "name": "signer_pubkeys_buf", - "type": "multisig::MultiSigPublicKey::vec_in_buf" - }, - { - "name": "round_one_public_buf", - "type": "multisig::RoundOnePublicOutput::vec_in_buf" - } - ], - "outArgs": [ - { - "name": "round_two_buf", - "type": "fq::out_buf" - }, - { - "name": "success", - "type": "bool *" - } - ], - "isAsync": false - }, - { - "functionName": "schnorr_multisig_combine_signatures", + "functionName": "blake2s_to_field_", "inArgs": [ { - "name": "message", + "name": "data", "type": "const uint8_t *" - }, - { - "name": "signer_pubkeys_buf", - "type": "multisig::MultiSigPublicKey::vec_in_buf" - }, - { - "name": "round_one_buf", - "type": "multisig::RoundOnePublicOutput::vec_in_buf" - }, - { - "name": "round_two_buf", - "type": "fq::vec_in_buf" } ], "outArgs": [ { - "name": "s", - "type": "out_buf32" - }, - { - "name": "e", - "type": "out_buf32" - }, - { - "name": "success", - "type": "bool *" + "name": "r", + "type": "fr::out_buf" } ], "isAsync": false @@ -515,10 +324,6 @@ } ], "outArgs": [ - { - "name": "exact", - "type": "uint32_t *" - }, { "name": "total", "type": "uint32_t *" @@ -530,20 +335,6 @@ ], "isAsync": false }, - { - "functionName": "acir_gates_client_ivc", - "inArgs": [ - { - "name": "acir_stack", - "type": "const uint8_t *" - }, - { - "name": "totals", - "type": "uint8_t **" - } - ], - "isAsync": false - }, { "functionName": "acir_new_acir_composer", "inArgs": [ @@ -625,10 +416,6 @@ "name": "constraint_system_buf", "type": "const uint8_t *" }, - { - "name": "recursive", - "type": "const bool *" - }, { "name": "witness_buf", "type": "const uint8_t *" @@ -649,10 +436,6 @@ "name": "constraint_system_buf", "type": "const uint8_t *" }, - { - "name": "recursive", - "type": "const bool *" - }, { "name": "witness_buf", "type": "const uint8_t *" @@ -667,18 +450,34 @@ "isAsync": false }, { - "functionName": "acir_fold_and_verify_program_stack", + "functionName": "acir_prove_aztec_client", "inArgs": [ { - "name": "constraint_system_buf", + "name": "ivc_inputs_buf", "type": "const uint8_t *" + } + ], + "outArgs": [ + { + "name": "out_proof", + "type": "uint8_t **" }, { - "name": "recursive", - "type": "const bool *" + "name": "out_vk", + "type": "uint8_t **" + } + ], + "isAsync": false + }, + { + "functionName": "acir_verify_aztec_client", + "inArgs": [ + { + "name": "proof_buf", + "type": "const uint8_t *" }, { - "name": "witness_buf", + "name": "vk_buf", "type": "const uint8_t *" } ], @@ -792,6 +591,26 @@ ], "isAsync": false }, + { + "functionName": "acir_honk_solidity_verifier", + "inArgs": [ + { + "name": "proof_buf", + "type": "const uint8_t *" + }, + { + "name": "vk_buf", + "type": "const uint8_t *" + } + ], + "outArgs": [ + { + "name": "out", + "type": "out_str_buf" + } + ], + "isAsync": false + }, { "functionName": "acir_serialize_proof_into_fields", "inArgs": [ @@ -844,8 +663,24 @@ "type": "const uint8_t *" }, { - "name": "recursive", - "type": "const bool *" + "name": "witness_vec", + "type": "const uint8_t *" + } + ], + "outArgs": [ + { + "name": "out", + "type": "uint8_t **" + } + ], + "isAsync": false + }, + { + "functionName": "acir_prove_ultra_keccak_honk", + "inArgs": [ + { + "name": "acir_vec", + "type": "const uint8_t *" }, { "name": "witness_vec", @@ -880,16 +715,48 @@ ], "isAsync": false }, + { + "functionName": "acir_verify_ultra_keccak_honk", + "inArgs": [ + { + "name": "proof_buf", + "type": "const uint8_t *" + }, + { + "name": "vk_buf", + "type": "const uint8_t *" + } + ], + "outArgs": [ + { + "name": "result", + "type": "bool *" + } + ], + "isAsync": false + }, { "functionName": "acir_write_vk_ultra_honk", "inArgs": [ { "name": "acir_vec", "type": "const uint8_t *" - }, + } + ], + "outArgs": [ { - "name": "recursive", - "type": "const bool *" + "name": "out", + "type": "uint8_t **" + } + ], + "isAsync": false + }, + { + "functionName": "acir_write_vk_ultra_keccak_honk", + "inArgs": [ + { + "name": "acir_vec", + "type": "const uint8_t *" } ], "outArgs": [ @@ -947,5 +814,21 @@ } ], "isAsync": false + }, + { + "functionName": "acir_gates_aztec_client", + "inArgs": [ + { + "name": "ivc_inputs_buf", + "type": "const uint8_t *" + } + ], + "outArgs": [ + { + "name": "out", + "type": "uint8_t **" + } + ], + "isAsync": false } -] \ No newline at end of file +] diff --git a/barretenberg/scripts/c_bind_files.txt b/barretenberg/scripts/c_bind_files.txt index e799a3895f11..740bf9283c70 100644 --- a/barretenberg/scripts/c_bind_files.txt +++ b/barretenberg/scripts/c_bind_files.txt @@ -2,7 +2,6 @@ ./cpp/src/barretenberg/crypto/pedersen_hash/c_bind.hpp ./cpp/src/barretenberg/crypto/poseidon2/c_bind.hpp ./cpp/src/barretenberg/crypto/blake2s/c_bind.hpp -./cpp/src/barretenberg/crypto/schnorr/c_bind.hpp ./cpp/src/barretenberg/crypto/aes128/c_bind.hpp ./cpp/src/barretenberg/srs/c_bind.hpp ./cpp/src/barretenberg/examples/simple/c_bind.hpp diff --git a/barretenberg/scripts/decls_json.py b/barretenberg/scripts/decls_json.py index a27edd0401e4..b1a9c19a4e64 100755 --- a/barretenberg/scripts/decls_json.py +++ b/barretenberg/scripts/decls_json.py @@ -32,8 +32,12 @@ def process_files(files: List[str]) -> List[dict]: result = [] idx = clang.cindex.Index.create() for path in files: + print(f"Processing {path}", file=sys.stderr) tu = idx.parse(path, args=[ "-I./cpp/src", + # HACK: this is a workaround for bindings gen finding tracy as it is referenced by the bb field header. + "-I./cpp/build/_deps/tracy-src/public", + "-I./cpp/build/_deps/msgpack-c/src/msgpack-c/include", '-std=gnu++20', '-Wall', '-Wextra']) for diag in tu.diagnostics: print_diagnostic(diag, file=sys.stderr) diff --git a/barretenberg/ts/bootstrap.sh b/barretenberg/ts/bootstrap.sh index fa9aa7d3c577..c2afaed86b8f 100755 --- a/barretenberg/ts/bootstrap.sh +++ b/barretenberg/ts/bootstrap.sh @@ -7,7 +7,7 @@ hash=$(cache_content_hash ../cpp/.rebuild_patterns .rebuild_patterns) function build { echo_header "bb.js build" - npm_install_deps + yarn install if ! cache_download bb.js-$hash.tar.gz; then find . -exec touch -d "@0" {} + 2>/dev/null || true diff --git a/barretenberg/ts/package.json b/barretenberg/ts/package.json index 9c965bac096b..2e5761878a96 100644 --- a/barretenberg/ts/package.json +++ b/barretenberg/ts/package.json @@ -72,7 +72,6 @@ }, "devDependencies": { "@jest/globals": "^29.4.3", - "@msgpack/msgpack": "^3.0.0-beta2", "@swc/core": "^1.10.1", "@swc/jest": "^0.2.37", "@types/debug": "^4.1.7", @@ -89,6 +88,7 @@ "html-webpack-plugin": "^5.5.1", "idb-keyval": "^6.2.1", "jest": "^29.5.0", + "msgpackr": "^1.11.2", "prettier": "^2.8.4", "resolve-typescript-plugin": "^2.0.1", "source-map-support": "^0.5.21", diff --git a/barretenberg/ts/src/barretenberg/backend.ts b/barretenberg/ts/src/barretenberg/backend.ts index 2aee4496bcdf..063ab0ceb13b 100644 --- a/barretenberg/ts/src/barretenberg/backend.ts +++ b/barretenberg/ts/src/barretenberg/backend.ts @@ -10,6 +10,7 @@ import { splitHonkProof, AGGREGATION_OBJECT_LENGTH, } from '../proof/index.js'; +import { Encoder } from 'msgpackr/pack'; export class AztecClientBackendError extends Error { constructor(message: string) { @@ -17,6 +18,24 @@ export class AztecClientBackendError extends Error { } } +// Utility for parsing gate counts from buffer +// TODO: Where should this logic live? Should go away with move to msgpack. +function parseBigEndianU32Array(buffer: Uint8Array): number[] { + const dv = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength); + + let offset = 0; + const count = buffer.byteLength >>> 2; // default is entire buffer length / 4 + console.log(buffer); + + const out: number[] = new Array(count); + for (let i = 0; i < count; i++) { + out[i] = dv.getUint32(offset, false); + offset += 4; + } + + return out; +} + export class UltraPlonkBackend { // These type assertions are used so that we don't // have to initialize `api` and `acirComposer` in the constructor. @@ -183,7 +202,7 @@ export class UltraHonkBackend { this.acirUncompressedBytecode = acirToUint8Array(acirBytecode); } /** @ignore */ - async instantiate(): Promise { + private async instantiate(): Promise { if (!this.api) { const api = await Barretenberg.new(this.backendOptions); const honkRecursion = true; @@ -291,6 +310,35 @@ export class UltraHonkBackend { await this.api.destroy(); } } +interface AztecClientExecutionStep { + functionName: string; + gateCount?: number; + // Note: not gzipped like in native code + bytecode: Uint8Array; + // Note: not gzipped like in native code. Already bincoded. + witness: Uint8Array; + /* TODO(https://github.com/AztecProtocol/barretenberg/issues/1328) this should get its own proper class. */ + vk: Uint8Array; +} + +function serializeAztecClientExecutionSteps(acirBuf: Uint8Array[], witnessBuf: Uint8Array[], vksBuf: Uint8Array[]): Uint8Array { + const steps: AztecClientExecutionStep[] = []; + for (let i = 0; i < acirBuf.length; i++) { + const bytecode = acirBuf[i]; + // Witnesses are not provided at all for gates info. + const witness = witnessBuf[i] || Buffer.from([]); + // VKs are optional for proving (deprecated feature) or not provided at all for gates info. + const vk = vksBuf[i] || Buffer.from([]); + const functionName = `unknown_wasm_${i}`; + steps.push({ + bytecode, + witness, + vk, + functionName, + }); + } + return new Encoder({ useRecords: false }).pack(steps); +} export class AztecClientBackend { // These type assertions are used so that we don't @@ -300,10 +348,10 @@ export class AztecClientBackend { protected api!: Barretenberg; - constructor(protected acirMsgpack: Uint8Array[], protected options: BackendOptions = { threads: 1 }) {} + constructor(protected acirBuf: Uint8Array[], protected options: BackendOptions = { threads: 1 }) {} /** @ignore */ - async instantiate(): Promise { + private async instantiate(): Promise { if (!this.api) { const api = await Barretenberg.new(this.options); await api.initSRSClientIVC(); @@ -311,9 +359,17 @@ export class AztecClientBackend { } } - async prove(witnessMsgpack: Uint8Array[]): Promise<[Uint8Array, Uint8Array]> { + async prove(witnessBuf: Uint8Array[], vksBuf: Uint8Array[] = []): Promise<[Uint8Array, Uint8Array]> { + if (vksBuf.length !== 0 && this.acirBuf.length !== witnessBuf.length) { + throw new AztecClientBackendError('Witness and bytecodes must have the same stack depth!'); + } + if (vksBuf.length !== 0 && vksBuf.length !== witnessBuf.length) { + // NOTE: we allow 0 as an explicit 'I have no VKs'. This is a deprecated feature. + throw new AztecClientBackendError('Witness and VKs must have the same stack depth!'); + } await this.instantiate(); - const proofAndVk = await this.api.acirProveAztecClient(this.acirMsgpack, witnessMsgpack); + const ivcInputsBuf = serializeAztecClientExecutionSteps(this.acirBuf, witnessBuf, vksBuf); + const proofAndVk = await this.api.acirProveAztecClient(ivcInputsBuf); const [proof, vk] = proofAndVk; if (!(await this.verify(proof, vk))) { throw new AztecClientBackendError('Failed to verify the private (ClientIVC) transaction proof!'); @@ -326,15 +382,12 @@ export class AztecClientBackend { return this.api.acirVerifyAztecClient(proof, vk); } - async proveAndVerify(witnessMsgpack: Uint8Array[]): Promise { - await this.instantiate(); - return this.api.acirProveAndVerifyAztecClient(this.acirMsgpack, witnessMsgpack); - } - async gates(): Promise { // call function on API await this.instantiate(); - return this.api.acirGatesAztecClient(this.acirMsgpack); + const ivcInputsBuf = serializeAztecClientExecutionSteps(this.acirBuf, [], []); + const resultBuffer = await this.api.acirGatesAztecClient(ivcInputsBuf); + return parseBigEndianU32Array(resultBuffer); } async destroy(): Promise { diff --git a/barretenberg/ts/src/barretenberg/schnorr.test.ts b/barretenberg/ts/src/barretenberg/schnorr.test.ts deleted file mode 100644 index b82195f2a08d..000000000000 --- a/barretenberg/ts/src/barretenberg/schnorr.test.ts +++ /dev/null @@ -1,182 +0,0 @@ -import { TextEncoder } from 'util'; -import { Buffer128, Buffer32, Fq, Fr, Point } from '../types/index.js'; -import { Barretenberg } from './index.js'; -import { asyncMap } from '../async_map/index.js'; - -describe('schnorr', () => { - const msg = Buffer.from(new TextEncoder().encode('The quick brown dog jumped over the lazy fox.')); - let api: Barretenberg; - - beforeAll(async () => { - api = await Barretenberg.new({ threads: 1 }); - }, 30000); - - afterAll(async () => { - await api.destroy(); - }); - - it('should verify signature', async () => { - const pk = Fr.fromBuffer( - new Uint8Array([ - 0x0b, 0x9b, 0x3a, 0xde, 0xe6, 0xb3, 0xd8, 0x1b, 0x28, 0xa0, 0x88, 0x6b, 0x2a, 0x84, 0x15, 0xc7, 0xda, 0x31, - 0x29, 0x1a, 0x5e, 0x96, 0xbb, 0x7a, 0x56, 0x63, 0x9e, 0x17, 0x7d, 0x30, 0x1b, 0xeb, - ]), - ); - const pubKey = await api.schnorrComputePublicKey(pk); - const [s, e] = await api.schnorrConstructSignature(msg, pk); - const verified = await api.schnorrVerifySignature(msg, pubKey, s, e); - - expect(verified).toBe(true); - }); - - it('public key negation should work', async () => { - const publicKeyStr = - '0x164f01b1011a1b292217acf53eef4d74f625f6e9bd5edfdb74c56fd81aafeebb21912735f9266a3719f61c1eb747ddee0cac9917f5c807485d356709b529b62c'; - const publicKey = Point.fromString(publicKeyStr); - // hardcoded expected negated public key - const expectedInvertedStr = - '0x164f01b1011a1b292217acf53eef4d74f625f6e9bd5edfdb74c56fd81aafeebb0ed3273ce80b35f29e5a2997ca397a6f1b874f3083f16948e6ac8e8a3ad649d5'; - const expectedInverted = Point.fromString(expectedInvertedStr); - - // negate - should match expected negated key - const negatedPublicKey = await api.schnorrNegatePublicKey(publicKey); - expect(negatedPublicKey.equals(expectedInverted)).toEqual(true); - // negate again - should be original public key now - expect((await api.schnorrNegatePublicKey(negatedPublicKey)).equals(publicKey)).toEqual(true); - }); - - it('should create + verify multi signature', async () => { - // set up multisig accounts - const numSigners = 7; - const pks = [...Array(numSigners)].map(() => Fq.random()); - const pubKeys = await asyncMap(pks, pk => api.schnorrMultisigCreateMultisigPublicKey(pk)); - - // round one - const roundOnePublicOutputs: Buffer128[] = []; - const roundOnePrivateOutputs: Buffer128[] = []; - for (let i = 0; i < numSigners; ++i) { - const [publicOutput, privateOutput] = await api.schnorrMultisigConstructSignatureRound1(); - roundOnePublicOutputs.push(publicOutput); - roundOnePrivateOutputs.push(privateOutput); - } - - // round two - const roundTwoOutputs = await asyncMap( - pks, - async (pk, i) => - ( - await api.schnorrMultisigConstructSignatureRound2( - msg, - pk, - roundOnePrivateOutputs[i], - pubKeys, - roundOnePublicOutputs, - ) - )[0], - ); - - // generate signature - const [s, e] = await api.schnorrMultisigCombineSignatures(msg, pubKeys, roundOnePublicOutputs, roundTwoOutputs)!; - const [combinedKey] = await api.schnorrMultisigValidateAndCombineSignerPubkeys(pubKeys); - expect(combinedKey).not.toEqual(Buffer.alloc(64)); - const verified = await api.schnorrVerifySignature(msg, combinedKey, s, e); - expect(verified).toBe(true); - }); - - it('should identify invalid multi signature', async () => { - const pks = [...Array(3)].map(() => Fq.random()); - const pubKeys = await asyncMap(pks, pk => api.schnorrMultisigCreateMultisigPublicKey(pk)); - const [combinedKey] = await api.schnorrMultisigValidateAndCombineSignerPubkeys(pubKeys); - - const verified = await api.schnorrVerifySignature(msg, combinedKey, Buffer32.random(), Buffer32.random()); - expect(verified).toBe(false); - }); - - it('should not construct invalid multi signature', async () => { - // set up multisig accounts - const numSigners = 7; - const pks = [...Array(numSigners)].map(() => Fq.random()); - const pubKeys = await asyncMap(pks, pk => api.schnorrMultisigCreateMultisigPublicKey(pk)); - - // round one - const roundOnePublicOutputs: Buffer128[] = []; - const roundOnePrivateOutputs: Buffer128[] = []; - for (let i = 0; i < numSigners; ++i) { - const [publicOutput, privateOutput] = await api.schnorrMultisigConstructSignatureRound1(); - roundOnePublicOutputs.push(publicOutput); - roundOnePrivateOutputs.push(privateOutput); - } - - // round two - const roundTwoOutputs = await asyncMap( - pks, - async (pk, i) => - ( - await api.schnorrMultisigConstructSignatureRound2( - msg, - pk, - roundOnePrivateOutputs[i], - pubKeys, - roundOnePublicOutputs, - ) - )[0], - ); - - // wrong number of data - { - expect( - ( - await api.schnorrMultisigCombineSignatures( - msg, - pubKeys.slice(0, -1), - roundOnePublicOutputs.slice(0, -1), - roundTwoOutputs.slice(0, -1), - ) - )[2], - ).toBe(false); - } - - // invalid round two output - { - const invalidOutputs = [...roundTwoOutputs]; - invalidOutputs[1] = ( - await api.schnorrMultisigConstructSignatureRound2( - msg, - pks[2], // <- Wrong private key. - roundOnePrivateOutputs[1], - pubKeys, - roundOnePublicOutputs, - ) - )[0]; - expect((await api.schnorrMultisigCombineSignatures(msg, pubKeys, roundOnePublicOutputs, invalidOutputs))[2]).toBe( - false, - ); - } - - // contains duplicates - { - const invalidOutputs = [...roundTwoOutputs]; - invalidOutputs[1] = roundTwoOutputs[2]; - expect((await api.schnorrMultisigCombineSignatures(msg, pubKeys, roundOnePublicOutputs, invalidOutputs))[2]).toBe( - false, - ); - } - }); - - it('should not create combined key from public keys containing invalid key', async () => { - const pks = [...Array(5)].map(() => Fq.random()); - const pubKeys = await asyncMap(pks, pk => api.schnorrMultisigCreateMultisigPublicKey(pk)); - - // not a valid point - { - pubKeys[1] = new Buffer128(Buffer.alloc(128)); - expect((await api.schnorrMultisigValidateAndCombineSignerPubkeys(pubKeys))[1]).toBe(false); - } - - // contains duplicates - { - pubKeys[1] = pubKeys[2]; - expect((await api.schnorrMultisigValidateAndCombineSignerPubkeys(pubKeys))[1]).toBe(false); - } - }); -}); diff --git a/barretenberg/ts/src/barretenberg_api/index.ts b/barretenberg/ts/src/barretenberg_api/index.ts index e08ab1f9bd0f..9825f183a32f 100644 --- a/barretenberg/ts/src/barretenberg_api/index.ts +++ b/barretenberg/ts/src/barretenberg_api/index.ts @@ -12,26 +12,6 @@ import { OutputType, } from '../serialize/index.js'; import { Fr, Fq, Point, Buffer32, Buffer128, Ptr } from '../types/index.js'; -function parseBigEndianU32Array(buffer: Uint8Array, hasSizePrefix = false): number[] { - const dv = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength); - - let offset = 0; - let count = buffer.byteLength >>> 2; // default is entire buffer length / 4 - - if (hasSizePrefix) { - // Read the first 4 bytes as the size (big-endian). - count = dv.getUint32(0, /* littleEndian= */ false); - offset = 4; - } - - const out: number[] = new Array(count); - for (let i = 0; i < count; i++) { - out[i] = dv.getUint32(offset, false); - offset += 4; - } - - return out; -} export class BarretenbergApi { constructor(protected wasm: BarretenbergWasmWorker | BarretenbergWasmMain) {} @@ -96,11 +76,11 @@ export class BarretenbergApi { return out[0]; } - async poseidon2HashAccumulate(inputsBuffer: Fr[]): Promise { + async poseidon2Hashes(inputsBuffer: Fr[]): Promise { const inArgs = [inputsBuffer].map(serializeBufferable); const outTypes: OutputType[] = [Fr]; const result = await this.wasm.callWasmExport( - 'poseidon2_hash_accumulate', + 'poseidon2_hashes', inArgs, outTypes.map(t => t.SIZE_IN_BYTES), ); @@ -108,11 +88,11 @@ export class BarretenbergApi { return out[0]; } - async poseidon2Hashes(inputsBuffer: Fr[]): Promise { + async poseidon2Permutation(inputsBuffer: Fr[]): Promise { const inArgs = [inputsBuffer].map(serializeBufferable); - const outTypes: OutputType[] = [Fr]; + const outTypes: OutputType[] = [VectorDeserializer(Fr)]; const result = await this.wasm.callWasmExport( - 'poseidon2_hashes', + 'poseidon2_permutation', inArgs, outTypes.map(t => t.SIZE_IN_BYTES), ); @@ -120,11 +100,11 @@ export class BarretenbergApi { return out[0]; } - async poseidon2Permutation(inputsBuffer: Fr[]): Promise { + async poseidon2HashAccumulate(inputsBuffer: Fr[]): Promise { const inArgs = [inputsBuffer].map(serializeBufferable); - const outTypes: OutputType[] = [VectorDeserializer(Fr)]; + const outTypes: OutputType[] = [Fr]; const result = await this.wasm.callWasmExport( - 'poseidon2_permutation', + 'poseidon2_hash_accumulate', inArgs, outTypes.map(t => t.SIZE_IN_BYTES), ); @@ -156,127 +136,6 @@ export class BarretenbergApi { return out[0]; } - async schnorrComputePublicKey(privateKey: Fr): Promise { - const inArgs = [privateKey].map(serializeBufferable); - const outTypes: OutputType[] = [Point]; - const result = await this.wasm.callWasmExport( - 'schnorr_compute_public_key', - inArgs, - outTypes.map(t => t.SIZE_IN_BYTES), - ); - const out = result.map((r, i) => outTypes[i].fromBuffer(r)); - return out[0]; - } - - async schnorrNegatePublicKey(publicKeyBuffer: Point): Promise { - const inArgs = [publicKeyBuffer].map(serializeBufferable); - const outTypes: OutputType[] = [Point]; - const result = await this.wasm.callWasmExport( - 'schnorr_negate_public_key', - inArgs, - outTypes.map(t => t.SIZE_IN_BYTES), - ); - const out = result.map((r, i) => outTypes[i].fromBuffer(r)); - return out[0]; - } - - async schnorrConstructSignature(message: Uint8Array, privateKey: Fr): Promise<[Buffer32, Buffer32]> { - const inArgs = [message, privateKey].map(serializeBufferable); - const outTypes: OutputType[] = [Buffer32, Buffer32]; - const result = await this.wasm.callWasmExport( - 'schnorr_construct_signature', - inArgs, - outTypes.map(t => t.SIZE_IN_BYTES), - ); - const out = result.map((r, i) => outTypes[i].fromBuffer(r)); - return out as any; - } - - async schnorrVerifySignature(message: Uint8Array, pubKey: Point, sigS: Buffer32, sigE: Buffer32): Promise { - const inArgs = [message, pubKey, sigS, sigE].map(serializeBufferable); - const outTypes: OutputType[] = [BoolDeserializer()]; - const result = await this.wasm.callWasmExport( - 'schnorr_verify_signature', - inArgs, - outTypes.map(t => t.SIZE_IN_BYTES), - ); - const out = result.map((r, i) => outTypes[i].fromBuffer(r)); - return out[0]; - } - - async schnorrMultisigCreateMultisigPublicKey(privateKey: Fq): Promise { - const inArgs = [privateKey].map(serializeBufferable); - const outTypes: OutputType[] = [Buffer128]; - const result = await this.wasm.callWasmExport( - 'schnorr_multisig_create_multisig_public_key', - inArgs, - outTypes.map(t => t.SIZE_IN_BYTES), - ); - const out = result.map((r, i) => outTypes[i].fromBuffer(r)); - return out[0]; - } - - async schnorrMultisigValidateAndCombineSignerPubkeys(signerPubkeyBuf: Buffer128[]): Promise<[Point, boolean]> { - const inArgs = [signerPubkeyBuf].map(serializeBufferable); - const outTypes: OutputType[] = [Point, BoolDeserializer()]; - const result = await this.wasm.callWasmExport( - 'schnorr_multisig_validate_and_combine_signer_pubkeys', - inArgs, - outTypes.map(t => t.SIZE_IN_BYTES), - ); - const out = result.map((r, i) => outTypes[i].fromBuffer(r)); - return out as any; - } - - async schnorrMultisigConstructSignatureRound1(): Promise<[Buffer128, Buffer128]> { - const inArgs = [].map(serializeBufferable); - const outTypes: OutputType[] = [Buffer128, Buffer128]; - const result = await this.wasm.callWasmExport( - 'schnorr_multisig_construct_signature_round_1', - inArgs, - outTypes.map(t => t.SIZE_IN_BYTES), - ); - const out = result.map((r, i) => outTypes[i].fromBuffer(r)); - return out as any; - } - - async schnorrMultisigConstructSignatureRound2( - message: Uint8Array, - privateKey: Fq, - signerRoundOnePrivateBuf: Buffer128, - signerPubkeysBuf: Buffer128[], - roundOnePublicBuf: Buffer128[], - ): Promise<[Fq, boolean]> { - const inArgs = [message, privateKey, signerRoundOnePrivateBuf, signerPubkeysBuf, roundOnePublicBuf].map( - serializeBufferable, - ); - const outTypes: OutputType[] = [Fq, BoolDeserializer()]; - const result = await this.wasm.callWasmExport( - 'schnorr_multisig_construct_signature_round_2', - inArgs, - outTypes.map(t => t.SIZE_IN_BYTES), - ); - const out = result.map((r, i) => outTypes[i].fromBuffer(r)); - return out as any; - } - - async schnorrMultisigCombineSignatures( - message: Uint8Array, - signerPubkeysBuf: Buffer128[], - roundOneBuf: Buffer128[], - roundTwoBuf: Fq[], - ): Promise<[Buffer32, Buffer32, boolean]> { - const inArgs = [message, signerPubkeysBuf, roundOneBuf, roundTwoBuf].map(serializeBufferable); - const outTypes: OutputType[] = [Buffer32, Buffer32, BoolDeserializer()]; - const result = await this.wasm.callWasmExport( - 'schnorr_multisig_combine_signatures', - inArgs, - outTypes.map(t => t.SIZE_IN_BYTES), - ); - const out = result.map((r, i) => outTypes[i].fromBuffer(r)); - return out as any; - } - async aesEncryptBufferCbc(input: Uint8Array, iv: Uint8Array, key: Uint8Array, length: number): Promise { const inArgs = [input, iv, key, length].map(serializeBufferable); const outTypes: OutputType[] = [BufferDeserializer()]; @@ -374,22 +233,7 @@ export class BarretenbergApi { outTypes.map(t => t.SIZE_IN_BYTES), ); const out = result.map((r, i) => outTypes[i].fromBuffer(r)); - return out as [number, number]; - } - - async acirGatesAztecClient( - // cf acirProveAztecClient - acirVec: Uint8Array[], - ): Promise { - const inArgs = [acirVec].map(serializeBufferable); - const outTypes: OutputType[] = [BufferDeserializer()]; - const resultBuffer = await this.wasm.callWasmExport( - 'acir_gates_aztec_client', - inArgs, - outTypes.map(t => t.SIZE_IN_BYTES), - ); - - return parseBigEndianU32Array(resultBuffer[0], /*hasSizePrefix=*/ true); + return out as any; } async acirNewAcirComposer(sizeHint: number): Promise { @@ -445,10 +289,7 @@ export class BarretenbergApi { return out[0]; } - async acirProveAndVerifyUltraHonk( - constraintSystemBuf: Uint8Array, - witnessBuf: Uint8Array, - ): Promise { + async acirProveAndVerifyUltraHonk(constraintSystemBuf: Uint8Array, witnessBuf: Uint8Array): Promise { const inArgs = [constraintSystemBuf, witnessBuf].map(serializeBufferable); const outTypes: OutputType[] = [BoolDeserializer()]; const result = await this.wasm.callWasmExport( @@ -460,10 +301,7 @@ export class BarretenbergApi { return out[0]; } - async acirProveAndVerifyMegaHonk( - constraintSystemBuf: Uint8Array, - witnessBuf: Uint8Array, - ): Promise { + async acirProveAndVerifyMegaHonk(constraintSystemBuf: Uint8Array, witnessBuf: Uint8Array): Promise { const inArgs = [constraintSystemBuf, witnessBuf].map(serializeBufferable); const outTypes: OutputType[] = [BoolDeserializer()]; const result = await this.wasm.callWasmExport( @@ -475,6 +313,30 @@ export class BarretenbergApi { return out[0]; } + async acirProveAztecClient(ivcInputsBuf: Uint8Array): Promise<[Uint8Array, Uint8Array]> { + const inArgs = [ivcInputsBuf].map(serializeBufferable); + const outTypes: OutputType[] = [BufferDeserializer(), BufferDeserializer()]; + const result = await this.wasm.callWasmExport( + 'acir_prove_aztec_client', + inArgs, + outTypes.map(t => t.SIZE_IN_BYTES), + ); + const out = result.map((r, i) => outTypes[i].fromBuffer(r)); + return out as any; + } + + async acirVerifyAztecClient(proofBuf: Uint8Array, vkBuf: Uint8Array): Promise { + const inArgs = [proofBuf, vkBuf].map(serializeBufferable); + const outTypes: OutputType[] = [BoolDeserializer()]; + const result = await this.wasm.callWasmExport( + 'acir_verify_aztec_client', + inArgs, + outTypes.map(t => t.SIZE_IN_BYTES), + ); + const out = result.map((r, i) => outTypes[i].fromBuffer(r)); + return out[0]; + } + async acirLoadVerificationKey(acirComposerPtr: Ptr, vkBuf: Uint8Array): Promise { const inArgs = [acirComposerPtr, vkBuf].map(serializeBufferable); const outTypes: OutputType[] = []; @@ -547,6 +409,18 @@ export class BarretenbergApi { return out[0]; } + async acirHonkSolidityVerifier(proofBuf: Uint8Array, vkBuf: Uint8Array): Promise { + const inArgs = [proofBuf, vkBuf].map(serializeBufferable); + const outTypes: OutputType[] = [StringDeserializer()]; + const result = await this.wasm.callWasmExport( + 'acir_honk_solidity_verifier', + inArgs, + outTypes.map(t => t.SIZE_IN_BYTES), + ); + const out = result.map((r, i) => outTypes[i].fromBuffer(r)); + return out[0]; + } + async acirSerializeProofIntoFields( acirComposerPtr: Ptr, proofBuf: Uint8Array, @@ -575,42 +449,6 @@ export class BarretenbergApi { return out as any; } - async acirProveAndVerifyAztecClient(acirVec: Uint8Array[], witnessVec: Uint8Array[]): Promise { - const inArgs = [acirVec, witnessVec].map(serializeBufferable); - const outTypes: OutputType[] = [BoolDeserializer()]; - const result = await this.wasm.callWasmExport( - 'acir_prove_and_verify_aztec_client', - inArgs, - outTypes.map(t => t.SIZE_IN_BYTES), - ); - const out = result.map((r, i) => outTypes[i].fromBuffer(r)); - return out[0]; - } - - async acirProveAztecClient(acirVec: Uint8Array[], witnessVec: Uint8Array[]): Promise<[Uint8Array, Uint8Array]> { - const inArgs = [acirVec, witnessVec].map(serializeBufferable); - const outTypes: OutputType[] = [BufferDeserializer(), BufferDeserializer()]; - const result = await this.wasm.callWasmExport( - 'acir_prove_aztec_client', - inArgs, - outTypes.map(t => t.SIZE_IN_BYTES), - ); - const out = result.map((r, i) => outTypes[i].fromBuffer(r)); - return [out[0], out[1]]; - } - - async acirVerifyAztecClient(proofBuf: Uint8Array, vkBuf: Uint8Array): Promise { - const inArgs = [proofBuf, vkBuf].map(serializeBufferable); - const outTypes: OutputType[] = [BoolDeserializer()]; - const result = await this.wasm.callWasmExport( - 'acir_verify_aztec_client', - inArgs, - outTypes.map(t => t.SIZE_IN_BYTES), - ); - const out = result.map((r, i) => outTypes[i].fromBuffer(r)); - return out[0]; - } - async acirProveUltraHonk(acirVec: Uint8Array, witnessVec: Uint8Array): Promise { const inArgs = [acirVec, witnessVec].map(serializeBufferable); const outTypes: OutputType[] = [BufferDeserializer()]; @@ -683,18 +521,6 @@ export class BarretenbergApi { return out[0]; } - async acirHonkSolidityVerifier(acirVec: Uint8Array, vkBuf: Uint8Array): Promise { - const inArgs = [acirVec, vkBuf].map(serializeBufferable); - const outTypes: OutputType[] = [BufferDeserializer()]; - const result = await this.wasm.callWasmExport( - 'acir_honk_solidity_verifier', - inArgs, - outTypes.map(t => t.SIZE_IN_BYTES), - ); - const out = result.map((r, i) => outTypes[i].fromBuffer(r)); - return out[0]; - } - async acirProofAsFieldsUltraHonk(proofBuf: Uint8Array): Promise { const inArgs = [proofBuf].map(serializeBufferable); const outTypes: OutputType[] = [VectorDeserializer(Fr)]; @@ -730,6 +556,18 @@ export class BarretenbergApi { const out = result.map((r, i) => outTypes[i].fromBuffer(r)); return out[0]; } + + async acirGatesAztecClient(ivcInputsBuf: Uint8Array): Promise { + const inArgs = [ivcInputsBuf].map(serializeBufferable); + const outTypes: OutputType[] = [BufferDeserializer()]; + const result = await this.wasm.callWasmExport( + 'acir_gates_aztec_client', + inArgs, + outTypes.map(t => t.SIZE_IN_BYTES), + ); + const out = result.map((r, i) => outTypes[i].fromBuffer(r)); + return out[0]; + } } export class BarretenbergApiSync { constructor(protected wasm: BarretenbergWasm) {} @@ -794,11 +632,11 @@ export class BarretenbergApiSync { return out[0]; } - poseidon2HashAccumulate(inputsBuffer: Fr[]): Fr { + poseidon2Hashes(inputsBuffer: Fr[]): Fr { const inArgs = [inputsBuffer].map(serializeBufferable); const outTypes: OutputType[] = [Fr]; const result = this.wasm.callWasmExport( - 'poseidon2_hash_accumulate', + 'poseidon2_hashes', inArgs, outTypes.map(t => t.SIZE_IN_BYTES), ); @@ -806,11 +644,11 @@ export class BarretenbergApiSync { return out[0]; } - poseidon2Hashes(inputsBuffer: Fr[]): Fr { + poseidon2Permutation(inputsBuffer: Fr[]): Fr[] { const inArgs = [inputsBuffer].map(serializeBufferable); - const outTypes: OutputType[] = [Fr]; + const outTypes: OutputType[] = [VectorDeserializer(Fr)]; const result = this.wasm.callWasmExport( - 'poseidon2_hashes', + 'poseidon2_permutation', inArgs, outTypes.map(t => t.SIZE_IN_BYTES), ); @@ -818,11 +656,11 @@ export class BarretenbergApiSync { return out[0]; } - poseidon2Permutation(inputsBuffer: Fr[]): Fr[] { + poseidon2HashAccumulate(inputsBuffer: Fr[]): Fr { const inArgs = [inputsBuffer].map(serializeBufferable); - const outTypes: OutputType[] = [VectorDeserializer(Fr)]; + const outTypes: OutputType[] = [Fr]; const result = this.wasm.callWasmExport( - 'poseidon2_permutation', + 'poseidon2_hash_accumulate', inArgs, outTypes.map(t => t.SIZE_IN_BYTES), ); @@ -854,127 +692,6 @@ export class BarretenbergApiSync { return out[0]; } - schnorrComputePublicKey(privateKey: Fr): Point { - const inArgs = [privateKey].map(serializeBufferable); - const outTypes: OutputType[] = [Point]; - const result = this.wasm.callWasmExport( - 'schnorr_compute_public_key', - inArgs, - outTypes.map(t => t.SIZE_IN_BYTES), - ); - const out = result.map((r, i) => outTypes[i].fromBuffer(r)); - return out[0]; - } - - schnorrNegatePublicKey(publicKeyBuffer: Point): Point { - const inArgs = [publicKeyBuffer].map(serializeBufferable); - const outTypes: OutputType[] = [Point]; - const result = this.wasm.callWasmExport( - 'schnorr_negate_public_key', - inArgs, - outTypes.map(t => t.SIZE_IN_BYTES), - ); - const out = result.map((r, i) => outTypes[i].fromBuffer(r)); - return out[0]; - } - - schnorrConstructSignature(message: Uint8Array, privateKey: Fr): [Buffer32, Buffer32] { - const inArgs = [message, privateKey].map(serializeBufferable); - const outTypes: OutputType[] = [Buffer32, Buffer32]; - const result = this.wasm.callWasmExport( - 'schnorr_construct_signature', - inArgs, - outTypes.map(t => t.SIZE_IN_BYTES), - ); - const out = result.map((r, i) => outTypes[i].fromBuffer(r)); - return out as any; - } - - schnorrVerifySignature(message: Uint8Array, pubKey: Point, sigS: Buffer32, sigE: Buffer32): boolean { - const inArgs = [message, pubKey, sigS, sigE].map(serializeBufferable); - const outTypes: OutputType[] = [BoolDeserializer()]; - const result = this.wasm.callWasmExport( - 'schnorr_verify_signature', - inArgs, - outTypes.map(t => t.SIZE_IN_BYTES), - ); - const out = result.map((r, i) => outTypes[i].fromBuffer(r)); - return out[0]; - } - - schnorrMultisigCreateMultisigPublicKey(privateKey: Fq): Buffer128 { - const inArgs = [privateKey].map(serializeBufferable); - const outTypes: OutputType[] = [Buffer128]; - const result = this.wasm.callWasmExport( - 'schnorr_multisig_create_multisig_public_key', - inArgs, - outTypes.map(t => t.SIZE_IN_BYTES), - ); - const out = result.map((r, i) => outTypes[i].fromBuffer(r)); - return out[0]; - } - - schnorrMultisigValidateAndCombineSignerPubkeys(signerPubkeyBuf: Buffer128[]): [Point, boolean] { - const inArgs = [signerPubkeyBuf].map(serializeBufferable); - const outTypes: OutputType[] = [Point, BoolDeserializer()]; - const result = this.wasm.callWasmExport( - 'schnorr_multisig_validate_and_combine_signer_pubkeys', - inArgs, - outTypes.map(t => t.SIZE_IN_BYTES), - ); - const out = result.map((r, i) => outTypes[i].fromBuffer(r)); - return out as any; - } - - schnorrMultisigConstructSignatureRound1(): [Buffer128, Buffer128] { - const inArgs = [].map(serializeBufferable); - const outTypes: OutputType[] = [Buffer128, Buffer128]; - const result = this.wasm.callWasmExport( - 'schnorr_multisig_construct_signature_round_1', - inArgs, - outTypes.map(t => t.SIZE_IN_BYTES), - ); - const out = result.map((r, i) => outTypes[i].fromBuffer(r)); - return out as any; - } - - schnorrMultisigConstructSignatureRound2( - message: Uint8Array, - privateKey: Fq, - signerRoundOnePrivateBuf: Buffer128, - signerPubkeysBuf: Buffer128[], - roundOnePublicBuf: Buffer128[], - ): [Fq, boolean] { - const inArgs = [message, privateKey, signerRoundOnePrivateBuf, signerPubkeysBuf, roundOnePublicBuf].map( - serializeBufferable, - ); - const outTypes: OutputType[] = [Fq, BoolDeserializer()]; - const result = this.wasm.callWasmExport( - 'schnorr_multisig_construct_signature_round_2', - inArgs, - outTypes.map(t => t.SIZE_IN_BYTES), - ); - const out = result.map((r, i) => outTypes[i].fromBuffer(r)); - return out as any; - } - - schnorrMultisigCombineSignatures( - message: Uint8Array, - signerPubkeysBuf: Buffer128[], - roundOneBuf: Buffer128[], - roundTwoBuf: Fq[], - ): [Buffer32, Buffer32, boolean] { - const inArgs = [message, signerPubkeysBuf, roundOneBuf, roundTwoBuf].map(serializeBufferable); - const outTypes: OutputType[] = [Buffer32, Buffer32, BoolDeserializer()]; - const result = this.wasm.callWasmExport( - 'schnorr_multisig_combine_signatures', - inArgs, - outTypes.map(t => t.SIZE_IN_BYTES), - ); - const out = result.map((r, i) => outTypes[i].fromBuffer(r)); - return out as any; - } - aesEncryptBufferCbc(input: Uint8Array, iv: Uint8Array, key: Uint8Array, length: number): Uint8Array { const inArgs = [input, iv, key, length].map(serializeBufferable); const outTypes: OutputType[] = [BufferDeserializer()]; @@ -1059,11 +776,7 @@ export class BarretenbergApiSync { return; } - acirGetCircuitSizes( - constraintSystemBuf: Uint8Array, - recursive: boolean, - honkRecursion: boolean, - ): [number, number, number] { + acirGetCircuitSizes(constraintSystemBuf: Uint8Array, recursive: boolean, honkRecursion: boolean): [number, number] { const inArgs = [constraintSystemBuf, recursive, honkRecursion].map(serializeBufferable); const outTypes: OutputType[] = [NumberDeserializer(), NumberDeserializer()]; const result = this.wasm.callWasmExport( @@ -1152,6 +865,30 @@ export class BarretenbergApiSync { return out[0]; } + acirProveAztecClient(ivcInputsBuf: Uint8Array): [Uint8Array, Uint8Array] { + const inArgs = [ivcInputsBuf].map(serializeBufferable); + const outTypes: OutputType[] = [BufferDeserializer(), BufferDeserializer()]; + const result = this.wasm.callWasmExport( + 'acir_prove_aztec_client', + inArgs, + outTypes.map(t => t.SIZE_IN_BYTES), + ); + const out = result.map((r, i) => outTypes[i].fromBuffer(r)); + return out as any; + } + + acirVerifyAztecClient(proofBuf: Uint8Array, vkBuf: Uint8Array): boolean { + const inArgs = [proofBuf, vkBuf].map(serializeBufferable); + const outTypes: OutputType[] = [BoolDeserializer()]; + const result = this.wasm.callWasmExport( + 'acir_verify_aztec_client', + inArgs, + outTypes.map(t => t.SIZE_IN_BYTES), + ); + const out = result.map((r, i) => outTypes[i].fromBuffer(r)); + return out[0]; + } + acirLoadVerificationKey(acirComposerPtr: Ptr, vkBuf: Uint8Array): void { const inArgs = [acirComposerPtr, vkBuf].map(serializeBufferable); const outTypes: OutputType[] = []; @@ -1224,6 +961,18 @@ export class BarretenbergApiSync { return out[0]; } + acirHonkSolidityVerifier(proofBuf: Uint8Array, vkBuf: Uint8Array): string { + const inArgs = [proofBuf, vkBuf].map(serializeBufferable); + const outTypes: OutputType[] = [StringDeserializer()]; + const result = this.wasm.callWasmExport( + 'acir_honk_solidity_verifier', + inArgs, + outTypes.map(t => t.SIZE_IN_BYTES), + ); + const out = result.map((r, i) => outTypes[i].fromBuffer(r)); + return out[0]; + } + acirSerializeProofIntoFields(acirComposerPtr: Ptr, proofBuf: Uint8Array, numInnerPublicInputs: number): Fr[] { const inArgs = [acirComposerPtr, proofBuf, numInnerPublicInputs].map(serializeBufferable); const outTypes: OutputType[] = [VectorDeserializer(Fr)]; @@ -1260,6 +1009,18 @@ export class BarretenbergApiSync { return out[0]; } + acirProveUltraKeccakHonk(acirVec: Uint8Array, witnessVec: Uint8Array): Uint8Array { + const inArgs = [acirVec, witnessVec].map(serializeBufferable); + const outTypes: OutputType[] = [BufferDeserializer()]; + const result = this.wasm.callWasmExport( + 'acir_prove_ultra_keccak_honk', + inArgs, + outTypes.map(t => t.SIZE_IN_BYTES), + ); + const out = result.map((r, i) => outTypes[i].fromBuffer(r)); + return out[0]; + } + acirVerifyUltraHonk(proofBuf: Uint8Array, vkBuf: Uint8Array): boolean { const inArgs = [proofBuf, vkBuf].map(serializeBufferable); const outTypes: OutputType[] = [BoolDeserializer()]; @@ -1272,6 +1033,18 @@ export class BarretenbergApiSync { return out[0]; } + acirVerifyUltraKeccakHonk(proofBuf: Uint8Array, vkBuf: Uint8Array): boolean { + const inArgs = [proofBuf, vkBuf].map(serializeBufferable); + const outTypes: OutputType[] = [BoolDeserializer()]; + const result = this.wasm.callWasmExport( + 'acir_verify_ultra_keccak_honk', + inArgs, + outTypes.map(t => t.SIZE_IN_BYTES), + ); + const out = result.map((r, i) => outTypes[i].fromBuffer(r)); + return out[0]; + } + acirWriteVkUltraHonk(acirVec: Uint8Array): Uint8Array { const inArgs = [acirVec].map(serializeBufferable); const outTypes: OutputType[] = [BufferDeserializer()]; @@ -1284,6 +1057,18 @@ export class BarretenbergApiSync { return out[0]; } + acirWriteVkUltraKeccakHonk(acirVec: Uint8Array): Uint8Array { + const inArgs = [acirVec].map(serializeBufferable); + const outTypes: OutputType[] = [BufferDeserializer()]; + const result = this.wasm.callWasmExport( + 'acir_write_vk_ultra_keccak_honk', + inArgs, + outTypes.map(t => t.SIZE_IN_BYTES), + ); + const out = result.map((r, i) => outTypes[i].fromBuffer(r)); + return out[0]; + } + acirProofAsFieldsUltraHonk(proofBuf: Uint8Array): Fr[] { const inArgs = [proofBuf].map(serializeBufferable); const outTypes: OutputType[] = [VectorDeserializer(Fr)]; @@ -1319,4 +1104,16 @@ export class BarretenbergApiSync { const out = result.map((r, i) => outTypes[i].fromBuffer(r)); return out[0]; } + + acirGatesAztecClient(ivcInputsBuf: Uint8Array): Uint8Array { + const inArgs = [ivcInputsBuf].map(serializeBufferable); + const outTypes: OutputType[] = [BufferDeserializer()]; + const result = this.wasm.callWasmExport( + 'acir_gates_aztec_client', + inArgs, + outTypes.map(t => t.SIZE_IN_BYTES), + ); + const out = result.map((r, i) => outTypes[i].fromBuffer(r)); + return out[0]; + } } diff --git a/barretenberg/ts/src/bindgen/mappings.ts b/barretenberg/ts/src/bindgen/mappings.ts index 834a3c0fe882..e6285da6ec89 100644 --- a/barretenberg/ts/src/bindgen/mappings.ts +++ b/barretenberg/ts/src/bindgen/mappings.ts @@ -13,6 +13,7 @@ const typeMap: { [key: string]: string } = { 'fq::vec_in_buf': 'Fq[]', 'fq::vec_out_buf': 'Fq[]', 'const uint8_t *': 'Uint8Array', + 'uint8_vec_vec_in_buf': 'Uint8Array[]', 'uint8_t **': 'Uint8Array', in_str_buf: 'string', out_str_buf: 'string', diff --git a/barretenberg/ts/src/bindgen/typescript.ts b/barretenberg/ts/src/bindgen/typescript.ts index b62453b95ae8..da6b253adf73 100644 --- a/barretenberg/ts/src/bindgen/typescript.ts +++ b/barretenberg/ts/src/bindgen/typescript.ts @@ -9,6 +9,7 @@ export function generateTypeScriptCode(filename: string) { let output = `// WARNING: FILE CODE GENERATED BY BINDGEN UTILITY. DO NOT EDIT! /* eslint-disable @typescript-eslint/no-unused-vars */ +import { BarretenbergWasmMain } from '../barretenberg_wasm/barretenberg_wasm_main/index.js'; import { BarretenbergWasmWorker, BarretenbergWasm } from '../barretenberg_wasm/index.js'; import { BufferDeserializer, NumberDeserializer, VectorDeserializer, BoolDeserializer, StringDeserializer, serializeBufferable, OutputType } from '../serialize/index.js'; import { Fr, Fq, Point, Buffer32, Buffer128, Ptr } from '../types/index.js'; @@ -24,7 +25,7 @@ import { Fr, Fq, Point, Buffer32, Buffer128, Ptr } from '../types/index.js'; function generateClass(functionDeclarations: FunctionDeclaration[]) { let output = ` export class BarretenbergApi { - constructor(protected wasm: BarretenbergWasmWorker) {} + constructor(protected wasm: BarretenbergWasmWorker | BarretenbergWasmMain) {} `; diff --git a/barretenberg/ts/src/main.ts b/barretenberg/ts/src/main.ts index 7dd6c6d78e37..3afac3f5ad10 100755 --- a/barretenberg/ts/src/main.ts +++ b/barretenberg/ts/src/main.ts @@ -4,9 +4,7 @@ import { Crs, GrumpkinCrs, Barretenberg, RawBuffer } from './index.js'; import createDebug from 'debug'; import { readFileSync, writeFileSync } from 'fs'; import { gunzipSync } from 'zlib'; -import { ungzip } from 'pako'; import { Command } from 'commander'; -import { decode } from '@msgpack/msgpack'; import { Timer, writeBenchmark } from './benchmark/index.js'; import path from 'path'; import { UltraHonkBackendOptions } from './barretenberg/backend.js'; @@ -46,16 +44,6 @@ function base64ToUint8Array(base64: string) { return bytes; } -function readStack(bytecodePath: string, numToDrop = 0) { - const encodedPackedZippedBytecodeArray = readFileSync(bytecodePath, 'utf-8'); - const packedZippedBytecodeArray = base64ToUint8Array(encodedPackedZippedBytecodeArray); - const zipped = decode( - packedZippedBytecodeArray.subarray(0, packedZippedBytecodeArray.length - numToDrop), - ) as Uint8Array[]; - const bytecodeArray = zipped.map((arr: Uint8Array) => ungzip(arr)); - return bytecodeArray; -} - // TODO(https://github.com/AztecProtocol/barretenberg/issues/1126): split this into separate Plonk and Honk functions as their gate count differs async function getGatesUltra(bytecodePath: string, recursive: boolean, honkRecursion: boolean, api: Barretenberg) { const { total } = await computeCircuitSize(bytecodePath, recursive, honkRecursion, api); @@ -222,22 +210,6 @@ export async function proveAndVerifyMegaHonk( /* eslint-enable camelcase */ } -export async function proveAndVerifyAztecClient(bytecodePath: string, witnessPath: string, crsPath: string) { - /* eslint-disable camelcase */ - const { api } = await initClientIVC(crsPath); - try { - const bytecode = readStack(bytecodePath); - const witness = readStack(witnessPath); - - const verified = await api.acirProveAndVerifyAztecClient(bytecode, witness); - debug(`Verification ${verified ? 'successful' : 'failed'}`); - return verified; - } finally { - await api.destroy(); - } - /* eslint-enable camelcase */ -} - export async function prove( bytecodePath: string, recursive: boolean, @@ -586,17 +558,6 @@ program process.exit(result ? 0 : 1); }); -program - .command('client_ivc_prove_and_verify') - .description('Generate a ClientIVC proof.') - .option('-b, --bytecode-path ', 'Specify the bytecode path', './target/acir.msgpack.b64') - .option('-w, --witness-path ', 'Specify the witness path', './target/witnesses.msgpack.b64') - .action(async ({ bytecodePath, witnessPath }) => { - const { crsPath } = handleGlobalOptions(); - const result = await proveAndVerifyAztecClient(bytecodePath, witnessPath, crsPath); - process.exit(result ? 0 : 1); - }); - program .command('prove') .description('Generate a proof and write it to a file.') diff --git a/barretenberg/ts/tsconfig.json b/barretenberg/ts/tsconfig.json index 63e9e9d02a3f..4363e17e42c5 100644 --- a/barretenberg/ts/tsconfig.json +++ b/barretenberg/ts/tsconfig.json @@ -12,6 +12,7 @@ "declarationMap": true, "importHelpers": true, "resolveJsonModule": true, + "skipLibCheck": true, "composite": true, "outDir": "dest/node", "rootDir": "src", diff --git a/barretenberg/ts/yarn.lock b/barretenberg/ts/yarn.lock index 8dcaa277e4f5..29e5606202d1 100644 --- a/barretenberg/ts/yarn.lock +++ b/barretenberg/ts/yarn.lock @@ -27,7 +27,6 @@ __metadata: resolution: "@aztec/bb.js@workspace:." dependencies: "@jest/globals": "npm:^29.4.3" - "@msgpack/msgpack": "npm:^3.0.0-beta2" "@swc/core": "npm:^1.10.1" "@swc/jest": "npm:^0.2.37" "@types/debug": "npm:^4.1.7" @@ -48,6 +47,7 @@ __metadata: html-webpack-plugin: "npm:^5.5.1" idb-keyval: "npm:^6.2.1" jest: "npm:^29.5.0" + msgpackr: "npm:^1.11.2" pako: "npm:^2.1.0" prettier: "npm:^2.8.4" resolve-typescript-plugin: "npm:^2.0.1" @@ -927,10 +927,45 @@ __metadata: languageName: node linkType: hard -"@msgpack/msgpack@npm:^3.0.0-beta2": - version: 3.0.0-beta2 - resolution: "@msgpack/msgpack@npm:3.0.0-beta2" - checksum: 10/d02f9221aa152cbd2977d1f56dc591baa2a37420a694cbc7e54ff0724f56ac0523e94de010e56bb845d67a1f2226c1761064b5777e63e9fc26884f4144d391a7 +"@msgpackr-extract/msgpackr-extract-darwin-arm64@npm:3.0.3": + version: 3.0.3 + resolution: "@msgpackr-extract/msgpackr-extract-darwin-arm64@npm:3.0.3" + conditions: os=darwin & cpu=arm64 + languageName: node + linkType: hard + +"@msgpackr-extract/msgpackr-extract-darwin-x64@npm:3.0.3": + version: 3.0.3 + resolution: "@msgpackr-extract/msgpackr-extract-darwin-x64@npm:3.0.3" + conditions: os=darwin & cpu=x64 + languageName: node + linkType: hard + +"@msgpackr-extract/msgpackr-extract-linux-arm64@npm:3.0.3": + version: 3.0.3 + resolution: "@msgpackr-extract/msgpackr-extract-linux-arm64@npm:3.0.3" + conditions: os=linux & cpu=arm64 + languageName: node + linkType: hard + +"@msgpackr-extract/msgpackr-extract-linux-arm@npm:3.0.3": + version: 3.0.3 + resolution: "@msgpackr-extract/msgpackr-extract-linux-arm@npm:3.0.3" + conditions: os=linux & cpu=arm + languageName: node + linkType: hard + +"@msgpackr-extract/msgpackr-extract-linux-x64@npm:3.0.3": + version: 3.0.3 + resolution: "@msgpackr-extract/msgpackr-extract-linux-x64@npm:3.0.3" + conditions: os=linux & cpu=x64 + languageName: node + linkType: hard + +"@msgpackr-extract/msgpackr-extract-win32-x64@npm:3.0.3": + version: 3.0.3 + resolution: "@msgpackr-extract/msgpackr-extract-win32-x64@npm:3.0.3" + conditions: os=win32 & cpu=x64 languageName: node linkType: hard @@ -2903,6 +2938,13 @@ __metadata: languageName: node linkType: hard +"detect-libc@npm:^2.0.1": + version: 2.0.3 + resolution: "detect-libc@npm:2.0.3" + checksum: 10/b4ea018d623e077bd395f168a9e81db77370dde36a5b01d067f2ad7989924a81d31cb547ff764acb2aa25d50bb7fdde0b0a93bec02212b0cb430621623246d39 + languageName: node + linkType: hard + "detect-newline@npm:^3.0.0": version: 3.1.0 resolution: "detect-newline@npm:3.1.0" @@ -5425,6 +5467,49 @@ __metadata: languageName: node linkType: hard +"msgpackr-extract@npm:^3.0.2": + version: 3.0.3 + resolution: "msgpackr-extract@npm:3.0.3" + dependencies: + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "npm:3.0.3" + "@msgpackr-extract/msgpackr-extract-darwin-x64": "npm:3.0.3" + "@msgpackr-extract/msgpackr-extract-linux-arm": "npm:3.0.3" + "@msgpackr-extract/msgpackr-extract-linux-arm64": "npm:3.0.3" + "@msgpackr-extract/msgpackr-extract-linux-x64": "npm:3.0.3" + "@msgpackr-extract/msgpackr-extract-win32-x64": "npm:3.0.3" + node-gyp: "npm:latest" + node-gyp-build-optional-packages: "npm:5.2.2" + dependenciesMeta: + "@msgpackr-extract/msgpackr-extract-darwin-arm64": + optional: true + "@msgpackr-extract/msgpackr-extract-darwin-x64": + optional: true + "@msgpackr-extract/msgpackr-extract-linux-arm": + optional: true + "@msgpackr-extract/msgpackr-extract-linux-arm64": + optional: true + "@msgpackr-extract/msgpackr-extract-linux-x64": + optional: true + "@msgpackr-extract/msgpackr-extract-win32-x64": + optional: true + bin: + download-msgpackr-prebuilds: bin/download-prebuilds.js + checksum: 10/4bfe45cf6968310570765951691f1b8e85b6a837e5197b8232fc9285eef4b457992e73118d9d07c92a52cc23f9e837897b135e17ea0f73e3604540434051b62f + languageName: node + linkType: hard + +"msgpackr@npm:^1.11.2": + version: 1.11.2 + resolution: "msgpackr@npm:1.11.2" + dependencies: + msgpackr-extract: "npm:^3.0.2" + dependenciesMeta: + msgpackr-extract: + optional: true + checksum: 10/7602f1e91e5ba13f4289ec9cab0d3f3db87d4ed323bebcb40a0c43ba2f6153192bffb63a5bb4755faacb6e0985f307c35084f40eaba1c325b7035da91381f01a + languageName: node + linkType: hard + "multicast-dns@npm:^7.2.5": version: 7.2.5 resolution: "multicast-dns@npm:7.2.5" @@ -5482,6 +5567,19 @@ __metadata: languageName: node linkType: hard +"node-gyp-build-optional-packages@npm:5.2.2": + version: 5.2.2 + resolution: "node-gyp-build-optional-packages@npm:5.2.2" + dependencies: + detect-libc: "npm:^2.0.1" + bin: + node-gyp-build-optional-packages: bin.js + node-gyp-build-optional-packages-optional: optional.js + node-gyp-build-optional-packages-test: build-test.js + checksum: 10/f448a328cf608071dc8cc4426ac5be0daec4788e4e1759e9f7ffcd286822cc799384edce17a8c79e610c4bbfc8e3aff788f3681f1d88290e0ca7aaa5342a090f + languageName: node + linkType: hard + "node-gyp@npm:latest": version: 10.0.1 resolution: "node-gyp@npm:10.0.1" diff --git a/playground/vite.config.ts b/playground/vite.config.ts index 3b2304c78c76..1d928bb67ed8 100644 --- a/playground/vite.config.ts +++ b/playground/vite.config.ts @@ -59,7 +59,9 @@ export default defineConfig(({ mode }) => { // ], // }), bundlesize({ - limits: [{ name: 'assets/index-*', limit: '1600kB' }], + // Bump log: + // - AD: bumped from 1600 => 1680 as we now have a 20kb msgpack lib in bb.js and other logic got us 50kb higher, adding some wiggle room. + limits: [{ name: 'assets/index-*', limit: '1680kB' }], }), ], define: { diff --git a/yarn-project/bb-prover/package.json b/yarn-project/bb-prover/package.json index a40fb3604404..db26ebc63668 100644 --- a/yarn-project/bb-prover/package.json +++ b/yarn-project/bb-prover/package.json @@ -76,7 +76,6 @@ "@aztec/stdlib": "workspace:^", "@aztec/telemetry-client": "workspace:^", "@aztec/world-state": "workspace:^", - "@msgpack/msgpack": "^3.0.0-beta2", "commander": "^12.1.0", "pako": "^2.1.0", "source-map-support": "^0.5.21", diff --git a/yarn-project/bb-prover/scripts/generate_civc_vks.sh b/yarn-project/bb-prover/scripts/generate_civc_vks.sh index a024e9eba7eb..a7a2c91fc48c 100755 --- a/yarn-project/bb-prover/scripts/generate_civc_vks.sh +++ b/yarn-project/bb-prover/scripts/generate_civc_vks.sh @@ -11,9 +11,9 @@ cd .. # and for purposes of VK generation, stale inputs work just fine. # IF NEW INPUTS SUDDENLY DO NOT VERIFY IN THE IVC BENCH - we need to redo this: # - Generate inputs: $root/yarn-project/end-to-end/bootstrap.sh generate_example_app_ivc_inputs -# - Upload the compressed results: aws s3 cp ./bb-civc-inputs-v2.tar.gz s3://aztec-ci-artifacts/protocol/bb-civc-inputs-[version].tar.gz +# - Upload the compressed results: aws s3 cp bb-civc-inputs-[version].tar.gz s3://aztec-ci-artifacts/protocol/bb-civc-inputs-[version].tar.gz -pinned_civc_inputs_url="https://aztec-ci-artifacts.s3.us-east-2.amazonaws.com/protocol/bb-civc-inputs-v2.tar.gz" +pinned_civc_inputs_url="https://aztec-ci-artifacts.s3.us-east-2.amazonaws.com/protocol/bb-civc-inputs-v3.tar.gz" hash=$(hash_str $(../bootstrap.sh hash) "$pinned_civc_inputs_url") if cache_download bb-prover-vks-$hash.tar.gz; then diff --git a/yarn-project/bb-prover/src/bb/execute.ts b/yarn-project/bb-prover/src/bb/execute.ts index 48ec9247309f..699465c6c6d3 100644 --- a/yarn-project/bb-prover/src/bb/execute.ts +++ b/yarn-project/bb-prover/src/bb/execute.ts @@ -119,8 +119,7 @@ export function executeBB( export async function executeBbClientIvcProof( pathToBB: string, workingDirectory: string, - bytecodeStackPath: string, - witnessStackPath: string, + inputsPath: string, log: LogFn, writeVk = false, ): Promise { @@ -144,30 +143,16 @@ export async function executeBbClientIvcProof( try { // Write the bytecode to the working directory - log(`bytecodePath ${bytecodeStackPath}`); - log(`outputPath ${outputPath}`); - const args = [ - '-o', - outputPath, - '-b', - bytecodeStackPath, - '-w', - witnessStackPath, - '-v', - '--scheme', - 'client_ivc', - '--input_type', - 'runtime_stack', - ]; - if (writeVk) { - args.push('--write_vk'); - } - + log(`inputsPath ${inputsPath}`); const timer = new Timer(); const logFunction = (message: string) => { log(`bb - ${message}`); }; + const args = ['-o', outputPath, '--ivc_inputs_path', inputsPath, '-v', '--scheme', 'client_ivc']; + if (writeVk) { + args.push('--write_vk'); + } const result = await executeBB(pathToBB, 'prove', args, logFunction); const durationMs = timer.ms(); diff --git a/yarn-project/bb-prover/src/prover/bb_native_private_kernel_prover.ts b/yarn-project/bb-prover/src/prover/bb_native_private_kernel_prover.ts index fbe3ed5ccbf2..2a69a3516ebd 100644 --- a/yarn-project/bb-prover/src/prover/bb_native_private_kernel_prover.ts +++ b/yarn-project/bb-prover/src/prover/bb_native_private_kernel_prover.ts @@ -1,12 +1,10 @@ import { runInDirectory } from '@aztec/foundation/fs'; import { type Logger, createLogger } from '@aztec/foundation/log'; -import { serializeWitness } from '@aztec/noir-noirc_abi'; import { BundleArtifactProvider } from '@aztec/noir-protocol-circuits-types/client/bundle'; import type { SimulationProvider } from '@aztec/simulator/server'; -import type { PrivateExecutionStep } from '@aztec/stdlib/kernel'; +import { type PrivateExecutionStep, serializePrivateExecutionSteps } from '@aztec/stdlib/kernel'; import type { ClientIvcProof } from '@aztec/stdlib/proofs'; -import { encode } from '@msgpack/msgpack'; import { promises as fs } from 'fs'; import path from 'path'; @@ -40,33 +38,13 @@ export class BBNativePrivateKernelProver extends BBPrivateKernelProver { ); } - // TODO(#7371): This is duplicated. - // Longer term we won't use this hacked together msgpack format - // Leaving duplicated as this eventually bb will provide a serialization - // helper for passing to a generic msgpack RPC endpoint. - private async _createClientIvcProofFiles(directory: string, executionSteps: PrivateExecutionStep[]) { - const acirPath = path.join(directory, 'acir.msgpack'); - const witnessPath = path.join(directory, 'witnesses.msgpack'); - await fs.writeFile(acirPath, encode(executionSteps.map(map => map.bytecode))); - await fs.writeFile(witnessPath, encode(executionSteps.map(map => serializeWitness(map.witness)))); - return { - acirPath, - witnessPath, - }; - } - private async _createClientIvcProof( directory: string, executionSteps: PrivateExecutionStep[], ): Promise { - await this._createClientIvcProofFiles(directory, executionSteps); - const provingResult = await executeBbClientIvcProof( - this.bbBinaryPath, - directory, - path.join(directory, 'acir.msgpack'), - path.join(directory, 'witnesses.msgpack'), - this.log.info, - ); + const inputsPath = path.join(directory, 'ivc-inputs.msgpack'); + await fs.writeFile(inputsPath, serializePrivateExecutionSteps(executionSteps)); + const provingResult = await executeBbClientIvcProof(this.bbBinaryPath, directory, inputsPath, this.log.info); if (provingResult.status === BB_RESULT.FAILURE) { this.log.error(`Failed to generate client ivc proof`); diff --git a/yarn-project/bb-prover/src/prover/bb_private_kernel_prover.ts b/yarn-project/bb-prover/src/prover/bb_private_kernel_prover.ts index 7b23f62a2c62..b490c4715b97 100644 --- a/yarn-project/bb-prover/src/prover/bb_private_kernel_prover.ts +++ b/yarn-project/bb-prover/src/prover/bb_private_kernel_prover.ts @@ -219,7 +219,7 @@ export abstract class BBPrivateKernelProver implements PrivateKernelProver { outputSize: output.toBuffer().length, } satisfies CircuitWitnessGenerationStats); - const verificationKey = (await this.artifactProvider.getCircuitVkByName(circuitType)).keyAsFields; + const verificationKey = await this.artifactProvider.getCircuitVkByName(circuitType); const bytecode = Buffer.from(compiledCircuit.bytecode, 'base64'); const kernelOutput: PrivateKernelSimulateOutput = { @@ -236,7 +236,7 @@ export abstract class BBPrivateKernelProver implements PrivateKernelProver { >(publicInputs: PublicInputsType, circuitType: ClientProtocolArtifact) { const kernelProofOutput: PrivateKernelSimulateOutput = { publicInputs, - verificationKey: (await this.artifactProvider.getCircuitVkByName(circuitType)).keyAsFields, + verificationKey: await this.artifactProvider.getCircuitVkByName(circuitType), outputWitness: new Map(), bytecode: Buffer.from([]), }; diff --git a/yarn-project/bb-prover/src/wasm/bb_wasm_private_kernel_prover.ts b/yarn-project/bb-prover/src/wasm/bb_wasm_private_kernel_prover.ts index aaf4d75ffc4d..426b04a83cfc 100644 --- a/yarn-project/bb-prover/src/wasm/bb_wasm_private_kernel_prover.ts +++ b/yarn-project/bb-prover/src/wasm/bb_wasm_private_kernel_prover.ts @@ -31,7 +31,10 @@ export abstract class BBWASMPrivateKernelProver extends BBPrivateKernelProver { // TODO(https://github.com/AztecProtocol/barretenberg/issues/1297): the vk is not provided to the network anymore. // Move this sanity check inside the wasm code and remove the vk from the return value. - const [proof, _vk] = await backend.prove(executionSteps.map(step => ungzip(serializeWitness(step.witness)))); + const [proof, _vk] = await backend.prove( + executionSteps.map(step => ungzip(serializeWitness(step.witness))), + executionSteps.map(step => step.vk), + ); await backend.destroy(); this.log.info(`Generated ClientIVC proof`, { eventName: 'client-ivc-proof-generation', @@ -42,6 +45,7 @@ export abstract class BBWASMPrivateKernelProver extends BBPrivateKernelProver { } public override async computeGateCountForCircuit(_bytecode: Buffer, _circuitName: string): Promise { + // Note we do not pass the vk to the backend. This is unneeded for gate counts. const backend = new AztecClientBackend([ungzip(_bytecode)], { threads: this.threads, logger: this.log.verbose, diff --git a/yarn-project/cli-wallet/package.json b/yarn-project/cli-wallet/package.json index 97972e85d905..5ed0d7056cd8 100644 --- a/yarn-project/cli-wallet/package.json +++ b/yarn-project/cli-wallet/package.json @@ -71,10 +71,9 @@ "@aztec/foundation": "workspace:^", "@aztec/kv-store": "workspace:^", "@aztec/noir-contracts.js": "workspace:^", - "@aztec/noir-noirc_abi": "workspace:^", + "@aztec/noir-noirc_abi": "portal:../../noir/packages/noirc_abi", "@aztec/pxe": "workspace:^", "@aztec/stdlib": "workspace:^", - "@msgpack/msgpack": "^3.0.0-beta2", "commander": "^12.1.0", "inquirer": "^10.1.8", "source-map-support": "^0.5.21", diff --git a/yarn-project/cli-wallet/src/cmds/profile.ts b/yarn-project/cli-wallet/src/cmds/profile.ts index 296a54572d8e..e143826bbe81 100644 --- a/yarn-project/cli-wallet/src/cmds/profile.ts +++ b/yarn-project/cli-wallet/src/cmds/profile.ts @@ -1,11 +1,9 @@ import { type AccountWalletWithSecretKey, AuthWitness, type AztecAddress, Contract } from '@aztec/aztec.js'; import { prepTx } from '@aztec/cli/utils'; import type { LogFn } from '@aztec/foundation/log'; -import { serializeWitness } from '@aztec/noir-noirc_abi'; -import type { PrivateExecutionStep } from '@aztec/stdlib/kernel'; +import { serializePrivateExecutionSteps } from '@aztec/stdlib/kernel'; import type { TxProfileResult } from '@aztec/stdlib/tx'; -import { encode } from '@msgpack/msgpack'; import { promises as fs } from 'fs'; import path from 'path'; import { format } from 'util'; @@ -31,21 +29,6 @@ function printProfileResult(result: TxProfileResult, log: LogFn) { log(format('\nTotal gates:', acc.toLocaleString())); } -// TODO(#7371): This is duplicated. -// Longer term we won't use this hacked together msgpack format -// Leaving duplicated as this eventually bb will provide a serialization -// helper for passing to a generic msgpack RPC endpoint. -async function _createClientIvcProofFiles(directory: string, executionSteps: PrivateExecutionStep[]) { - const acirPath = path.join(directory, 'acir.msgpack'); - const witnessPath = path.join(directory, 'witnesses.msgpack'); - await fs.writeFile(acirPath, encode(executionSteps.map(map => map.bytecode))); - await fs.writeFile(witnessPath, encode(executionSteps.map(map => serializeWitness(map.witness)))); - return { - acirPath, - witnessPath, - }; -} - export async function profile( wallet: AccountWalletWithSecretKey, functionName: string, @@ -65,7 +48,8 @@ export async function profile( const result = await call.profile({ profileMode, authWitnesses }); printProfileResult(result, log); if (debugOutputPath) { - log(`Debug output written to ${debugOutputPath} (witnesses.msgpack and acir.msgpack)`); - await _createClientIvcProofFiles(debugOutputPath, result.executionSteps); + const ivcInputsPath = path.join(debugOutputPath, 'ivc-inputs.msgpack'); + log(`Debug output written to ${ivcInputsPath}.`); + await fs.writeFile(ivcInputsPath, serializePrivateExecutionSteps(result.executionSteps)); } } diff --git a/yarn-project/end-to-end/package.json b/yarn-project/end-to-end/package.json index 13ac813e0435..10c029afb94a 100644 --- a/yarn-project/end-to-end/package.json +++ b/yarn-project/end-to-end/package.json @@ -44,7 +44,7 @@ "@aztec/l1-artifacts": "workspace:^", "@aztec/merkle-tree": "workspace:^", "@aztec/noir-contracts.js": "workspace:^", - "@aztec/noir-noirc_abi": "workspace:^", + "@aztec/noir-noirc_abi": "portal:../../noir/packages/noirc_abi", "@aztec/noir-protocol-circuits-types": "workspace:^", "@aztec/p2p": "workspace:^", "@aztec/protocol-contracts": "workspace:^", @@ -59,7 +59,6 @@ "@aztec/world-state": "workspace:^", "@iarna/toml": "^2.2.5", "@jest/globals": "^29.5.0", - "@msgpack/msgpack": "^3.0.0-beta2", "@noble/curves": "^1.0.0", "@swc/core": "^1.4.11", "@swc/jest": "^0.2.36", @@ -84,6 +83,7 @@ "lodash.compact": "^3.0.1", "lodash.every": "^4.6.0", "lodash.omit": "^4.5.0", + "msgpackr": "^1.11.2", "process": "^0.11.10", "stream-browserify": "^3.0.0", "string-argv": "^0.3.2", diff --git a/yarn-project/end-to-end/src/bench/client_flows/data_extractor.ts b/yarn-project/end-to-end/src/bench/client_flows/data_extractor.ts index 1ee1a1815d84..cfe51828800a 100644 --- a/yarn-project/end-to-end/src/bench/client_flows/data_extractor.ts +++ b/yarn-project/end-to-end/src/bench/client_flows/data_extractor.ts @@ -5,7 +5,7 @@ import { createLogger, logger } from '@aztec/foundation/log'; import { WASMSimulator } from '@aztec/simulator/client'; import type { PrivateExecutionStep } from '@aztec/stdlib/kernel'; -import { decode } from '@msgpack/msgpack'; +import { Decoder } from 'msgpackr'; import assert from 'node:assert'; import { readFile, readdir, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; @@ -149,9 +149,10 @@ async function main() { for (const flow of flows) { userLog.info(`Processing flow ${flow}`); - const bytecode = await readFile(join(ivcFolder, flow, 'acir.msgpack')); - const acirStack = decode(bytecode) as Buffer[]; + const ivcInputs = await readFile(join(ivcFolder, flow, 'ivc-inputs.msgpack')); + const stepsFromFile: PrivateExecutionStep[] = new Decoder({ useRecords: false }).unpack(ivcInputs); const witnesses = await readFile(join(ivcFolder, flow, 'witnesses.json')); + const witnessStack = JSON.parse(witnesses.toString()).map((witnessMap: Record) => { return new Map(Object.entries(witnessMap).map(([k, v]) => [Number(k), v])); }); @@ -160,8 +161,11 @@ async function main() { const privateExecutionSteps: PrivateExecutionStep[] = executionSteps.map((step, i) => ({ functionName: step.fnName, gateCount: step.gateCount, - bytecode: acirStack[i], + bytecode: stepsFromFile[i].bytecode, + // TODO(AD) do we still want to take this from witness.json? witness: witnessStack[i], + // This can be left empty. If so, the prover will generate a vk on the fly (~25% slower). + vk: Buffer.from([]), })); let stats: { duration: number; eventName: string; proofSize: number } | undefined; // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/yarn-project/end-to-end/src/shared/capture_private_execution_steps.ts b/yarn-project/end-to-end/src/shared/capture_private_execution_steps.ts index 8aff46943d5e..8d3609ada372 100644 --- a/yarn-project/end-to-end/src/shared/capture_private_execution_steps.ts +++ b/yarn-project/end-to-end/src/shared/capture_private_execution_steps.ts @@ -9,30 +9,13 @@ import type { ProfileMethodOptions, } from '@aztec/aztec.js/contracts'; import { createLogger } from '@aztec/foundation/log'; -import { serializeWitness } from '@aztec/noir-noirc_abi'; -import type { PrivateExecutionStep } from '@aztec/stdlib/kernel'; +import { serializePrivateExecutionSteps } from '@aztec/stdlib/kernel'; -import { encode } from '@msgpack/msgpack'; import { promises as fs } from 'fs'; import path from 'path'; const logger = createLogger('e2e:capture-private-execution-steps'); -// TODO(#7371): This is duplicated. -// Longer term we won't use this hacked together msgpack format -// Leaving duplicated as this eventually bb will provide a serialization -// helper for passing to a generic msgpack RPC endpoint. -async function _createClientIvcProofFiles(directory: string, executionSteps: PrivateExecutionStep[]) { - const acirPath = path.join(directory, 'acir.msgpack'); - const witnessPath = path.join(directory, 'witnesses.msgpack'); - await fs.writeFile(acirPath, encode(executionSteps.map(map => map.bytecode))); - await fs.writeFile(witnessPath, encode(executionSteps.map(map => serializeWitness(map.witness)))); - return { - acirPath, - witnessPath, - }; -} - export async function capturePrivateExecutionStepsIfEnvSet( label: string, interaction: ContractFunctionInteraction | DeployMethod, @@ -56,7 +39,8 @@ export async function capturePrivateExecutionStepsIfEnvSet( logger.info(`Writing private execution steps to ${resultsDirectory}`); await fs.mkdir(resultsDirectory, { recursive: true }); // Write the client IVC files read by the prover. - await _createClientIvcProofFiles(resultsDirectory, result.executionSteps); + const ivcInputsPath = path.join(resultsDirectory, 'ivc-inputs.msgpack'); + await fs.writeFile(ivcInputsPath, serializePrivateExecutionSteps(result.executionSteps)); if (profileMode === 'full') { // If we have gate counts, write the steps in human-readable format. await fs.writeFile( diff --git a/yarn-project/ivc-integration/package.json b/yarn-project/ivc-integration/package.json index b28f6d71b23f..8021cbc5db01 100644 --- a/yarn-project/ivc-integration/package.json +++ b/yarn-project/ivc-integration/package.json @@ -81,7 +81,6 @@ "@aztec/telemetry-client": "workspace:^", "@aztec/world-state": "workspace:^", "@jest/globals": "^29.5.0", - "@msgpack/msgpack": "^3.0.0-beta2", "@playwright/test": "1.49.0", "@types/jest": "^29.5.0", "@types/node": "^22.8.1", @@ -91,6 +90,7 @@ "html-webpack-plugin": "^5.6.0", "jest": "^29.5.0", "jest-mock-extended": "^4.0.0-beta1", + "msgpackr": "^1.11.2", "resolve-typescript-plugin": "^2.0.1", "serve": "^14.2.1", "ts-loader": "^9.5.1", diff --git a/yarn-project/ivc-integration/src/native_client_ivc_integration.test.ts b/yarn-project/ivc-integration/src/native_client_ivc_integration.test.ts index 2b0932aea65b..2732721ff353 100644 --- a/yarn-project/ivc-integration/src/native_client_ivc_integration.test.ts +++ b/yarn-project/ivc-integration/src/native_client_ivc_integration.test.ts @@ -9,8 +9,6 @@ import { getWorkingDirectory } from './bb_working_directory.js'; import { generate3FunctionTestingIVCStack, generate6FunctionTestingIVCStack } from './index.js'; import { proveClientIVC } from './prove_native.js'; -/* eslint-disable camelcase */ - const logger = createLogger('ivc-integration:test:native'); jest.setTimeout(120_000); diff --git a/yarn-project/ivc-integration/src/prove_native.ts b/yarn-project/ivc-integration/src/prove_native.ts index cee86666af7e..5dfbc2d2d144 100644 --- a/yarn-project/ivc-integration/src/prove_native.ts +++ b/yarn-project/ivc-integration/src/prove_native.ts @@ -29,8 +29,8 @@ import type { NoirCompiledCircuit } from '@aztec/stdlib/noir'; import type { ClientIvcProof, Proof } from '@aztec/stdlib/proofs'; import type { VerificationKeyData } from '@aztec/stdlib/vks'; -import { encode } from '@msgpack/msgpack'; import * as fs from 'fs/promises'; +import { Encoder } from 'msgpackr'; import * as path from 'path'; export async function proveClientIVC( @@ -40,17 +40,22 @@ export async function proveClientIVC( bytecodes: string[], logger: Logger, ): Promise { - await fs.writeFile( - path.join(bbWorkingDirectory, 'acir.msgpack'), - encode(bytecodes.map(bytecode => Buffer.from(bytecode, 'base64'))), - ); + const stepToStruct = (bytecode: string, index: number) => { + return { + bytecode: Buffer.from(bytecode, 'base64'), + witness: witnessStack[index], + vk: Buffer.from([]), + functionName: `unknown_${index}`, + }; + }; + const encoded = new Encoder({ useRecords: false }).pack(bytecodes.map(stepToStruct)); + const ivcInputsPath = path.join(bbWorkingDirectory, 'ivc-inputs.msgpack'); + await fs.writeFile(ivcInputsPath, encoded); - await fs.writeFile(path.join(bbWorkingDirectory, 'witnesses.msgpack'), encode(witnessStack)); const provingResult = await executeBbClientIvcProof( bbBinaryPath, bbWorkingDirectory, - path.join(bbWorkingDirectory, 'acir.msgpack'), - path.join(bbWorkingDirectory, 'witnesses.msgpack'), + ivcInputsPath, logger.info, true, ); diff --git a/yarn-project/ivc-integration/src/prove_wasm.ts b/yarn-project/ivc-integration/src/prove_wasm.ts index 1ad50deb9492..2352ec1226d0 100644 --- a/yarn-project/ivc-integration/src/prove_wasm.ts +++ b/yarn-project/ivc-integration/src/prove_wasm.ts @@ -1,8 +1,11 @@ +import { createLogger } from '@aztec/foundation/log'; import { ClientIvcProof } from '@aztec/stdlib/proofs'; import os from 'os'; import { ungzip } from 'pako'; +const logger = createLogger('ivc-integration:prove_wasm'); + function base64ToUint8Array(base64: string): Uint8Array { return Uint8Array.from(atob(base64), c => c.charCodeAt(0)); } @@ -15,7 +18,7 @@ export async function proveClientIVC( const { AztecClientBackend } = await import('@aztec/bb.js'); const backend = new AztecClientBackend( bytecodes.map(base64ToUint8Array).map((arr: Uint8Array) => ungzip(arr)), - { threads: threads || Math.min(os.cpus().length, 16) }, + { threads: threads || Math.min(os.cpus().length, 16), logger: logger.info }, ); try { const [proof] = await backend.prove(witnessStack.map((arr: Uint8Array) => ungzip(arr))); @@ -33,9 +36,10 @@ export async function proveThenVerifyAztecClient( const { AztecClientBackend } = await import('@aztec/bb.js'); const backend = new AztecClientBackend( bytecodes.map(base64ToUint8Array).map((arr: Uint8Array) => ungzip(arr)), - { threads: threads || Math.min(os.cpus().length, 16) }, + { threads: threads || Math.min(os.cpus().length, 16), logger: logger.info }, ); try { + // These are optional - easier not to pass them. const [proof, vk] = await backend.prove(witnessStack.map((arr: Uint8Array) => ungzip(arr))); const verified = await backend.verify(proof, vk); return verified; diff --git a/yarn-project/ivc-integration/src/rollup_ivc_integration.test.ts b/yarn-project/ivc-integration/src/rollup_ivc_integration.test.ts index a2b9c31715e5..cbadd8481f49 100644 --- a/yarn-project/ivc-integration/src/rollup_ivc_integration.test.ts +++ b/yarn-project/ivc-integration/src/rollup_ivc_integration.test.ts @@ -54,7 +54,6 @@ describe('Rollup IVC Integration', () => { // Create a client IVC proof const clientIVCWorkingDirectory = await getWorkingDirectory('bb-rollup-ivc-integration-client-ivc-'); - const [bytecodes, witnessStack, tailPublicInputs] = await generate3FunctionTestingIVCStack(); clientIVCPublicInputs = tailPublicInputs; const proof = await proveClientIVC(bbBinaryPath, clientIVCWorkingDirectory, witnessStack, bytecodes, logger); diff --git a/yarn-project/ivc-integration/src/wasm_client_ivc_integration.test.ts b/yarn-project/ivc-integration/src/wasm_client_ivc_integration.test.ts index b6c2aa38aa22..7427fcd0b0a0 100644 --- a/yarn-project/ivc-integration/src/wasm_client_ivc_integration.test.ts +++ b/yarn-project/ivc-integration/src/wasm_client_ivc_integration.test.ts @@ -85,7 +85,10 @@ describe('Client IVC Integration', () => { ]; // Initialize AztecClientBackend with the given bytecodes - const backend = new AztecClientBackend(bytecodes.map(base64ToUint8Array).map((arr: Uint8Array) => ungzip(arr))); + const backend = new AztecClientBackend( + bytecodes.map(base64ToUint8Array).map((arr: Uint8Array) => ungzip(arr)), + { threads: 1, logger: logger.info }, + ); // Compute the numbers of gates in each circuit const gateNumbers = await backend.gates(); diff --git a/yarn-project/noir-protocol-circuits-types/src/conversion/client.ts b/yarn-project/noir-protocol-circuits-types/src/conversion/client.ts index cbcf7ec850e9..2f19a2c426a4 100644 --- a/yarn-project/noir-protocol-circuits-types/src/conversion/client.ts +++ b/yarn-project/noir-protocol-circuits-types/src/conversion/client.ts @@ -657,7 +657,10 @@ export function mapPrivateKernelDataToNoir( privateKernelInnerData: PrivateKernelData, ): PrivateKernelDataWithoutPublicInputsNoir { return { - vk: mapVerificationKeyToNoir(privateKernelInnerData.vk, CLIENT_IVC_VERIFICATION_KEY_LENGTH_IN_FIELDS), + vk: mapVerificationKeyToNoir( + privateKernelInnerData.verificationKey.keyAsFields, + CLIENT_IVC_VERIFICATION_KEY_LENGTH_IN_FIELDS, + ), vk_index: mapFieldToNoir(new Fr(privateKernelInnerData.vkIndex)), vk_path: mapTuple(privateKernelInnerData.vkPath, mapFieldToNoir), }; diff --git a/yarn-project/pxe/package.json b/yarn-project/pxe/package.json index b453ebb89572..c7ce94c722e6 100644 --- a/yarn-project/pxe/package.json +++ b/yarn-project/pxe/package.json @@ -70,7 +70,6 @@ "@aztec/protocol-contracts": "workspace:^", "@aztec/simulator": "workspace:^", "@aztec/stdlib": "workspace:^", - "@msgpack/msgpack": "^3.0.0-beta2", "koa": "^2.16.1", "koa-router": "^12.0.0", "lodash.omit": "^4.5.0", diff --git a/yarn-project/pxe/src/private_kernel/hints/build_private_kernel_reset_private_inputs.ts b/yarn-project/pxe/src/private_kernel/hints/build_private_kernel_reset_private_inputs.ts index 0bc99cc42b9f..cad280136aa7 100644 --- a/yarn-project/pxe/src/private_kernel/hints/build_private_kernel_reset_private_inputs.ts +++ b/yarn-project/pxe/src/private_kernel/hints/build_private_kernel_reset_private_inputs.ts @@ -163,7 +163,9 @@ export class PrivateKernelResetPrivateInputsBuilder { allowRemainder, ); - const previousVkMembershipWitness = await oracle.getVkMembershipWitness(this.previousKernelOutput.verificationKey); + const previousVkMembershipWitness = await oracle.getVkMembershipWitness( + this.previousKernelOutput.verificationKey.keyAsFields, + ); const previousKernelData = new PrivateKernelData( this.previousKernelOutput.publicInputs, this.previousKernelOutput.verificationKey, diff --git a/yarn-project/pxe/src/private_kernel/private_kernel_execution_prover.test.ts b/yarn-project/pxe/src/private_kernel/private_kernel_execution_prover.test.ts index e60baa415824..229914ef0ef0 100644 --- a/yarn-project/pxe/src/private_kernel/private_kernel_execution_prover.test.ts +++ b/yarn-project/pxe/src/private_kernel/private_kernel_execution_prover.test.ts @@ -1,9 +1,4 @@ -import { - CLIENT_IVC_VERIFICATION_KEY_LENGTH_IN_FIELDS, - MAX_NOTE_HASHES_PER_CALL, - MAX_NOTE_HASHES_PER_TX, - VK_TREE_HEIGHT, -} from '@aztec/constants'; +import { MAX_NOTE_HASHES_PER_CALL, MAX_NOTE_HASHES_PER_TX, VK_TREE_HEIGHT } from '@aztec/constants'; import { makeTuple } from '@aztec/foundation/array'; import { Fr } from '@aztec/foundation/fields'; import { MembershipWitness } from '@aztec/foundation/trees'; @@ -21,7 +16,7 @@ import { PublicKeys } from '@aztec/stdlib/keys'; import { Note } from '@aztec/stdlib/note'; import { makeTxRequest } from '@aztec/stdlib/testing'; import { NoteAndSlot, PrivateCallExecutionResult, PrivateExecutionResult, type TxRequest } from '@aztec/stdlib/tx'; -import { VerificationKey, VerificationKeyAsFields } from '@aztec/stdlib/vks'; +import { VerificationKey, VerificationKeyData } from '@aztec/stdlib/vks'; import { mock } from 'jest-mock-extended'; @@ -87,7 +82,7 @@ describe('Private Kernel Sequencer', () => { publicInputs.end.noteHashes = noteHashes; return { publicInputs, - verificationKey: VerificationKeyAsFields.makeEmpty(CLIENT_IVC_VERIFICATION_KEY_LENGTH_IN_FIELDS), + verificationKey: VerificationKeyData.empty(), outputWitness: new Map(), bytecode: Buffer.from([]), }; @@ -104,7 +99,7 @@ describe('Private Kernel Sequencer', () => { return { publicInputs, outputWitness: new Map(), - verificationKey: VerificationKeyAsFields.makeEmpty(CLIENT_IVC_VERIFICATION_KEY_LENGTH_IN_FIELDS), + verificationKey: VerificationKeyData.empty(), bytecode: Buffer.from([]), }; }; diff --git a/yarn-project/pxe/src/private_kernel/private_kernel_execution_prover.ts b/yarn-project/pxe/src/private_kernel/private_kernel_execution_prover.ts index c10b3506075c..d9f01cb47c1b 100644 --- a/yarn-project/pxe/src/private_kernel/private_kernel_execution_prover.ts +++ b/yarn-project/pxe/src/private_kernel/private_kernel_execution_prover.ts @@ -1,4 +1,4 @@ -import { CLIENT_IVC_VERIFICATION_KEY_LENGTH_IN_FIELDS, VK_TREE_HEIGHT } from '@aztec/constants'; +import { VK_TREE_HEIGHT } from '@aztec/constants'; import { vkAsFieldsMegaHonk } from '@aztec/foundation/crypto'; import { Fr } from '@aztec/foundation/fields'; import { createLogger } from '@aztec/foundation/log'; @@ -33,14 +33,14 @@ import { collectNoteHashNullifierCounterMap, getFinalMinRevertibleSideEffectCounter, } from '@aztec/stdlib/tx'; -import { VerificationKeyAsFields } from '@aztec/stdlib/vks'; +import { VerificationKeyAsFields, VerificationKeyData } from '@aztec/stdlib/vks'; import { PrivateKernelResetPrivateInputsBuilder } from './hints/build_private_kernel_reset_private_inputs.js'; import type { PrivateKernelOracle } from './private_kernel_oracle.js'; const NULL_SIMULATE_OUTPUT: PrivateKernelSimulateOutput = { publicInputs: PrivateKernelCircuitPublicInputs.empty(), - verificationKey: VerificationKeyAsFields.makeEmpty(CLIENT_IVC_VERIFICATION_KEY_LENGTH_IN_FIELDS), + verificationKey: VerificationKeyData.empty(), outputWitness: new Map(), bytecode: Buffer.from([]), }; @@ -123,6 +123,7 @@ export class PrivateKernelExecutionProver { functionName: 'private_kernel_reset', bytecode: output.bytecode, witness: output.outputWitness, + vk: output.verificationKey.keyAsBytes, }); resetBuilder = new PrivateKernelResetPrivateInputsBuilder( output, @@ -146,6 +147,7 @@ export class PrivateKernelExecutionProver { functionName: functionName!, bytecode: currentExecution.acir, witness: currentExecution.partialWitness, + vk: currentExecution.vk, }); const privateCallData = await this.createPrivateCallData(currentExecution); @@ -173,9 +175,12 @@ export class PrivateKernelExecutionProver { functionName: 'private_kernel_init', bytecode: output.bytecode, witness: output.outputWitness, + vk: output.verificationKey.keyAsBytes, }); } else { - const previousVkMembershipWitness = await this.oracle.getVkMembershipWitness(output.verificationKey); + const previousVkMembershipWitness = await this.oracle.getVkMembershipWitness( + output.verificationKey.keyAsFields, + ); const previousKernelData = new PrivateKernelData( output.publicInputs, output.verificationKey, @@ -194,6 +199,7 @@ export class PrivateKernelExecutionProver { functionName: 'private_kernel_inner', bytecode: output.bytecode, witness: output.outputWitness, + vk: output.verificationKey.keyAsBytes, }); } firstIteration = false; @@ -216,6 +222,7 @@ export class PrivateKernelExecutionProver { functionName: 'private_kernel_reset', bytecode: output.bytecode, witness: output.outputWitness, + vk: output.verificationKey.keyAsBytes, }); resetBuilder = new PrivateKernelResetPrivateInputsBuilder( @@ -233,7 +240,7 @@ export class PrivateKernelExecutionProver { output.publicInputs.feePayer = new AztecAddress(Fr.MAX_FIELD_VALUE); } // Private tail. - const previousVkMembershipWitness = await this.oracle.getVkMembershipWitness(output.verificationKey); + const previousVkMembershipWitness = await this.oracle.getVkMembershipWitness(output.verificationKey.keyAsFields); const previousKernelData = new PrivateKernelData( output.publicInputs, output.verificationKey, @@ -257,6 +264,7 @@ export class PrivateKernelExecutionProver { functionName: 'private_kernel_tail', bytecode: tailOutput.bytecode, witness: tailOutput.outputWitness, + vk: tailOutput.verificationKey.keyAsBytes, }); if (profileMode == 'gates' || profileMode == 'full') { @@ -289,7 +297,7 @@ export class PrivateKernelExecutionProver { publicInputs: tailOutput.publicInputs, executionSteps, clientIvcProof, - verificationKey: tailOutput.verificationKey, + vk: tailOutput.verificationKey.keyAsBytes, }; } diff --git a/yarn-project/scripts/run_test.sh b/yarn-project/scripts/run_test.sh index 189a840a65c6..c854cf205645 100755 --- a/yarn-project/scripts/run_test.sh +++ b/yarn-project/scripts/run_test.sh @@ -6,6 +6,7 @@ source $(git rev-parse --show-toplevel)/ci3/source test=$1 +shift 1 dir=${test%%/*} # Default to 2 CPUs and 4GB of memory when running with ISOLATE=1. @@ -30,11 +31,11 @@ if [ "${ISOLATE:-0}" -eq 1 ]; then -e NODE_OPTIONS="--no-warnings --experimental-vm-modules --loader @swc-node/register" \ -e LOG_LEVEL \ aztecprotocol/build:3.0 \ - node ../node_modules/.bin/jest --forceExit --runInBand $test & + node ../node_modules/.bin/jest --forceExit --runInBand $test $@ & wait $! else export NODE_OPTIONS="--no-warnings --experimental-vm-modules --loader @swc-node/register" export LOG_LEVEL=${LOG_LEVEL:-info} cd ../$dir - node ../node_modules/.bin/jest --forceExit --runInBand $test + node ../node_modules/.bin/jest --forceExit --runInBand $test $@ fi diff --git a/yarn-project/stdlib/package.json b/yarn-project/stdlib/package.json index cf9ac5940678..3d5d59cf19e5 100644 --- a/yarn-project/stdlib/package.json +++ b/yarn-project/stdlib/package.json @@ -71,6 +71,7 @@ "@aztec/constants": "workspace:^", "@aztec/ethereum": "workspace:^", "@aztec/foundation": "workspace:^", + "@aztec/noir-noirc_abi": "portal:../../noir/packages/noirc_abi", "@google-cloud/storage": "^7.15.0", "lodash.chunk": "^4.2.0", "lodash.isequal": "^4.5.0", diff --git a/yarn-project/stdlib/src/kernel/private_kernel_data.ts b/yarn-project/stdlib/src/kernel/private_kernel_data.ts index d8196668e5ad..a0a1d9d1d7ad 100644 --- a/yarn-project/stdlib/src/kernel/private_kernel_data.ts +++ b/yarn-project/stdlib/src/kernel/private_kernel_data.ts @@ -1,10 +1,10 @@ -import { CLIENT_IVC_VERIFICATION_KEY_LENGTH_IN_FIELDS, VK_TREE_HEIGHT } from '@aztec/constants'; +import { VK_TREE_HEIGHT } from '@aztec/constants'; import { makeTuple } from '@aztec/foundation/array'; import { Fr } from '@aztec/foundation/fields'; import { BufferReader, type Tuple, serializeToBuffer } from '@aztec/foundation/serialize'; import type { UInt32 } from '../types/shared.js'; -import { VerificationKeyAsFields } from '../vks/verification_key.js'; +import { VerificationKeyData } from '../vks/verification_key.js'; import { PrivateKernelCircuitPublicInputs } from './private_kernel_circuit_public_inputs.js'; /** @@ -21,7 +21,7 @@ export class PrivateKernelData { /** * Verification key of the previous kernel. */ - public vk: VerificationKeyAsFields, + public verificationKey: VerificationKeyData, /** * Index of the previous kernel's vk in a tree of vks. */ @@ -37,14 +37,14 @@ export class PrivateKernelData { * @returns The buffer. */ toBuffer() { - return serializeToBuffer(this.publicInputs, this.vk, this.vkIndex, this.vkPath); + return serializeToBuffer(this.publicInputs, this.verificationKey, this.vkIndex, this.vkPath); } static fromBuffer(buffer: Buffer | BufferReader): PrivateKernelData { const reader = BufferReader.asReader(buffer); return new this( reader.readObject(PrivateKernelCircuitPublicInputs), - reader.readObject(VerificationKeyAsFields), + reader.readObject(VerificationKeyData), reader.readNumber(), reader.readArray(VK_TREE_HEIGHT, Fr), ); @@ -53,7 +53,7 @@ export class PrivateKernelData { static empty(): PrivateKernelData { return new PrivateKernelData( PrivateKernelCircuitPublicInputs.empty(), - VerificationKeyAsFields.makeFake(CLIENT_IVC_VERIFICATION_KEY_LENGTH_IN_FIELDS), + VerificationKeyData.empty(), 0, makeTuple(VK_TREE_HEIGHT, Fr.zero), ); diff --git a/yarn-project/stdlib/src/kernel/private_kernel_prover_output.ts b/yarn-project/stdlib/src/kernel/private_kernel_prover_output.ts index ddbc78afdfa3..f82290246308 100644 --- a/yarn-project/stdlib/src/kernel/private_kernel_prover_output.ts +++ b/yarn-project/stdlib/src/kernel/private_kernel_prover_output.ts @@ -1,10 +1,11 @@ import { bufferSchema, mapSchema } from '@aztec/foundation/schemas'; import type { WitnessMap } from '@aztec/noir-acvm_js'; +import { serializeWitness } from '@aztec/noir-noirc_abi'; +import { Encoder } from 'msgpackr'; import { z } from 'zod'; import type { ClientIvcProof } from '../proofs/client_ivc_proof.js'; -import type { VerificationKeyAsFields } from '../vks/verification_key.js'; import type { PrivateKernelCircuitPublicInputs } from './private_kernel_circuit_public_inputs.js'; import type { PrivateKernelTailCircuitPublicInputs } from './private_kernel_tail_circuit_public_inputs.js'; @@ -13,6 +14,7 @@ export const PrivateExecutionStepSchema = z.object({ gateCount: z.number().optional(), bytecode: bufferSchema, witness: mapSchema(z.number(), z.string()), + vk: bufferSchema, }); /** @@ -23,6 +25,8 @@ export interface PrivateExecutionStep { gateCount?: number; bytecode: Buffer; witness: WitnessMap; + /* TODO(https://github.com/AztecProtocol/barretenberg/issues/1328) this should get its own proper class. */ + vk: Buffer; } /** Represents the output of proven PrivateKernelSimulateOutput.*/ @@ -34,7 +38,7 @@ export interface PrivateKernelExecutionProofOutput< /** The private IVC proof optimized for user devices. It will be consumed by an Aztec prover, * which recursively verifies it through the "tube" circuit.*/ clientIvcProof: ClientIvcProof; - verificationKey: VerificationKeyAsFields; + vk: Buffer; /** * The trace the clientIvcProof corresponds to. * A trace of app circuits interleaved with private kernel circuits. @@ -42,3 +46,15 @@ export interface PrivateKernelExecutionProofOutput< */ executionSteps: PrivateExecutionStep[]; } + +export function serializePrivateExecutionSteps(steps: PrivateExecutionStep[]) { + const stepToStruct = (step: PrivateExecutionStep) => { + return { + bytecode: step.bytecode, + witness: serializeWitness(step.witness), + vk: step.vk, + functionName: step.functionName, + }; + }; + return new Encoder({ useRecords: false }).pack(steps.map(stepToStruct)); +} diff --git a/yarn-project/stdlib/src/kernel/private_kernel_simulated_output.ts b/yarn-project/stdlib/src/kernel/private_kernel_simulated_output.ts index cf1cf02358b9..4628c6e0e0fd 100644 --- a/yarn-project/stdlib/src/kernel/private_kernel_simulated_output.ts +++ b/yarn-project/stdlib/src/kernel/private_kernel_simulated_output.ts @@ -1,6 +1,6 @@ import type { WitnessMap } from '@aztec/noir-acvm_js'; -import type { VerificationKeyAsFields } from '../vks/verification_key.js'; +import type { VerificationKeyData } from '../vks/verification_key.js'; import type { PrivateKernelCircuitPublicInputs } from './private_kernel_circuit_public_inputs.js'; import type { PrivateKernelTailCircuitPublicInputs } from './private_kernel_tail_circuit_public_inputs.js'; @@ -14,7 +14,7 @@ export interface PrivateKernelSimulateOutput< /** The public inputs required for the proof generation process. */ publicInputs: PublicInputsType; outputWitness: WitnessMap; - verificationKey: VerificationKeyAsFields; + verificationKey: VerificationKeyData; bytecode: Buffer; } @@ -22,5 +22,5 @@ export interface PrivateKernelSimulateOutput< * Represents the output of the circuit simulation process for init and inner private kernel circuit. */ export type AppCircuitSimulateOutput = { - verificationKey: VerificationKeyAsFields; + verificationKey: VerificationKeyData; }; diff --git a/yarn-project/stdlib/src/tx/profiled_tx.ts b/yarn-project/stdlib/src/tx/profiled_tx.ts index 8e6a4a86f56b..8d2ffe7b0ee7 100644 --- a/yarn-project/stdlib/src/tx/profiled_tx.ts +++ b/yarn-project/stdlib/src/tx/profiled_tx.ts @@ -21,6 +21,7 @@ export class TxProfileResult { functionName: 'random', bytecode: Buffer.from('random'), witness: new Map([[1, 'random']]), + vk: Buffer.from('random'), }, ]); } diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index d90b7c4226be..12e9dbe8807e 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -328,7 +328,6 @@ __metadata: "@aztec/telemetry-client": "workspace:^" "@aztec/world-state": "workspace:^" "@jest/globals": "npm:^29.5.0" - "@msgpack/msgpack": "npm:^3.0.0-beta2" "@types/jest": "npm:^29.5.0" "@types/node": "npm:^18.7.23" "@types/source-map-support": "npm:^0.5.10" @@ -468,11 +467,10 @@ __metadata: "@aztec/foundation": "workspace:^" "@aztec/kv-store": "workspace:^" "@aztec/noir-contracts.js": "workspace:^" - "@aztec/noir-noirc_abi": "workspace:^" + "@aztec/noir-noirc_abi": "portal:../../noir/packages/noirc_abi" "@aztec/pxe": "workspace:^" "@aztec/stdlib": "workspace:^" "@jest/globals": "npm:^29.5.0" - "@msgpack/msgpack": "npm:^3.0.0-beta2" "@types/jest": "npm:^29.5.0" "@types/node": "npm:^18.7.23" "@types/source-map-support": "npm:^0.5.10" @@ -588,7 +586,7 @@ __metadata: "@aztec/l1-artifacts": "workspace:^" "@aztec/merkle-tree": "workspace:^" "@aztec/noir-contracts.js": "workspace:^" - "@aztec/noir-noirc_abi": "workspace:^" + "@aztec/noir-noirc_abi": "portal:../../noir/packages/noirc_abi" "@aztec/noir-protocol-circuits-types": "workspace:^" "@aztec/p2p": "workspace:^" "@aztec/protocol-contracts": "workspace:^" @@ -603,7 +601,6 @@ __metadata: "@aztec/world-state": "workspace:^" "@iarna/toml": "npm:^2.2.5" "@jest/globals": "npm:^29.5.0" - "@msgpack/msgpack": "npm:^3.0.0-beta2" "@noble/curves": "npm:^1.0.0" "@swc/core": "npm:^1.4.11" "@swc/jest": "npm:^0.2.36" @@ -632,6 +629,7 @@ __metadata: lodash.compact: "npm:^3.0.1" lodash.every: "npm:^4.6.0" lodash.omit: "npm:^4.5.0" + msgpackr: "npm:^1.11.2" process: "npm:^0.11.10" stream-browserify: "npm:^3.0.0" string-argv: "npm:^0.3.2" @@ -788,7 +786,6 @@ __metadata: "@aztec/telemetry-client": "workspace:^" "@aztec/world-state": "workspace:^" "@jest/globals": "npm:^29.5.0" - "@msgpack/msgpack": "npm:^3.0.0-beta2" "@playwright/test": "npm:1.49.0" "@types/jest": "npm:^29.5.0" "@types/node": "npm:^22.8.1" @@ -800,6 +797,7 @@ __metadata: html-webpack-plugin: "npm:^5.6.0" jest: "npm:^29.5.0" jest-mock-extended: "npm:^4.0.0-beta1" + msgpackr: "npm:^1.11.2" pako: "npm:^2.1.0" playwright: "npm:1.49.0" puppeteer: "npm:^22.4.1" @@ -1227,7 +1225,6 @@ __metadata: "@aztec/simulator": "workspace:^" "@aztec/stdlib": "workspace:^" "@jest/globals": "npm:^29.5.0" - "@msgpack/msgpack": "npm:^3.0.0-beta2" "@types/jest": "npm:^29.5.0" "@types/lodash.omit": "npm:^4.5.7" "@types/lodash.times": "npm:^4.3.9" @@ -1359,6 +1356,7 @@ __metadata: "@aztec/constants": "workspace:^" "@aztec/ethereum": "workspace:^" "@aztec/foundation": "workspace:^" + "@aztec/noir-noirc_abi": "portal:../../noir/packages/noirc_abi" "@google-cloud/storage": "npm:^7.15.0" "@jest/expect": "npm:^29.5.0" "@jest/globals": "npm:^29.5.0" @@ -3706,13 +3704,6 @@ __metadata: languageName: node linkType: hard -"@msgpack/msgpack@npm:^3.0.0-beta2": - version: 3.0.0-beta2 - resolution: "@msgpack/msgpack@npm:3.0.0-beta2" - checksum: 10/d02f9221aa152cbd2977d1f56dc591baa2a37420a694cbc7e54ff0724f56ac0523e94de010e56bb845d67a1f2226c1761064b5777e63e9fc26884f4144d391a7 - languageName: node - linkType: hard - "@msgpackr-extract/msgpackr-extract-darwin-arm64@npm:3.0.2": version: 3.0.2 resolution: "@msgpackr-extract/msgpackr-extract-darwin-arm64@npm:3.0.2"