feat: merge-train/barretenberg

barretenberg/acir_tests/browser-test-app/src/index.ts

@@ -87,7 +87,10 @@ function installClientIvcGlobal() {
       threads,
       logger: console.log,
     });
-    const [proof, verificationKey] = await backend.prove(witnessBufs, vkBufs);
+    const [_, proof, verificationKey] = await backend.prove(
+      witnessBufs,
+      vkBufs
+    );
     await backend.destroy();
     return { proof, verificationKey };
   }

barretenberg/cpp/scripts/ci_benchmark_ivc_flows.sh

@@ -25,18 +25,18 @@ function verify_ivc_flow {
   # TODO(AD): Checking which one would be good, but there isn't too much that can go wrong here.
   set +e
   echo_stderr "Private verify."
-  "./$native_build_dir/bin/bb" verify --scheme client_ivc -p "$proof" -k ../../yarn-project/bb-prover/artifacts/private-civc-vk 1>&2
+  "./$native_build_dir/bin/bb" verify --scheme client_ivc -p "$proof" -k ../../noir-projects/noir-protocol-circuits/target/keys/hiding_kernel_to_rollup.ivc.vk 1>&2
   local private_result=$?
   echo_stderr "Private verify: $private_result."
-  "./$native_build_dir/bin/bb" verify --scheme client_ivc -p "$proof" -k ../../yarn-project/bb-prover/artifacts/public-civc-vk 1>&2
+  "./$native_build_dir/bin/bb" verify --scheme client_ivc -p "$proof" -k ../../noir-projects/noir-protocol-circuits/target/keys/hiding_kernel_to_public.ivc.vk 1>&2
   local public_result=$?
   echo_stderr "Public verify: $public_result."
   if [[ $private_result -eq $public_result ]]; then
     echo_stderr "Verification failed for $flow. Both keys returned $private_result - only one should."
     exit 1
   fi
   if [[ $private_result -ne 0 ]] && [[ $public_result -ne 0 ]]; then
-    echo_stderr "Verification failed for $flow. Did not verify with precalculated verification key - we may need to revisit how it is generated in yarn-project/bb-prover."
+    echo_stderr "Verification failed for $flow. Did not verify with precalculated verification key - we may need to revisit how it is generated in noir-projects/noir-protocol-circuits."
     exit 1
   fi
 }

barretenberg/cpp/scripts/test_civc_standalone_vks_havent_changed.sh

@@ -13,7 +13,7 @@ cd ..
 # - Generate a hash for versioning: sha256sum bb-civc-inputs.tar.gz
 # - Upload the compressed results: aws s3 cp bb-civc-inputs.tar.gz s3://aztec-ci-artifacts/protocol/bb-civc-inputs-[hash(0:8)].tar.gz
 # Note: In case of the "Test suite failed to run ... Unexpected token 'with' " error, need to run: docker pull aztecprotocol/build:3.0
-pinned_short_hash="16b530e4"
+pinned_short_hash="d06b78a2"
 pinned_civc_inputs_url="https://aztec-ci-artifacts.s3.us-east-2.amazonaws.com/protocol/bb-civc-inputs-${pinned_short_hash}.tar.gz"
 
 function compress_and_upload {

barretenberg/cpp/src/barretenberg/api/api_client_ivc.cpp

@@ -27,13 +27,16 @@ namespace { // anonymous namespace
  *
  * @param bytecode_path
  * @param witness_path
+ * @param use_structured_trace Whether to utilize structured trace when computing VK for circuit
  */
-void write_standalone_vk(const std::filesystem::path& bytecode_path, const std::filesystem::path& output_path)
+void write_standalone_vk(std::vector<uint8_t> bytecode,
+                         const std::filesystem::path& output_path,
+                         bool use_structured_trace = true)
 {
-    auto bytecode = get_bytecode(bytecode_path);
+    auto trace_settings = use_structured_trace ? TraceSettings{ AZTEC_TRACE_STRUCTURE } : TraceSettings{};
     auto response = bbapi::ClientIvcComputeStandaloneVk{
         .circuit = { .name = "standalone_circuit", .bytecode = std::move(bytecode) }
-    }.execute();
+    }.execute({ .trace_settings = trace_settings });
 
     bool is_stdout = output_path == "-";
     if (is_stdout) {
@@ -84,17 +87,18 @@ void ClientIVCAPI::prove(const Flags& flags,
     const bool output_to_stdout = output_dir == "-";
 
     const auto write_proof = [&]() {
-        const auto buf = to_buffer(proof);
+        const auto buf = to_buffer(proof.to_field_elements());
         if (output_to_stdout) {
             vinfo("writing ClientIVC proof to stdout");
             write_bytes_to_stdout(buf);
         } else {
             vinfo("writing ClientIVC proof in directory ", output_dir);
-            proof.to_file_msgpack(output_dir / "proof");
+            write_file(output_dir / "proof", buf);
         }
     };
 
     write_proof();
+
     if (flags.write_vk) {
         vinfo("writing ClientIVC vk in directory ", output_dir);
         // write CIVC vk using the bytecode of the hiding circuit (the last step of the execution)
@@ -108,8 +112,11 @@ bool ClientIVCAPI::verify([[maybe_unused]] const Flags& flags,
                           const std::filesystem::path& vk_path)
 {
     BB_BENCH_NAME("ClientIVCAPI::verify");
-    auto proof = ClientIVC::Proof::from_file_msgpack(proof_path);
+    auto proof_fields = many_from_buffer<fr>(read_file(proof_path));
+    auto proof = ClientIVC::Proof::from_field_elements(proof_fields);
+
     auto vk_buffer = read_file(vk_path);
+
     auto response = bbapi::ClientIvcVerify{ .proof = std::move(proof), .vk = std::move(vk_buffer) }.execute();
     return response.valid;
 }
@@ -177,10 +184,14 @@ void ClientIVCAPI::write_vk(const Flags& flags,
                             const std::filesystem::path& output_path)
 {
     BB_BENCH_NAME("ClientIVCAPI::write_vk");
+    auto bytecode = get_bytecode(bytecode_path);
     if (flags.verifier_type == "ivc") {
-        write_civc_vk(get_bytecode(bytecode_path), output_path);
+        write_civc_vk(bytecode, output_path);
     } else if (flags.verifier_type == "standalone") {
-        write_standalone_vk(bytecode_path, output_path);
+        write_standalone_vk(bytecode, output_path);
+    } else if (flags.verifier_type == "standalone_hiding") {
+        // write the VK for the hiding kernel which DOES NOT utilize a structured trace
+        write_standalone_vk(bytecode, output_path, false);
     } else {
         const std::string msg = std::string("Can't write vk for verifier type ") + flags.verifier_type;
         throw_or_abort(msg);

barretenberg/cpp/src/barretenberg/bb/cli.cpp

@@ -241,9 +241,11 @@ int parse_and_run_cli_command(int argc, char* argv[])
                          "recursive verifier) or is it for an ivc verifier? `standalone` produces a verification key "
                          "is sufficient for verifying proofs about a single circuit (including the non-encsapsulated "
                          "use case where an IVC scheme is manually constructed via recursive UltraHonk proof "
-                         "verification). `ivc` produces a verification key for verifying the stack of run though a "
-                         "dedicated ivc verifier class (currently the only option is the ClientIVC class) ")
-            ->check(CLI::IsMember({ "standalone", "ivc" }).name("is_member"));
+                         "verification). `standalone_hiding` is similar to `standalone` but is used for the last step "
+                         "where the structured trace is not utilized. `ivc` produces a verification key for verifying "
+                         "the stack of run though a dedicated ivc verifier class (currently the only option is the "
+                         "ClientIVC class)")
+            ->check(CLI::IsMember({ "standalone", "standalone_hiding", "ivc" }).name("is_member"));
     };
 
     const auto add_verbose_flag = [&](CLI::App* subcommand) {

barretenberg/cpp/src/barretenberg/circuit_checker/translator_circuit_checker.cpp

@@ -128,13 +128,36 @@ bool TranslatorCircuitChecker::check(const Builder& circuit)
             return check_accumulator_transfer(previous_accumulator, gate + 1);
         };
 
+    auto check_random_op_code = [&](const Fr op_code, size_t gate) {
+        if (gate % 2 == 0) {
+            if (op_code == Fr(0) || op_code == Fr(3) || op_code == Fr(4) || op_code == Fr(8)) {
+                return report_fail("Opcode should be random value at even gate = ", gate);
+            }
+        } else {
+            if (op_code == Fr(0)) {
+                return report_fail("Opcode should be 0 at odd gate = ", gate);
+            }
+        }
+        return true;
+    };
+
     // TODO(https: // github.com/AztecProtocol/barretenberg/issues/1367): Report all failures more explicitly and
     // consider making use of relations.
-
+    auto in_random_range = [&](size_t i) {
+        return (i >= 2 * Builder::NUM_NO_OPS_START && i < RESULT_ROW) ||
+               (i >= circuit.num_gates - (circuit.avm_mode ? 0 : 2 * Builder::NUM_RANDOM_OPS_END) &&
+                i < circuit.num_gates);
+    };
     for (size_t i = 2; i < circuit.num_gates - 1; i += 2) {
 
-        // Get the values of P.x
+        // Ensure random op is present  in expected ranges
         Fr op_code = circuit.get_variable(op_wire[i]);
+        if (in_random_range(i)) {
+            check_random_op_code(op_code, i);
+            Fr op_code_next = circuit.get_variable(op_wire[i + 1]);
+            check_random_op_code(op_code_next, i + 1);
+            continue;
+        }
 
         // Current accumulator (updated value)
         const std::vector current_accumulator_binary_limbs = {

barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.cpp

@@ -268,9 +268,9 @@ ClientIVC::perform_recursive_verification_and_databus_consistency_checks(
     pairing_points.aggregate(nested_pairing_points);
     if (is_hiding_kernel) {
         pairing_points.aggregate(decider_pairing_points);
-        // Placeholder for randomness at the end of the hiding circuit (to be handled in subsequent PR)
-        circuit.queue_ecc_no_op();
-        circuit.queue_ecc_no_op();
+        // Add randomness at the end of the hiding kernel (whose ecc ops fall right at the end of the op queue table) to
+        // ensure the CIVC proof doesn't leak information about the actual content of the op queue
+        hide_op_queue_content_in_hiding(circuit);
     }
 
     return { output_verifier_accumulator, pairing_points, merged_table_commitments };
@@ -317,9 +317,9 @@ void ClientIVC::complete_kernel_circuit_logic(ClientCircuit& circuit)
                      0U,
                      "tail kernel ecc ops table should be empty at this point");
         circuit.queue_ecc_no_op();
-        // Placeholder for randomness at the beginning of tail circuit
-        circuit.queue_ecc_no_op();
-        circuit.queue_ecc_no_op();
+        // Add randomness at the begining of the tail kernel (whose ecc ops fall at the beginning of the op queue table)
+        // to ensure the CIVC proof doesn't leak information about the actual content of the op queue
+        hide_op_queue_content_in_tail(circuit);
     }
     circuit.queue_ecc_eq();
 
@@ -524,7 +524,8 @@ void ClientIVC::accumulate(ClientCircuit& circuit, const std::shared_ptr<MegaVer
 }
 
 /**
- * @brief Add a random operation to the op queue to hide its content in Translator computation.
+ * @brief Add a *real* operation but with random data to the op queue to avoid information leak in Translator
+ * computation.
  *
  * @details Translator circuit builder computes the evaluation at some random challenge x of a batched polynomial
  * derived from processing the ultra_op version of op_queue. This result (referred to as accumulated_result in
@@ -542,11 +543,74 @@ void ClientIVC::hide_op_queue_accumulation_result(ClientCircuit& circuit)
     circuit.queue_ecc_eq();
 }
 
+/**
+ * @brief Adds three random ops to the tail kernel.
+ *
+ * @note The explanation below does not serve as a proof of zero-knowledge but rather as intuition for why the number
+ * of random ops and their position in the op queue.
+ *
+ * @details The ClientIVC proof is sent to the rollup and so it has to be zero-knowledge. In turn, this implies that
+ * commitments and evaluations to the op queue, when regarded as 4 polynomials in UltraOp format (op, x_lo_y_hi,
+ * x_hi_z_1, y_lo_z_2), should not leak information about the actual content of the op queue with provenance from
+ * circuit operations that have been accumulated in CIVC. Since the op queue is used across several provers,
+ * randomising these polynomials has to be handled in a special way. Normally, to hide a witness we'd add random
+ * coefficients at proving time when populating ProverPolynomials. However, due to the consistency checks present
+ * throughout CIVC, to ensure all components use the same op queue data (Merge and Translator on the entire op queue
+ * table and Merge and Oink on each subtable), randomness has to be added in a common place, this place naturally
+ * being ClientIVC. ECCVM is not affected by the concerns above, randomness being added to wires at proving time as per
+ * usual, because the consistency of ECCVMOps processing and UltraOps processing between Translator and ECCVM is
+ * achieved via the translation evaluation check and avoiding an information leak there is ensured by
+ * `ClientIVC::hide_op_queue_accumulation_result()` and SmallSubgroupIPA in ECCVM.
+ *
+ * We need each op queue polynomial to have 9 random coefficients (so the op queue needs to contain 5 random ops, every
+ * UltraOp adding two coefficients to each of the 4 polynomials).
+ *
+ * For the last subtable of ecc ops belonging to the hiding kernel, merged via appended to the full op queue, its data
+ * appears as the ecc_op_wires in the MegaZK proof, wires that are not going to be shifted, so the proof contains,
+ * for each wire, its commitment and evaluation to the Sumcheck challenge. As at least 3 random coefficients are
+ * needed in each op queue polynomial, we add 2 random ops to the hiding kernel.
+ *
+ * The op queue state previous to the append of the last subtable, is the `left_table` in the merge protocol, so for
+ * the degree check, we construct its inverse polynomial `left_table_inverse`. The MergeProof will contain the
+ * commitment to the `left_table_inverse` plus its evaluation at Merge protocol challenge κ. Also for the degree check,
+ * prover needs to send the evaluation of the `left_table` at κ⁻¹. We need to ensure random coefficients are added to
+ * one of the kernels as not to affect Apps verification keys so the best choice is to add them to the beginning of the
+ * tail kernel as to not complicate Translator relations. The above advises that another 4 random coefficients are
+ * needed in the `left_table` (so, 2 random ops).
+ *
+ * Finally, the 4 polynomials representing the full ecc op queue table are committed to (in fact, in both Merge
+ * protocol and Translator but they are commitments to the same data). `x_lo_y_hi`, `x_hi_z_1` and `x_lo_z_2` are
+ * shifted polynomials in Translator so the Translator proof will contain their evaluation and evaluation of their
+ * shifts at the Sumcheck challenge. On top of that, the Shplonk proof sent in the last iteration of Merge also
+ * ascertains the opening of partially_evaluated_difference = left_table + κ^{shift -1 } * right_table - merged_table
+ * at κ is 0, so a batched quotient commitment is sent in the Merge proof. In total, for each op queue polynomial (or
+ * parts of its data), there are 4 commitments and 5 evaluations across the CIVC proof so the sweet spot is 5 random
+ * ops.
+ */
+void ClientIVC::hide_op_queue_content_in_tail(ClientCircuit& circuit)
+{
+    circuit.queue_ecc_random_op();
+    circuit.queue_ecc_random_op();
+    circuit.queue_ecc_random_op();
+}
+
+/**
+ * @brief Adds two random ops to the hiding kernel.
+ *
+ * @details For the last subtable of ecc ops belonging to the hiding kernel, merged via appended to the full op
+ * queue, its data appears as the ecc_op_wires in the MegaZK proof, wires that are not going to be shifted, so the proof
+ * containts, for each wire, its commitment and evaluation to the Sumcheck challenge. As at least 3 random coefficients
+ * are needed in each op queue polynomial, we add 2 random ops. More details in `hide_op_queue_content_in_tail`.
+ */
+void ClientIVC::hide_op_queue_content_in_hiding(ClientCircuit& circuit)
+{
+    circuit.queue_ecc_random_op();
+    circuit.queue_ecc_random_op();
+}
+
 /**
  * @brief Construct a zero-knowledge proof for the hiding circuit, which recursively verifies the last folding,
  * merge and decider proof.
- *
- * @return HonkProof - a ZK Mega proof
  */
 HonkProof ClientIVC::construct_mega_proof_for_hiding_kernel(ClientCircuit& circuit)
 {
@@ -673,6 +737,43 @@ std::vector<ClientIVC::FF> ClientIVC::Proof::to_field_elements() const
     return proof;
 };
 
+ClientIVC::Proof ClientIVC::Proof::from_field_elements(const std::vector<ClientIVC::FF>& fields)
+{
+    HonkProof mega_proof;
+    GoblinProof goblin_proof;
+
+    size_t custom_public_inputs_size = fields.size() - ClientIVC::Proof::PROOF_LENGTH();
+
+    // Mega proof
+    auto start_idx = fields.begin();
+    auto end_idx = start_idx + static_cast<std::ptrdiff_t>(
+                                   MegaZKFlavor::PROOF_LENGTH_WITHOUT_PUB_INPUTS(MegaZKFlavor::VIRTUAL_LOG_N) +
+                                   bb::HidingKernelIO::PUBLIC_INPUTS_SIZE + custom_public_inputs_size);
+    mega_proof.insert(mega_proof.end(), start_idx, end_idx);
+
+    // Merge proof
+    start_idx = end_idx;
+    end_idx += static_cast<std::ptrdiff_t>(MERGE_PROOF_SIZE);
+    goblin_proof.merge_proof.insert(goblin_proof.merge_proof.end(), start_idx, end_idx);
+
+    // ECCVM pre-ipa proof
+    start_idx = end_idx;
+    end_idx += static_cast<std::ptrdiff_t>(ECCVMFlavor::PROOF_LENGTH_WITHOUT_PUB_INPUTS - IPA_PROOF_LENGTH);
+    goblin_proof.eccvm_proof.pre_ipa_proof.insert(goblin_proof.eccvm_proof.pre_ipa_proof.end(), start_idx, end_idx);
+
+    // ECCVM ipa proof
+    start_idx = end_idx;
+    end_idx += static_cast<std::ptrdiff_t>(IPA_PROOF_LENGTH);
+    goblin_proof.eccvm_proof.ipa_proof.insert(goblin_proof.eccvm_proof.ipa_proof.end(), start_idx, end_idx);
+
+    // Translator proof
+    start_idx = end_idx;
+    end_idx += static_cast<std::ptrdiff_t>(TranslatorFlavor::PROOF_LENGTH_WITHOUT_PUB_INPUTS);
+    goblin_proof.translator_proof.insert(goblin_proof.translator_proof.end(), start_idx, end_idx);
+
+    return { mega_proof, goblin_proof };
+};
+
 msgpack::sbuffer ClientIVC::Proof::to_msgpack_buffer() const
 {
     msgpack::sbuffer buffer;

barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.hpp

@@ -133,6 +133,8 @@ class ClientIVC {
          */
         std::vector<FF> to_field_elements() const;
 
+        static Proof from_field_elements(const std::vector<ClientIVC::FF>& fields);
+
         // TODO(https://github.com/AztecProtocol/barretenberg/issues/1299): The following msgpack methods are generic
         // and should leverage some kind of shared msgpack utility.
         msgpack::sbuffer to_msgpack_buffer() const;
@@ -323,6 +325,8 @@ class ClientIVC {
     Proof prove();
 
     static void hide_op_queue_accumulation_result(ClientCircuit& circuit);
+    static void hide_op_queue_content_in_tail(ClientCircuit& circuit);
+    static void hide_op_queue_content_in_hiding(ClientCircuit& circuit);
     HonkProof construct_mega_proof_for_hiding_kernel(ClientCircuit& circuit);
 
     static bool verify(const Proof& proof, const VerificationKey& vk);

barretenberg/cpp/src/barretenberg/dsl/acir_format/civc_recursion_constraints.cpp

@@ -113,7 +113,7 @@ create_civc_recursion_constraints(Builder& builder,
     // Recursively verify CIVC proof
     auto mega_vk = std::make_shared<VerificationKey>(builder, key_fields);
     auto mega_vk_and_hash = std::make_shared<RecursiveVKAndHash>(mega_vk, vk_hash);
-    ClientIVCRecursiveVerifier::StdlibProof stdlib_proof(proof_fields);
+    ClientIVCRecursiveVerifier::StdlibProof stdlib_proof(proof_fields, input.public_inputs.size());
 
     ClientIVCRecursiveVerifier verifier(&builder, mega_vk_and_hash);
     ClientIVCRecursiveVerifier::Output verification_output = verifier.verify(stdlib_proof);

barretenberg/cpp/src/barretenberg/goblin/goblin.cpp

@@ -43,7 +43,7 @@ void Goblin::prove_eccvm()
 void Goblin::prove_translator()
 {
     BB_BENCH_NAME("Goblin::prove_translator");
-    TranslatorBuilder translator_builder(translation_batching_challenge_v, evaluation_challenge_x, op_queue);
+    TranslatorBuilder translator_builder(translation_batching_challenge_v, evaluation_challenge_x, op_queue, avm_mode);
     auto translator_key = std::make_shared<TranslatorProvingKey>(translator_builder, commitment_key);
     TranslatorProver translator_prover(translator_key, transcript);
     goblin_proof.translator_proof = translator_prover.construct_proof();
@@ -124,4 +124,13 @@ bool Goblin::verify(const GoblinProof& proof,
            op_queue_consistency_verified;
 }
 
+void Goblin::ensure_well_formed_op_queue_for_avm(MegaBuilder& builder) const
+{
+    BB_ASSERT_EQ(avm_mode, true, "ensure_well_formed_op_queue should only be called for avm");
+    builder.queue_ecc_no_op();
+    builder.queue_ecc_random_op();
+    builder.queue_ecc_random_op();
+    builder.queue_ecc_random_op();
+}
+
 } // namespace bb