feat: Remove 4 byte metadata from bb-produced proof

barretenberg/acir_tests/bbjs-test/src/index.ts

@@ -81,6 +81,7 @@ async function verifyProof({ directory }: { directory: string }) {
   const publicInputs = JSON.parse(
     await fs.readFile(publicInputsAsFieldsPath(directory), "utf8")
   );
+  debug(`publicInputs: ${JSON.stringify(publicInputs)}`);
   const vkey = await fs.readFile(vkeyPath(directory));
 
   const verified = await verifier.verifyUltraHonkProof(
@@ -90,6 +91,7 @@ async function verifyProof({ directory }: { directory: string }) {
 
   await verifier.destroy();
   debug(`Proof verified: ${verified}`);
+  return verified;
 }
 
 // Prepare a minimal command line interface
@@ -107,6 +109,9 @@ program
 program
   .command("verify")
   .option("-d, --directory <path>", "directory")
-  .action((args) => verifyProof(args));
+  .action(async (args) => {
+    const result = await verifyProof(args);
+    process.exit(result ? 0 : 1);
+  });
 
 program.parse(process.argv);

barretenberg/acir_tests/flows/bb_prove_bbjs_verify.sh

@@ -29,20 +29,6 @@ $BIN write_vk \
   -b $artifact_dir/program.json \
   -o $output_dir
 
-# bb.js expects proof and public inputs to be separate files, so we need to split them
-# this will not be needed after #11024
-
-# Save public inputs as a separate file (first NUM_PUBLIC_INPUTS fields of proof_fields.json)
-PROOF_FIELDS_LENGTH=$(jq 'length' $output_dir/proof_fields.json)
-UH_PROOF_FIELDS_LENGTH=440
-NUM_PUBLIC_INPUTS=$((PROOF_FIELDS_LENGTH - UH_PROOF_FIELDS_LENGTH))
-jq ".[:$NUM_PUBLIC_INPUTS]" $output_dir/proof_fields.json > $output_dir/public_inputs_fields.json
-
-# Remove public inputs from the proof (first NUM_PUBLIC_INPUTS*32 bytes)
-# Also remove the first 4 bytes, which is the proof length in fields
-proof_hex=$(cat $output_dir/proof | xxd -p)
-echo -n ${proof_hex:$((NUM_PUBLIC_INPUTS * 64 + 8))} | xxd -r -p > $output_dir/proof
-
 # Verify the proof with bb.js classes
 node ../../bbjs-test verify \
   -d $output_dir

barretenberg/acir_tests/flows/bbjs_prove_bb_verify.sh

@@ -13,25 +13,14 @@ output_dir=$artifact_dir/bb-bbjs-tmp
 mkdir -p $output_dir
 
 # Cleanup on exit
-# trap "rm -rf $output_dir" EXIT
+trap "rm -rf $output_dir" EXIT
 
 # Writes the proof, public inputs ./target; this also writes the VK
 node ../../bbjs-test prove \
   -b $artifact_dir/program.json \
   -w $artifact_dir/witness.gz \
   -o $output_dir
 
-# Join the proof and public inputs to a single file
-# this will not be needed after #11024
-
-NUM_PUBLIC_INPUTS=$(cat $output_dir/public_inputs_fields.json | jq 'length')
-UH_PROOF_FIELDS_LENGTH=440
-PROOF_LENGTH_IN_FIELDS=$((UH_PROOF_FIELDS_LENGTH))
-PI_LENGTH_IN_FIELDS=$((NUM_PUBLIC_INPUTS))
-# First 4 bytes is PROOF_AND_PI_LENGTH_IN_FIELDS
-proof_header=$(printf "%08x" $PROOF_LENGTH_IN_FIELDS)
-pi_header=$(printf "%08x" $PI_LENGTH_IN_FIELDS)
-
 proof_bytes=$(cat $output_dir/proof | xxd -p)
 public_inputs=$(cat $output_dir/public_inputs_fields.json | jq -r '.[]')
 
@@ -41,8 +30,8 @@ for input in $public_inputs; do
 done
 
 # Combine proof header and the proof to a single file
-echo -n $proof_header$proof_bytes | xxd -r -p > $output_dir/proof
-echo -n $pi_header$public_inputs_bytes | xxd -r -p > $output_dir/public_inputs
+echo -n $proof_bytes | xxd -r -p > $output_dir/proof
+echo -n $public_inputs_bytes | xxd -r -p > $output_dir/public_inputs
 echo "$BIN verify \
   --scheme ultra_honk \
   -k $output_dir/vk \

barretenberg/acir_tests/flows/prove_then_verify_tube.sh

@@ -14,4 +14,5 @@ trap "rm -rf $outdir" EXIT
 # TODO(https://github.com/AztecProtocol/barretenberg/issues/1252): deprecate in favor of normal proving flow
 $BIN OLD_API write_arbitrary_valid_client_ivc_proof_and_vk_to_file -c $CRS_PATH ${VERBOSE:+-v} -o $outdir
 $BIN prove_tube -c $CRS_PATH ${VERBOSE:+-v} -k $outdir/vk -o $outdir
+# TODO(https://github.com/AztecProtocol/barretenberg/issues/1322): Just call verify.
 $BIN verify_tube -c $CRS_PATH ${VERBOSE:+-v} -o $outdir

barretenberg/acir_tests/sol-test/src/index.js

@@ -246,10 +246,6 @@ try {
     [numExtraPublicInputs, extraPublicInputs] = readPublicInputs(
       JSON.parse(proofAsFields.toString())
     );
-    if (testingHonk) {
-      // Honk proof from the CLI have field length as the first 4 bytes. This should go away in the future
-      proofStr = proofStr.substring(8);
-    }
   }
   // We need to do this because plonk doesn't define this path
   if (publicInputsAsFieldsPath) {

barretenberg/cpp/src/barretenberg/api/api_ultra_honk.cpp

@@ -98,7 +98,7 @@ PubInputsProofAndKey<VK> _prove(const bool compute_vk,
 }
 
 template <typename Flavor>
-bool _verify(const bool honk_recursion_2,
+bool _verify(const bool ipa_accumulation,
              const std::filesystem::path& public_inputs_path,
              const std::filesystem::path& proof_path,
              const std::filesystem::path& vk_path)
@@ -108,25 +108,25 @@ bool _verify(const bool honk_recursion_2,
 
     auto g2_data = get_bn254_g2_data(CRS_PATH);
     srs::init_crs_factory({}, g2_data);
-    auto public_inputs = from_buffer<std::vector<bb::fr>>(read_file(public_inputs_path));
-    auto proof = from_buffer<std::vector<bb::fr>>(read_file(proof_path));
-    // concatenate public inputs and proof
-    std::vector<fr> complete_proof = public_inputs;
-    complete_proof.insert(complete_proof.end(), proof.begin(), proof.end());
 
     auto vk = std::make_shared<VerificationKey>(from_buffer<VerificationKey>(read_file(vk_path)));
     vk->pcs_verification_key = std::make_shared<VerifierCommitmentKey<curve::BN254>>();
+    auto public_inputs = many_from_buffer<bb::fr>(read_file(public_inputs_path));
+    auto proof = many_from_buffer<bb::fr>(read_file(proof_path));
+    // concatenate public inputs and proof
+    std::vector<fr> complete_proof = public_inputs;
+    complete_proof.insert(complete_proof.end(), proof.begin(), proof.end());
 
     std::shared_ptr<VerifierCommitmentKey<curve::Grumpkin>> ipa_verification_key;
-    if (honk_recursion_2) {
+    if (ipa_accumulation) {
         init_grumpkin_crs(1 << CONST_ECCVM_LOG_N);
         ipa_verification_key = std::make_shared<VerifierCommitmentKey<curve::Grumpkin>>(1 << CONST_ECCVM_LOG_N);
     }
 
     Verifier verifier{ vk, ipa_verification_key };
 
     bool verified;
-    if (honk_recursion_2) {
+    if (ipa_accumulation) {
         const size_t HONK_PROOF_LENGTH = Flavor::PROOF_LENGTH_WITHOUT_PUB_INPUTS - IPA_PROOF_LENGTH;
         const size_t num_public_inputs = static_cast<size_t>(vk->num_public_inputs);
         // The extra calculation is for the IPA proof length.

barretenberg/cpp/src/barretenberg/api/prove_tube.cpp

@@ -73,8 +73,8 @@ void prove_tube(const std::string& output_path, const std::string& vk_path)
                            tube_proof.begin() + static_cast<std::ptrdiff_t>(num_inner_public_inputs)),
         HonkProof(tube_proof.begin() + static_cast<std::ptrdiff_t>(num_inner_public_inputs), tube_proof.end())
     };
-    write_file(tubePublicInputsPath, to_buffer<true>(public_inputs_and_proof.public_inputs));
-    write_file(tubeProofPath, to_buffer<true>(public_inputs_and_proof.proof));
+    write_file(tubePublicInputsPath, to_buffer(public_inputs_and_proof.public_inputs));
+    write_file(tubeProofPath, to_buffer(public_inputs_and_proof.proof));
 
     std::string tubePublicInputsAsFieldsPath = output_path + "/public_inputs_fields.json";
     std::string tubeProofAsFieldsPath = output_path + "/proof_fields.json";

barretenberg/cpp/src/barretenberg/api/write_prover_output.hpp

@@ -35,7 +35,7 @@ void write(const ProverOutput& prover_output,
         case ObjectToWrite::PUBLIC_INPUTS: {
             // TODO(https://github.com/AztecProtocol/barretenberg/issues/1312): Try to avoid include_size=true, which is
             // used for deserialization.
-            const auto buf = to_buffer</*include_size*/ true>(prover_output.public_inputs);
+            const auto buf = to_buffer(prover_output.public_inputs);
             if (output_to_stdout) {
                 write_bytes_to_stdout(buf);
             } else {
@@ -47,7 +47,7 @@ void write(const ProverOutput& prover_output,
         case ObjectToWrite::PROOF: {
             // TODO(https://github.com/AztecProtocol/barretenberg/issues/1312): Try to avoid include_size=true, which is
             // used for deserialization.
-            const auto buf = to_buffer</*include_size*/ true>(prover_output.proof);
+            const auto buf = to_buffer(prover_output.proof);
             if (output_to_stdout) {
                 write_bytes_to_stdout(buf);
             } else {

barretenberg/cpp/src/barretenberg/bb/main.cpp

@@ -760,8 +760,10 @@ int main(int argc, char* argv[])
 #endif
         // TUBE
         else if (prove_tube_command->parsed()) {
+            // TODO(https://github.com/AztecProtocol/barretenberg/issues/1201): Potentially remove this extra logic.
             prove_tube(prove_tube_output_path, vk_path);
         } else if (verify_tube_command->parsed()) {
+            // TODO(https://github.com/AztecProtocol/barretenberg/issues/1322): Remove verify_tube logic.
             auto tube_public_inputs_path = tube_proof_and_vk_path + "/public_inputs";
             auto tube_proof_path = tube_proof_and_vk_path + "/proof";
             auto tube_vk_path = tube_proof_and_vk_path + "/vk";

barretenberg/cpp/src/barretenberg/dsl/acir_proofs/c_bind.cpp

@@ -327,7 +327,7 @@ WASM_EXPORT void acir_prove_ultra_honk(uint8_t const* acir_vec,
     }();
 
     auto proof = prover.construct_proof();
-    *out = to_heap_buffer(to_buffer</*include_size=*/true>(proof));
+    *out = to_heap_buffer(to_buffer(proof));
 }
 
 WASM_EXPORT void acir_prove_ultra_keccak_honk(uint8_t const* acir_vec,
@@ -347,7 +347,7 @@ WASM_EXPORT void acir_prove_ultra_keccak_honk(uint8_t const* acir_vec,
         return UltraKeccakProver(builder);
     }();
     auto proof = prover.construct_proof();
-    *out = to_heap_buffer(to_buffer</*include_size=*/true>(proof));
+    *out = to_heap_buffer(to_buffer(proof));
 }
 
 WASM_EXPORT void acir_verify_ultra_honk(uint8_t const* proof_buf, uint8_t const* vk_buf, bool* result)
@@ -356,7 +356,7 @@ WASM_EXPORT void acir_verify_ultra_honk(uint8_t const* proof_buf, uint8_t const*
     using VerifierCommitmentKey = bb::VerifierCommitmentKey<curve::BN254>;
     using Verifier = UltraVerifier_<UltraFlavor>;
 
-    auto proof = from_buffer<std::vector<bb::fr>>(from_buffer<std::vector<uint8_t>>(proof_buf));
+    auto proof = many_from_buffer<bb::fr>(from_buffer<std::vector<uint8_t>>(proof_buf));
     auto verification_key = std::make_shared<VerificationKey>(from_buffer<VerificationKey>(vk_buf));
     verification_key->pcs_verification_key = std::make_shared<VerifierCommitmentKey>();
 
@@ -371,7 +371,7 @@ WASM_EXPORT void acir_verify_ultra_keccak_honk(uint8_t const* proof_buf, uint8_t
     using VerifierCommitmentKey = bb::VerifierCommitmentKey<curve::BN254>;
     using Verifier = UltraVerifier_<UltraKeccakFlavor>;
 
-    auto proof = from_buffer<std::vector<bb::fr>>(from_buffer<std::vector<uint8_t>>(proof_buf));
+    auto proof = many_from_buffer<bb::fr>(from_buffer<std::vector<uint8_t>>(proof_buf));
     auto verification_key = std::make_shared<VerificationKey>(from_buffer<VerificationKey>(vk_buf));
     verification_key->pcs_verification_key = std::make_shared<VerifierCommitmentKey>();
 
@@ -420,7 +420,7 @@ WASM_EXPORT void acir_honk_solidity_verifier(uint8_t const* proof_buf, uint8_t c
     using VerificationKey = UltraKeccakFlavor::VerificationKey;
     using VerifierCommitmentKey = bb::VerifierCommitmentKey<curve::BN254>;
 
-    auto proof = from_buffer<std::vector<bb::fr>>(from_buffer<std::vector<uint8_t>>(proof_buf));
+    auto proof = many_from_buffer<bb::fr>(from_buffer<std::vector<uint8_t>>(proof_buf));
     auto verification_key = from_buffer<VerificationKey>(vk_buf);
     verification_key.pcs_verification_key = std::make_shared<VerifierCommitmentKey>();
 
@@ -430,7 +430,7 @@ WASM_EXPORT void acir_honk_solidity_verifier(uint8_t const* proof_buf, uint8_t c
 
 WASM_EXPORT void acir_proof_as_fields_ultra_honk(uint8_t const* proof_buf, fr::vec_out_buf out)
 {
-    auto proof = from_buffer<std::vector<bb::fr>>(from_buffer<std::vector<uint8_t>>(proof_buf));
+    auto proof = many_from_buffer<bb::fr>(from_buffer<std::vector<uint8_t>>(proof_buf));
     *out = to_heap_buffer(proof);
 }
 

barretenberg/ts/src/proof/index.ts

@@ -1,4 +1,4 @@
-import { numToUInt32BE } from "../serialize/serialize.js";
+import { numToUInt32BE } from '../serialize/serialize.js';
 
 /**
  * @description
@@ -22,19 +22,15 @@ export type ProofDataForRecursion = {
   proof: string[];
 };
 
-// Honk proofs start with 4 bytes for the size of the proof in fields
-const metadataOffset = 4;
+// Fields are 32 bytes
 const fieldByteSize = 32;
 
 export function splitHonkProof(
   proofWithPublicInputs: Uint8Array,
   numPublicInputs: number,
 ): { publicInputs: Uint8Array; proof: Uint8Array } {
-  // Remove the metadata (proof size in fields)
-  const proofWithPI = proofWithPublicInputs.slice(metadataOffset);
-
-  const publicInputs = proofWithPI.slice(0, numPublicInputs * fieldByteSize);
-  const proof = proofWithPI.slice(numPublicInputs * fieldByteSize);
+  const publicInputs = proofWithPublicInputs.slice(0, numPublicInputs * fieldByteSize);
+  const proof = proofWithPublicInputs.slice(numPublicInputs * fieldByteSize);
 
   return {
     proof,
@@ -43,10 +39,7 @@ export function splitHonkProof(
 }
 
 export function reconstructHonkProof(publicInputs: Uint8Array, proof: Uint8Array): Uint8Array {
-  // Append proofWithPublicInputs size in fields
-  const proofSize = numToUInt32BE((publicInputs.length + proof.length) / fieldByteSize);
-
-  const proofWithPublicInputs = Uint8Array.from([...proofSize, ...publicInputs, ...proof]);
+  const proofWithPublicInputs = Uint8Array.from([...publicInputs, ...proof]);
   return proofWithPublicInputs;
 }
 

yarn-project/bb-prover/src/prover/bb_prover.ts

@@ -697,8 +697,9 @@ export class BBNativeRollupProver implements ServerCircuitProver {
       const publicInputsFileName = path.join(bbWorkingDirectory, PUBLIC_INPUTS_FILENAME);
       const proofFileName = path.join(bbWorkingDirectory, PROOF_FILENAME);
       const verificationKeyPath = path.join(bbWorkingDirectory, VK_FILENAME);
-      await fs.writeFile(publicInputsFileName, proof.buffer.slice(0, proof.numPublicInputs * 32 + 4));
-      await fs.writeFile(proofFileName, proof.buffer.slice(proof.numPublicInputs * 32 + 4));
+      // TODO(https://github.com/AztecProtocol/aztec-packages/issues/13189): Put this proof parsing logic in the proof class.
+      await fs.writeFile(publicInputsFileName, proof.buffer.slice(0, proof.numPublicInputs * 32));
+      await fs.writeFile(proofFileName, proof.buffer.slice(proof.numPublicInputs * 32));
       await fs.writeFile(verificationKeyPath, verificationKey.keyAsBytes);
 
       const result = await verificationFunction(proofFileName, verificationKeyPath!);
@@ -754,14 +755,10 @@ export class BBNativeRollupProver implements ServerCircuitProver {
     const fieldsWithoutPublicInputs = json.map(Fr.fromHexString);
 
     // Concat binary public inputs and binary proof
-    // This buffer will have the form: [4 bytes of metadata for public inputs, binary public inputs, 4 bytes of metadata for proof, binary proof]
+    // This buffer will have the form: [binary public inputs, binary proof]
     const binaryProofWithPublicInputs = Buffer.concat([binaryPublicInputs, binaryProof]);
     // TODO(https://github.com/AztecProtocol/barretenberg/issues/1312): Get rid of if possible.
-    const metadataLength = 4;
-    assert(
-      binaryProofWithPublicInputs.length ==
-        metadataLength + numPublicInputs * 32 + metadataLength + NESTED_RECURSIVE_PROOF_LENGTH * 32,
-    );
+    assert(binaryProofWithPublicInputs.length == numPublicInputs * 32 + NESTED_RECURSIVE_PROOF_LENGTH * 32);
     logger.debug(
       `Circuit path: ${filePath}, complete proof length: ${json.length}, num public inputs: ${numPublicInputs}, circuit size: ${vkData.circuitSize}, is recursive: ${vkData.isRecursive}, raw length: ${binaryProofWithPublicInputs.length}`,
     );

yarn-project/stdlib/src/proofs/proof.ts

@@ -17,11 +17,6 @@ export class Proof {
   // Make sure this type is not confused with other buffer wrappers
   readonly __proofBrand: any;
 
-  // Honk proofs start with a 4 byte length prefix
-  // the proof metadata starts immediately after
-  // TODO(https://github.com/AztecProtocol/barretenberg/issues/1312): Get rid of if possible.
-  private readonly metadataOffset = 4;
-
   constructor(
     /**
      * Holds the serialized proof data in a binary buffer format.
@@ -69,14 +64,14 @@ export class Proof {
       return this.buffer;
     }
     // We are indexing to this particular size because we are assuming the proof buffer looks like:
-    // [4 bytes of metadata for public inputs, binary public inputs, 4 bytes of metadata for proof, binary proof]
-    const proofStart = this.metadataOffset + Fr.SIZE_IN_BYTES * this.numPublicInputs + this.metadataOffset;
+    // [binary public inputs, binary proof]
+    const proofStart = Fr.SIZE_IN_BYTES * this.numPublicInputs;
     assert(this.buffer.length >= proofStart, 'Proof buffer is not appropriately sized to call withoutPublicInputs()');
     return this.buffer.subarray(proofStart);
   }
 
   // This function assumes that the proof will contain an aggregation object and look something like:
-  // [4 bytes of metadata for public inputs, binary public inputs, 4 bytes of metadata for proof, aggregation object, rest of proof]
+  // [binary public inputs, aggregation object, rest of proof]
   // We are extracting the binary public inputs and reading them as Frs, and also extracting the aggregation object.
   public extractPublicInputs(): Fr[] {
     if (this.isEmpty()) {
@@ -85,9 +80,7 @@ export class Proof {
     }
     assert(this.numPublicInputs >= AGGREGATION_OBJECT_LENGTH, 'Proof does not contain an aggregation object');
     const numInnerPublicInputs = this.numPublicInputs - AGGREGATION_OBJECT_LENGTH;
-    const reader = BufferReader.asReader(
-      this.buffer.subarray(this.metadataOffset, this.metadataOffset + Fr.SIZE_IN_BYTES * numInnerPublicInputs),
-    );
+    const reader = BufferReader.asReader(this.buffer.subarray(0, Fr.SIZE_IN_BYTES * numInnerPublicInputs));
     let publicInputs = reader.readArray(numInnerPublicInputs, Fr);
     // concatenate Fr[] with aggregation object
     publicInputs = publicInputs.concat(this.extractAggregationObject());
@@ -101,12 +94,9 @@ export class Proof {
     }
     assert(this.numPublicInputs >= AGGREGATION_OBJECT_LENGTH, 'Proof does not contain an aggregation object');
     const numInnerPublicInputs = this.numPublicInputs - AGGREGATION_OBJECT_LENGTH;
-    // The aggregation object is currently stored after the initial inner public inputs and after the 4 byte proof metadata.
+    // The aggregation object is currently stored after the initial inner public inputs.
     const reader = BufferReader.asReader(
-      this.buffer.subarray(
-        this.metadataOffset + Fr.SIZE_IN_BYTES * numInnerPublicInputs + this.metadataOffset,
-        this.metadataOffset + Fr.SIZE_IN_BYTES * this.numPublicInputs + this.metadataOffset,
-      ),
+      this.buffer.subarray(Fr.SIZE_IN_BYTES * numInnerPublicInputs, Fr.SIZE_IN_BYTES * this.numPublicInputs),
     );
     return reader.readArray(AGGREGATION_OBJECT_LENGTH, Fr);
   }