chore!: Use msgpack-compact format by default

acvm-repo/acir/codegen/witness.cpp

@@ -1,7 +1,7 @@
 #pragma once
 
 #include "serde.hpp"
-#include "msgpack.hpp"
+#include "barretenberg/serialize/msgpack_impl.hpp"
 #include "bincode.hpp"
 
 namespace Witnesses {
@@ -10,7 +10,7 @@ namespace Witnesses {
             msgpack::object const& o,
             std::string const& name
         ) {
-            if(o.type != msgpack::type::MAP) {
+            if (o.type != msgpack::type::MAP) {
                 std::cerr << o << std::endl;
                 throw_or_abort("expected MAP for " + name);
             }
@@ -28,6 +28,7 @@ namespace Witnesses {
             }
             return kvmap;
         }
+
         template<typename T>
         static void conv_fld_from_kvmap(
             std::map<std::string, msgpack::object const*> const& kvmap,
@@ -48,6 +49,26 @@ namespace Witnesses {
                 throw_or_abort("missing field: " + struct_name + "::" + field_name);
             }
         }
+
+        template<typename T>
+        static void conv_fld_from_array(
+            msgpack::object_array const& array,
+            std::string const& struct_name,
+            std::string const& field_name,
+            T& field,
+            uint32_t index
+        ) {
+            if (index >= array.size) {
+                throw_or_abort("index out of bounds: " + struct_name + "::" + field_name + " at " + std::to_string(index));
+            }
+            auto element = array.ptr[index];
+            try {
+                element.convert(field);
+            } catch (const msgpack::type_error&) {
+                std::cerr << element << std::endl;
+                throw_or_abort("error converting into field " + struct_name + "::" + field_name);
+            }
+        }
     };
     }
 
@@ -106,10 +127,18 @@ namespace Witnesses {
         }
 
         void msgpack_unpack(msgpack::object const& o) {
-            auto name = "StackItem";
-            auto kvmap = Helpers::make_kvmap(o, name);
-            Helpers::conv_fld_from_kvmap(kvmap, name, "index", index, false);
-            Helpers::conv_fld_from_kvmap(kvmap, name, "witness", witness, false);
+            std::string name = "StackItem";
+            if (o.type == msgpack::type::MAP) {
+                auto kvmap = Helpers::make_kvmap(o, name);
+                Helpers::conv_fld_from_kvmap(kvmap, name, "index", index, false);
+                Helpers::conv_fld_from_kvmap(kvmap, name, "witness", witness, false);
+            } else if (o.type == msgpack::type::ARRAY) {
+                auto array = o.via.array; 
+                Helpers::conv_fld_from_array(array, name, "index", index, 0);
+                Helpers::conv_fld_from_array(array, name, "witness", witness, 1);
+            } else {
+                throw_or_abort("expected MAP or ARRAY for " + name);
+            }
         }
     };
 
@@ -126,9 +155,16 @@ namespace Witnesses {
         }
 
         void msgpack_unpack(msgpack::object const& o) {
-            auto name = "WitnessStack";
-            auto kvmap = Helpers::make_kvmap(o, name);
-            Helpers::conv_fld_from_kvmap(kvmap, name, "stack", stack, false);
+            std::string name = "WitnessStack";
+            if (o.type == msgpack::type::MAP) {
+                auto kvmap = Helpers::make_kvmap(o, name);
+                Helpers::conv_fld_from_kvmap(kvmap, name, "stack", stack, false);
+            } else if (o.type == msgpack::type::ARRAY) {
+                auto array = o.via.array; 
+                Helpers::conv_fld_from_array(array, name, "stack", stack, 0);
+            } else {
+                throw_or_abort("expected MAP or ARRAY for " + name);
+            }
         }
     };
 

acvm-repo/acir/src/circuit/mod.rs

@@ -5,9 +5,10 @@ pub mod brillig;
 pub mod opcodes;
 
 use crate::{
+    SerializationFormat,
     circuit::opcodes::display_opcode,
     native_types::{Expression, Witness},
-    serialization::{deserialize_any_format, serialize_with_format_from_env},
+    serialization::{self, deserialize_any_format, serialize_with_format},
 };
 use acir_field::AcirField;
 pub use opcodes::Opcode;
@@ -266,24 +267,30 @@ impl<F: AcirField> Circuit<F> {
 }
 
 impl<F: Serialize + AcirField> Program<F> {
-    /// Serialize and compress the [Program] into bytes.
-    fn write<W: Write>(&self, writer: W) -> std::io::Result<()> {
-        let buf = serialize_with_format_from_env(self)?;
-
+    /// Compress a serialized [Program].
+    fn compress(buf: Vec<u8>) -> std::io::Result<Vec<u8>> {
+        let mut compressed: Vec<u8> = Vec::new();
         // Compress the data, which should help with formats that uses field names.
-        let mut encoder = flate2::write::GzEncoder::new(writer, Compression::default());
+        let mut encoder = flate2::write::GzEncoder::new(&mut compressed, Compression::default());
         encoder.write_all(&buf)?;
         encoder.finish()?;
-        Ok(())
+        Ok(compressed)
+    }
+
+    /// Serialize and compress a [Program] into bytes, using the given format.
+    pub fn serialize_program_with_format(program: &Self, format: serialization::Format) -> Vec<u8> {
+        let program_bytes =
+            serialize_with_format(program, format).expect("expected circuit to be serializable");
+        Self::compress(program_bytes).expect("expected circuit to compress")
     }
 
+    /// Serialize and compress a [Program] into bytes, using the format from the environment, or the default format.
     pub fn serialize_program(program: &Self) -> Vec<u8> {
-        let mut program_bytes: Vec<u8> = Vec::new();
-        program.write(&mut program_bytes).expect("expected circuit to be serializable");
-        program_bytes
+        let format = SerializationFormat::from_env().expect("invalid format");
+        Self::serialize_program_with_format(program, format.unwrap_or_default())
     }
 
-    /// Serialize and base64 encode program
+    /// Serialize, compress then base64 encode a [Program], using the format from the environment, or the default format,
     pub fn serialize_program_base64<S>(program: &Self, s: S) -> Result<S::Ok, S::Error>
     where
         S: Serializer,

acvm-repo/acir/src/lib.rs

@@ -19,6 +19,7 @@ pub use brillig;
 pub use circuit::black_box_functions::BlackBoxFunc;
 pub use circuit::opcodes::InvalidInputBitSize;
 pub use parser::parse_opcodes;
+pub use serialization::Format as SerializationFormat;
 
 #[cfg(test)]
 mod reflection {
@@ -75,6 +76,11 @@ mod reflection {
     #[derive(Clone, PartialEq, Eq, Serialize, Deserialize, Default, Hash)]
     struct ProgramWithoutBrillig<F: AcirField> {
         pub functions: Vec<Circuit<F>>,
+        /// We want to ignore this field. By setting its type as `unit`
+        /// it will not be deserialized, but it will correctly maintain
+        /// the position of the others (although in this case it doesn't)
+        /// matter since it's the last field.
+        pub unconstrained_functions: (),
     }
 
     #[test]
@@ -225,7 +231,10 @@ mod reflection {
         fn add_preamble(source: &mut String) {
             let inc = r#"#include "serde.hpp""#;
             let pos = source.find(inc).expect("serde.hpp missing");
-            source.insert_str(pos + inc.len(), "\n#include \"msgpack.hpp\"");
+            source.insert_str(
+                pos + inc.len(),
+                "\n#include \"barretenberg/serialize/msgpack_impl.hpp\"",
+            );
         }
 
         /// Add helper functions to cut down repetition in the generated code.
@@ -239,7 +248,7 @@ mod reflection {
             msgpack::object const& o,
             std::string const& name
         ) {
-            if(o.type != msgpack::type::MAP) {
+            if (o.type != msgpack::type::MAP) {
                 std::cerr << o << std::endl;
                 throw_or_abort("expected MAP for " + name);
             }
@@ -257,6 +266,7 @@ mod reflection {
             }
             return kvmap;
         }
+
         template<typename T>
         static void conv_fld_from_kvmap(
             std::map<std::string, msgpack::object const*> const& kvmap,
@@ -277,6 +287,26 @@ mod reflection {
                 throw_or_abort("missing field: " + struct_name + "::" + field_name);
             }
         }
+
+        template<typename T>
+        static void conv_fld_from_array(
+            msgpack::object_array const& array,
+            std::string const& struct_name,
+            std::string const& field_name,
+            T& field,
+            uint32_t index
+        ) {
+            if (index >= array.size) {
+                throw_or_abort("index out of bounds: " + struct_name + "::" + field_name + " at " + std::to_string(index));
+            }
+            auto element = array.ptr[index];
+            try {
+                element.convert(field);
+            } catch (const msgpack::type_error&) {
+                std::cerr << element << std::endl;
+                throw_or_abort("error converting into field " + struct_name + "::" + field_name);
+            }
+        }
     };
     "#;
             // cSpell:enable
@@ -361,13 +391,24 @@ mod reflection {
             // or we could reject the data if there was a new field we could
             // not recognize, or we could even handle aliases.
 
+            // We treat unit fields as special, using them to ignore fields during deserialization:
+            // * in 'map' format we skip over them, never try to deserialize them from the map
+            // * in 'tuple' format we jump over their index, ignoring whatever is in that position
+            fn is_unit(field: &Named<Format>) -> bool {
+                matches!(field.value, Format::Unit)
+            }
+
+            let non_unit_field_count = fields.iter().filter(|f| !is_unit(f)).count();
+
             self.msgpack_pack(name, &{
                 let mut body = format!(
                     "
-    packer.pack_map({});",
-                    fields.len()
+    packer.pack_map({non_unit_field_count});",
                 );
                 for field in fields {
+                    if is_unit(field) {
+                        continue;
+                    }
                     let field_name = &field.name;
                     body.push_str(&format!(
                         r#"
@@ -383,20 +424,48 @@ mod reflection {
                 // cSpell:disable
                 let mut body = format!(
                     r#"
-    auto name = "{name}";
-    auto kvmap = Helpers::make_kvmap(o, name);"#
+    std::string name = "{name}";
+    if (o.type == msgpack::type::MAP) {{
+        auto kvmap = Helpers::make_kvmap(o, name);"#
                 );
                 // cSpell:enable
                 for field in fields {
+                    if is_unit(field) {
+                        continue;
+                    }
                     let field_name = &field.name;
                     let is_optional = matches!(field.value, Format::Option(_));
                     // cSpell:disable
                     body.push_str(&format!(
                         r#"
-    Helpers::conv_fld_from_kvmap(kvmap, name, "{field_name}", {field_name}, {is_optional});"#
+        Helpers::conv_fld_from_kvmap(kvmap, name, "{field_name}", {field_name}, {is_optional});"#
                     ));
                     // cSpell:enable
                 }
+                body.push_str(
+                    "
+    } else if (o.type == msgpack::type::ARRAY) {
+        auto array = o.via.array; ",
+                );
+                for (index, field) in fields.iter().enumerate() {
+                    if is_unit(field) {
+                        continue;
+                    }
+                    let field_name = &field.name;
+                    // cSpell:disable
+                    body.push_str(&format!(
+                        r#"
+        Helpers::conv_fld_from_array(array, name, "{field_name}", {field_name}, {index});"#
+                    ));
+                    // cSpell:enable
+                }
+
+                body.push_str(
+                    r#"
+    } else {
+        throw_or_abort("expected MAP or ARRAY for " + name);
+    }"#,
+                );
                 body
             });
         }
@@ -466,9 +535,9 @@ mod reflection {
         packer.pack(tag);
     }} else {{
         std::visit([&packer, tag](const auto& arg) {{
-            std::map<std::string, msgpack::object> data;
-            data[tag] = msgpack::object(arg);
-            packer.pack(data);
+            packer.pack_map(1);
+            packer.pack(tag);
+            arg.msgpack_pack(packer);
         }}, value);
     }}"#
                 )

acvm-repo/acir/src/native_types/witness_map.rs

@@ -11,7 +11,7 @@ use flate2::bufread::GzEncoder;
 use serde::{Deserialize, Serialize};
 use thiserror::Error;
 
-use crate::{native_types::Witness, serialization};
+use crate::{SerializationFormat, native_types::Witness, serialization};
 
 #[derive(Debug, Error)]
 enum SerializationError {
@@ -91,7 +91,17 @@ impl<F> From<BTreeMap<Witness, F>> for WitnessMap<F> {
 impl<F: AcirField + Serialize> WitnessMap<F> {
     /// Serialize and compress.
     pub fn serialize(&self) -> Result<Vec<u8>, WitnessMapError> {
-        let buf = serialization::serialize_with_format_from_env(self)
+        let format = SerializationFormat::from_env()
+            .map_err(|err| SerializationError::Serialize(std::io::Error::other(err)))?;
+        self.serialize_with_format(format.unwrap_or_default())
+    }
+
+    /// Serialize and compress with a given format.
+    pub fn serialize_with_format(
+        &self,
+        format: SerializationFormat,
+    ) -> Result<Vec<u8>, WitnessMapError> {
+        let buf = serialization::serialize_with_format(self, format)
             .map_err(|e| WitnessMapError(SerializationError::Serialize(e)))?;
 
         let mut deflater = GzEncoder::new(buf.as_slice(), Compression::best());

acvm-repo/acir/src/native_types/witness_stack.rs

@@ -7,6 +7,7 @@ use flate2::bufread::GzEncoder;
 use serde::{Deserialize, Serialize};
 use thiserror::Error;
 
+use crate::SerializationFormat;
 use crate::serialization;
 
 use super::WitnessMap;
@@ -72,7 +73,17 @@ impl<F> WitnessStack<F> {
 impl<F: AcirField + Serialize> WitnessStack<F> {
     /// Serialize and compress.
     pub fn serialize(&self) -> Result<Vec<u8>, WitnessStackError> {
-        let buf = serialization::serialize_with_format_from_env(self)
+        let format = SerializationFormat::from_env()
+            .map_err(|err| SerializationError::Serialize(std::io::Error::other(err)))?;
+        self.serialize_with_format(format.unwrap_or_default())
+    }
+
+    /// Serialize and compress with a given format.
+    pub fn serialize_with_format(
+        &self,
+        format: SerializationFormat,
+    ) -> Result<Vec<u8>, WitnessStackError> {
+        let buf = serialization::serialize_with_format(self, format)
             .map_err(|e| WitnessStackError(SerializationError::Serialize(e)))?;
 
         let mut deflater = GzEncoder::new(buf.as_slice(), Compression::best());