Add Definition::Enum::tag_width field

The field allows specifying how many bytes the discriminant byte takes. This in turn allows for better support of custom encoding formats with more than 256 variants and untagged unions. Issue: #181
near · Sep 8, 2023 · edaf46f · edaf46f
1 parent 09c9295
commit edaf46f
Show file tree

Hide file tree

Showing 20 changed files with 206 additions and 140 deletions.
diff --git a/borsh-derive/src/internals/schema/enums/mod.rs b/borsh-derive/src/internals/schema/enums/mod.rs
@@ -53,8 +53,10 @@ pub fn process(input: &ItemEnum, cratename: Path) -> syn::Result<TokenStream2> {
         fn add_definitions_recursively(definitions: &mut #cratename::__private::maybestd::collections::BTreeMap<#cratename::schema::Declaration, #cratename::schema::Definition>) {
             #inner_defs
             #add_recursive_defs
-            let variants = #cratename::__private::maybestd::vec![#(#variants_defs),*];
-            let definition = #cratename::schema::Definition::Enum{variants};
+            let definition = #cratename::schema::Definition::Enum {
+                tag_width: 1,
+                variants: #cratename::__private::maybestd::vec![#(#variants_defs),*],
+            };
             #cratename::schema::add_definition(Self::declaration(), definition, definitions);
         }
     };

diff --git a/borsh-derive/src/internals/schema/enums/snapshots/complex_enum.snap b/borsh-derive/src/internals/schema/enums/snapshots/complex_enum.snap
@@ -35,13 +35,13 @@ impl borsh::BorshSchema for A {
         <AEggs as borsh::BorshSchema>::add_definitions_recursively(definitions);
         <ASalad as borsh::BorshSchema>::add_definitions_recursively(definitions);
         <ASausage as borsh::BorshSchema>::add_definitions_recursively(definitions);
-        let variants = borsh::__private::maybestd::vec![
-            ("Bacon".to_string(), < ABacon > ::declaration()), ("Eggs".to_string(), <
-            AEggs > ::declaration()), ("Salad".to_string(), < ASalad > ::declaration()),
-            ("Sausage".to_string(), < ASausage > ::declaration())
-        ];
         let definition = borsh::schema::Definition::Enum {
-            variants,
+            tag_width: 1,
+            variants: borsh::__private::maybestd::vec![
+                ("Bacon".to_string(), < ABacon > ::declaration()), ("Eggs".to_string(), <
+                AEggs > ::declaration()), ("Salad".to_string(), < ASalad >
+                ::declaration()), ("Sausage".to_string(), < ASausage > ::declaration())
+            ],
         };
         borsh::schema::add_definition(Self::declaration(), definition, definitions);
     }

diff --git a/borsh-derive/src/internals/schema/enums/snapshots/complex_enum_generics.snap b/borsh-derive/src/internals/schema/enums/snapshots/complex_enum_generics.snap
@@ -42,13 +42,14 @@ where
         <AEggs as borsh::BorshSchema>::add_definitions_recursively(definitions);
         <ASalad<C> as borsh::BorshSchema>::add_definitions_recursively(definitions);
         <ASausage<W> as borsh::BorshSchema>::add_definitions_recursively(definitions);
-        let variants = borsh::__private::maybestd::vec![
-            ("Bacon".to_string(), < ABacon > ::declaration()), ("Eggs".to_string(), <
-            AEggs > ::declaration()), ("Salad".to_string(), < ASalad < C > >
-            ::declaration()), ("Sausage".to_string(), < ASausage < W > > ::declaration())
-        ];
         let definition = borsh::schema::Definition::Enum {
-            variants,
+            tag_width: 1,
+            variants: borsh::__private::maybestd::vec![
+                ("Bacon".to_string(), < ABacon > ::declaration()), ("Eggs".to_string(), <
+                AEggs > ::declaration()), ("Salad".to_string(), < ASalad < C > >
+                ::declaration()), ("Sausage".to_string(), < ASausage < W > >
+                ::declaration())
+            ],
         };
         borsh::schema::add_definition(Self::declaration(), definition, definitions);
     }

diff --git a/...ve/src/internals/schema/enums/snapshots/complex_enum_generics_borsh_skip_named_field.snap b/...ve/src/internals/schema/enums/snapshots/complex_enum_generics_borsh_skip_named_field.snap
@@ -44,14 +44,14 @@ where
         <AEggs as borsh::BorshSchema>::add_definitions_recursively(definitions);
         <ASalad<C> as borsh::BorshSchema>::add_definitions_recursively(definitions);
         <ASausage<W, U> as borsh::BorshSchema>::add_definitions_recursively(definitions);
-        let variants = borsh::__private::maybestd::vec![
-            ("Bacon".to_string(), < ABacon > ::declaration()), ("Eggs".to_string(), <
-            AEggs > ::declaration()), ("Salad".to_string(), < ASalad < C > >
-            ::declaration()), ("Sausage".to_string(), < ASausage < W, U > >
-            ::declaration())
-        ];
         let definition = borsh::schema::Definition::Enum {
-            variants,
+            tag_width: 1,
+            variants: borsh::__private::maybestd::vec![
+                ("Bacon".to_string(), < ABacon > ::declaration()), ("Eggs".to_string(), <
+                AEggs > ::declaration()), ("Salad".to_string(), < ASalad < C > >
+                ::declaration()), ("Sausage".to_string(), < ASausage < W, U > >
+                ::declaration())
+            ],
         };
         borsh::schema::add_definition(Self::declaration(), definition, definitions);
     }

diff --git a/...ve/src/internals/schema/enums/snapshots/complex_enum_generics_borsh_skip_tuple_field.snap b/...ve/src/internals/schema/enums/snapshots/complex_enum_generics_borsh_skip_tuple_field.snap
@@ -43,13 +43,14 @@ where
         <AEggs as borsh::BorshSchema>::add_definitions_recursively(definitions);
         <ASalad<C> as borsh::BorshSchema>::add_definitions_recursively(definitions);
         <ASausage<W> as borsh::BorshSchema>::add_definitions_recursively(definitions);
-        let variants = borsh::__private::maybestd::vec![
-            ("Bacon".to_string(), < ABacon > ::declaration()), ("Eggs".to_string(), <
-            AEggs > ::declaration()), ("Salad".to_string(), < ASalad < C > >
-            ::declaration()), ("Sausage".to_string(), < ASausage < W > > ::declaration())
-        ];
         let definition = borsh::schema::Definition::Enum {
-            variants,
+            tag_width: 1,
+            variants: borsh::__private::maybestd::vec![
+                ("Bacon".to_string(), < ABacon > ::declaration()), ("Eggs".to_string(), <
+                AEggs > ::declaration()), ("Salad".to_string(), < ASalad < C > >
+                ::declaration()), ("Sausage".to_string(), < ASausage < W > >
+                ::declaration())
+            ],
         };
         borsh::schema::add_definition(Self::declaration(), definition, definitions);
     }

diff --git a/borsh-derive/src/internals/schema/enums/snapshots/filter_foreign_attrs.snap b/borsh-derive/src/internals/schema/enums/snapshots/filter_foreign_attrs.snap
@@ -29,12 +29,12 @@ impl borsh::BorshSchema for A {
         }
         <AB as borsh::BorshSchema>::add_definitions_recursively(definitions);
         <ANegative as borsh::BorshSchema>::add_definitions_recursively(definitions);
-        let variants = borsh::__private::maybestd::vec![
-            ("B".to_string(), < AB > ::declaration()), ("Negative".to_string(), <
-            ANegative > ::declaration())
-        ];
         let definition = borsh::schema::Definition::Enum {
-            variants,
+            tag_width: 1,
+            variants: borsh::__private::maybestd::vec![
+                ("B".to_string(), < AB > ::declaration()), ("Negative".to_string(), <
+                ANegative > ::declaration())
+            ],
         };
         borsh::schema::add_definition(Self::declaration(), definition, definitions);
     }

diff --git a/borsh-derive/src/internals/schema/enums/snapshots/generic_associated_type.snap b/borsh-derive/src/internals/schema/enums/snapshots/generic_associated_type.snap
@@ -55,12 +55,12 @@ where
         <EnumParametrizedC<
             T,
         > as borsh::BorshSchema>::add_definitions_recursively(definitions);
-        let variants = borsh::__private::maybestd::vec![
-            ("B".to_string(), < EnumParametrizedB < K, V > > ::declaration()), ("C"
-            .to_string(), < EnumParametrizedC < T > > ::declaration())
-        ];
         let definition = borsh::schema::Definition::Enum {
-            variants,
+            tag_width: 1,
+            variants: borsh::__private::maybestd::vec![
+                ("B".to_string(), < EnumParametrizedB < K, V > > ::declaration()), ("C"
+                .to_string(), < EnumParametrizedC < T > > ::declaration())
+            ],
         };
         borsh::schema::add_definition(Self::declaration(), definition, definitions);
     }

diff --git a/...h-derive/src/internals/schema/enums/snapshots/generic_associated_type_param_override.snap b/...h-derive/src/internals/schema/enums/snapshots/generic_associated_type_param_override.snap
@@ -56,12 +56,12 @@ where
         <EnumParametrizedC<
             T,
         > as borsh::BorshSchema>::add_definitions_recursively(definitions);
-        let variants = borsh::__private::maybestd::vec![
-            ("B".to_string(), < EnumParametrizedB < K, V > > ::declaration()), ("C"
-            .to_string(), < EnumParametrizedC < T > > ::declaration())
-        ];
         let definition = borsh::schema::Definition::Enum {
-            variants,
+            tag_width: 1,
+            variants: borsh::__private::maybestd::vec![
+                ("B".to_string(), < EnumParametrizedB < K, V > > ::declaration()), ("C"
+                .to_string(), < EnumParametrizedC < T > > ::declaration())
+            ],
         };
         borsh::schema::add_definition(Self::declaration(), definition, definitions);
     }

diff --git a/borsh-derive/src/internals/schema/enums/snapshots/recursive_enum.snap b/borsh-derive/src/internals/schema/enums/snapshots/recursive_enum.snap
@@ -36,12 +36,12 @@ where
         struct AC<K: Key>(K, Vec<A>);
         <AB<K, V> as borsh::BorshSchema>::add_definitions_recursively(definitions);
         <AC<K> as borsh::BorshSchema>::add_definitions_recursively(definitions);
-        let variants = borsh::__private::maybestd::vec![
-            ("B".to_string(), < AB < K, V > > ::declaration()), ("C".to_string(), < AC <
-            K > > ::declaration())
-        ];
         let definition = borsh::schema::Definition::Enum {
-            variants,
+            tag_width: 1,
+            variants: borsh::__private::maybestd::vec![
+                ("B".to_string(), < AB < K, V > > ::declaration()), ("C".to_string(), <
+                AC < K > > ::declaration())
+            ],
         };
         borsh::schema::add_definition(Self::declaration(), definition, definitions);
     }

diff --git a/borsh-derive/src/internals/schema/enums/snapshots/simple_enum.snap b/borsh-derive/src/internals/schema/enums/snapshots/simple_enum.snap
@@ -22,12 +22,12 @@ impl borsh::BorshSchema for A {
         struct AEggs;
         <ABacon as borsh::BorshSchema>::add_definitions_recursively(definitions);
         <AEggs as borsh::BorshSchema>::add_definitions_recursively(definitions);
-        let variants = borsh::__private::maybestd::vec![
-            ("Bacon".to_string(), < ABacon > ::declaration()), ("Eggs".to_string(), <
-            AEggs > ::declaration())
-        ];
         let definition = borsh::schema::Definition::Enum {
-            variants,
+            tag_width: 1,
+            variants: borsh::__private::maybestd::vec![
+                ("Bacon".to_string(), < ABacon > ::declaration()), ("Eggs".to_string(), <
+                AEggs > ::declaration())
+            ],
         };
         borsh::schema::add_definition(Self::declaration(), definition, definitions);
     }

diff --git a/borsh-derive/src/internals/schema/enums/snapshots/simple_enum_with_custom_crate.snap b/borsh-derive/src/internals/schema/enums/snapshots/simple_enum_with_custom_crate.snap
@@ -26,12 +26,12 @@ impl reexporter::borsh::BorshSchema for A {
         <AEggs as reexporter::borsh::BorshSchema>::add_definitions_recursively(
             definitions,
         );
-        let variants = reexporter::borsh::__private::maybestd::vec![
-            ("Bacon".to_string(), < ABacon > ::declaration()), ("Eggs".to_string(), <
-            AEggs > ::declaration())
-        ];
         let definition = reexporter::borsh::schema::Definition::Enum {
-            variants,
+            tag_width: 1,
+            variants: reexporter::borsh::__private::maybestd::vec![
+                ("Bacon".to_string(), < ABacon > ::declaration()), ("Eggs".to_string(), <
+                AEggs > ::declaration())
+            ],
         };
         reexporter::borsh::schema::add_definition(
             Self::declaration(),

diff --git a/borsh-derive/src/internals/schema/enums/snapshots/single_field_enum.snap b/borsh-derive/src/internals/schema/enums/snapshots/single_field_enum.snap
@@ -17,11 +17,11 @@ impl borsh::BorshSchema for A {
         #[borsh(crate = "borsh")]
         struct ABacon;
         <ABacon as borsh::BorshSchema>::add_definitions_recursively(definitions);
-        let variants = borsh::__private::maybestd::vec![
-            ("Bacon".to_string(), < ABacon > ::declaration())
-        ];
         let definition = borsh::schema::Definition::Enum {
-            variants,
+            tag_width: 1,
+            variants: borsh::__private::maybestd::vec![
+                ("Bacon".to_string(), < ABacon > ::declaration())
+            ],
         };
         borsh::schema::add_definition(Self::declaration(), definition, definitions);
     }

diff --git a/borsh-derive/src/internals/schema/enums/snapshots/trailing_comma_generics.snap b/borsh-derive/src/internals/schema/enums/snapshots/trailing_comma_generics.snap
@@ -39,12 +39,12 @@ where
             B: Display + Debug;
         <SideLeft<A> as borsh::BorshSchema>::add_definitions_recursively(definitions);
         <SideRight<B> as borsh::BorshSchema>::add_definitions_recursively(definitions);
-        let variants = borsh::__private::maybestd::vec![
-            ("Left".to_string(), < SideLeft < A > > ::declaration()), ("Right"
-            .to_string(), < SideRight < B > > ::declaration())
-        ];
         let definition = borsh::schema::Definition::Enum {
-            variants,
+            tag_width: 1,
+            variants: borsh::__private::maybestd::vec![
+                ("Left".to_string(), < SideLeft < A > > ::declaration()), ("Right"
+                .to_string(), < SideRight < B > > ::declaration())
+            ],
         };
         borsh::schema::add_definition(Self::declaration(), definition, definitions);
     }

diff --git a/borsh-derive/src/internals/schema/enums/snapshots/with_funcs_attr.snap b/borsh-derive/src/internals/schema/enums/snapshots/with_funcs_attr.snap
@@ -40,12 +40,12 @@ where
         );
         <CC3 as borsh::BorshSchema>::add_definitions_recursively(definitions);
         <CC4<K, V> as borsh::BorshSchema>::add_definitions_recursively(definitions);
-        let variants = borsh::__private::maybestd::vec![
-            ("C3".to_string(), < CC3 > ::declaration()), ("C4".to_string(), < CC4 < K, V
-            > > ::declaration())
-        ];
         let definition = borsh::schema::Definition::Enum {
-            variants,
+            tag_width: 1,
+            variants: borsh::__private::maybestd::vec![
+                ("C3".to_string(), < CC3 > ::declaration()), ("C4".to_string(), < CC4 <
+                K, V > > ::declaration())
+            ],
         };
         borsh::schema::add_definition(Self::declaration(), definition, definitions);
     }

diff --git a/borsh/src/schema.rs b/borsh/src/schema.rs
@@ -33,6 +33,16 @@ pub type VariantName = String;
 pub type FieldName = String;
 /// The type that we use to represent the definition of the Borsh type.
 
+/// Description of data encoding on the wire.
+///
+/// Note: Since at the end of the day users can define arbitrary serialisation,
+/// it’s not always possible to express using definitions how a type is encoded.
+/// For example, let’s say programmer uses [varint] encoding for their data.
+/// Such type cannot be fully expressed using `BorshSchema` (or at least not
+/// easily).  As a consequence, a tool which validates whether binary data
+/// matches a schema wouldn’t be able to validate data including such types.
+///
+/// [varint]: https://en.wikipedia.org/wiki/Variable-length_quantity#Variants
 #[derive(Clone, PartialEq, Eq, Debug, BorshSerialize, BorshDeserialize, BorshSchemaMacro)]
 pub enum Definition {
     /// A fixed-size array with the length known at the compile time and the same-type elements.
@@ -42,10 +52,31 @@ pub enum Definition {
     /// A fixed-size tuple with the length known at the compile time and the elements of different
     /// types.
     Tuple { elements: Vec<Declaration> },
-    /// A tagged union, a.k.a enum. Tagged-unions have variants with associated structures.
+
+    /// A possibly tagged union, a.k.a enum.
+    ///
+    /// Tagged unions are prefixed by a tag identifying encoded variant followed
+    /// by encoding of that variant.
+    ///
+    /// Untagged unions don’t have a separate tag which means that knowledge of
+    /// the type is necessary to fully analyse the binary.  Variants may still
+    /// be used to list possible values or determine the longest possible
+    /// encoding.
     Enum {
+        /// Width in bytes of the discriminant tag.
+        ///
+        /// Zero indicates this is an untagged union.  In standard borsh
+        /// encoding this is one however custom encoding formats may use larger
+        /// width if they need to encode more than 256 variants.
+        ///
+        /// Note: This definition must not be used if the tag is not encoded
+        /// using little-endian format.
+        tag_width: u8,
+
+        /// Possible variants of the enumeration.
         variants: Vec<(VariantName, Declaration)>,
     },
+
     /// A structure, structurally similar to a tuple.
     Struct { fields: Fields },
 }
@@ -260,6 +291,7 @@ where
 {
     fn add_definitions_recursively(definitions: &mut BTreeMap<Declaration, Definition>) {
         let definition = Definition::Enum {
+            tag_width: 1,
             variants: vec![
                 ("None".to_string(), <()>::declaration()),
                 ("Some".to_string(), T::declaration()),
@@ -281,6 +313,7 @@ where
 {
     fn add_definitions_recursively(definitions: &mut BTreeMap<Declaration, Definition>) {
         let definition = Definition::Enum {
+            tag_width: 1,
             variants: vec![
                 ("Ok".to_string(), T::declaration()),
                 ("Err".to_string(), E::declaration()),
@@ -333,7 +366,7 @@ where
 #[cfg(hash_collections)]
 pub mod hashes {
     //!
-    //! Module defines [BorshSchema](crate::schema::BorshSchema) implementation for  
+    //! Module defines [BorshSchema](crate::schema::BorshSchema) implementation for
     //! [HashMap](std::collections::HashMap)/[HashSet](std::collections::HashSet).
     use crate::BorshSchema;
 
@@ -500,11 +533,14 @@ mod tests {
         Option::<u64>::add_definitions_recursively(&mut actual_defs);
         assert_eq!("Option<u64>", actual_name);
         assert_eq!(
-            map! {"Option<u64>" =>
-            Definition::Enum{ variants: vec![
-                ("None".to_string(), "nil".to_string()),
-                ("Some".to_string(), "u64".to_string()),
-            ]}
+            map! {
+                "Option<u64>" => Definition::Enum {
+                    tag_width: 1,
+                    variants: vec![
+                        ("None".to_string(), "nil".to_string()),
+                        ("Some".to_string(), "u64".to_string()),
+                    ]
+                }
             },
             actual_defs
         );
@@ -518,16 +554,20 @@ mod tests {
         assert_eq!("Option<Option<u64>>", actual_name);
         assert_eq!(
             map! {
-            "Option<u64>" =>
-                Definition::Enum {variants: vec![
-                ("None".to_string(), "nil".to_string()),
-                ("Some".to_string(), "u64".to_string()),
-                ]},
-            "Option<Option<u64>>" =>
-                Definition::Enum {variants: vec![
-                ("None".to_string(), "nil".to_string()),
-                ("Some".to_string(), "Option<u64>".to_string()),
-                ]}
+                "Option<u64>" => Definition::Enum {
+                    tag_width: 1,
+                    variants: vec![
+                        ("None".to_string(), "nil".to_string()),
+                        ("Some".to_string(), "u64".to_string()),
+                    ]
+                },
+                "Option<Option<u64>>" => Definition::Enum {
+                    tag_width: 1,
+                    variants: vec![
+                        ("None".to_string(), "nil".to_string()),
+                        ("Some".to_string(), "Option<u64>".to_string()),
+                    ]
+                }
             },
             actual_defs
         );

diff --git a/borsh/tests/snapshots/test_schema_enums__complex_enum_with_schema.snap b/borsh/tests/snapshots/test_schema_enums__complex_enum_with_schema.snap
@@ -18,6 +18,7 @@ expression: data
     0,
     65,
     3,
+    1,
     4,
     0,
     0,