From 1bcdadbd1c5eca5d1ff911a0dfcf912e9e310a80 Mon Sep 17 00:00:00 2001 From: Jack Klamer Date: Thu, 21 Apr 2022 21:16:32 -0500 Subject: [PATCH 01/66] Encoer v1 with interop data --- .../apache/avro/message/TestInteropData.java | 39 +++++++ lang/rust/avro/src/encode.rs | 12 +-- lang/rust/avro/src/error.rs | 2 + lang/rust/avro/src/schema.rs | 80 +++++++++++++++ lang/rust/avro/src/ser.rs | 4 +- lang/rust/avro/src/types.rs | 10 +- lang/rust/avro/src/writer.rs | 97 +++++++++++++++++- share/test/data/messageV1/README.md | 36 +++++++ share/test/data/messageV1/test_message.bin | Bin 0 -> 38 bytes share/test/data/messageV1/test_schema.json | 22 ++++ 10 files changed, 286 insertions(+), 16 deletions(-) create mode 100644 lang/java/avro/src/test/java/org/apache/avro/message/TestInteropData.java create mode 100644 share/test/data/messageV1/README.md create mode 100644 share/test/data/messageV1/test_message.bin create mode 100644 share/test/data/messageV1/test_schema.json diff --git a/lang/java/avro/src/test/java/org/apache/avro/message/TestInteropData.java b/lang/java/avro/src/test/java/org/apache/avro/message/TestInteropData.java new file mode 100644 index 00000000000..c7489c1a4ec --- /dev/null +++ b/lang/java/avro/src/test/java/org/apache/avro/message/TestInteropData.java @@ -0,0 +1,39 @@ +package org.apache.avro.message; + +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecordBuilder; +import org.junit.Test; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; + +public class TestInteropData { + private static String inDir = System.getProperty("share.dir", "../../../share") + "/test/data/messageV1"; + private static File SCHEMA_FILE = new File(inDir + "/test_schema.json"); + private static File MESSAGE_FILE = new File(inDir + "/test_message.bin"); + private static final Schema SCHEMA; + private static final GenericRecordBuilder BUILDER; + + static { + try { + SCHEMA = new Schema.Parser().parse(new FileInputStream(SCHEMA_FILE)); + BUILDER = new GenericRecordBuilder(SCHEMA); + } catch (IOException e) { + throw new RuntimeException("Interop Message Data Schema not found"); + } + } + + @Test + public void generateData() throws IOException { + MessageEncoder encoder = new BinaryMessageEncoder<>(GenericData.get(), SCHEMA); + BUILDER.set("id", 5L).set("name", "Bill").set("tags", Arrays.asList("dog_lover", "cat_hater")).build(); + ByteBuffer buffer = encoder + .encode(BUILDER.set("id", 5L).set("name", "Bill").set("tags", Arrays.asList("dog_lover", "cat_hater")).build()); + new FileOutputStream(MESSAGE_FILE).write(buffer.array()); + } +} diff --git a/lang/rust/avro/src/encode.rs b/lang/rust/avro/src/encode.rs index c4c0bd3feaa..84e665f41c1 100644 --- a/lang/rust/avro/src/encode.rs +++ b/lang/rust/avro/src/encode.rs @@ -16,12 +16,12 @@ // under the License. use crate::{ - schema::{NamesRef, Namespace, ResolvedSchema, Schema, SchemaKind}, + schema::{Namespace, ResolvedSchema, Schema, SchemaKind, Name}, types::{Value, ValueKind}, util::{zig_i32, zig_i64}, AvroResult, Error, }; -use std::convert::{TryFrom, TryInto}; +use std::{convert::{TryFrom, TryInto}, borrow::Borrow, collections::HashMap}; /// Encode a `Value` into avro format. /// @@ -47,19 +47,19 @@ fn encode_int(i: i32, buffer: &mut Vec) { zig_i32(i, buffer) } -pub(crate) fn encode_internal( +pub(crate) fn encode_internal< S :Borrow>( value: &Value, schema: &Schema, - names: &NamesRef, + names: &HashMap, enclosing_namespace: &Namespace, buffer: &mut Vec, ) -> AvroResult<()> { if let Schema::Ref { ref name } = schema { let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); - let resolved = *names + let resolved = names .get(&fully_qualified_name) .ok_or(Error::SchemaResolutionError(fully_qualified_name))?; - return encode_internal(value, resolved, names, enclosing_namespace, buffer); + return encode_internal(value, resolved.borrow(), names, enclosing_namespace, buffer); } match value { diff --git a/lang/rust/avro/src/error.rs b/lang/rust/avro/src/error.rs index 908e040e9be..900e59b4662 100644 --- a/lang/rust/avro/src/error.rs +++ b/lang/rust/avro/src/error.rs @@ -408,6 +408,8 @@ pub enum Error { value_kind: ValueKind, supported_schema: Vec, }, + #[error("Internal buffer not drained properly. Reinitialize the writer struct")] + IllegalSingleObjectWriterState, } impl serde::ser::Error for Error { diff --git a/lang/rust/avro/src/schema.rs b/lang/rust/avro/src/schema.rs index 6134c8256ab..95a21b853d3 100644 --- a/lang/rust/avro/src/schema.rs +++ b/lang/rust/avro/src/schema.rs @@ -434,6 +434,86 @@ impl<'s> ResolvedSchema<'s> { } } +pub(crate) struct ResolvedOwnedSchema { + names: Names, + root_schema: Schema, +} + +impl<'s> TryFrom for ResolvedOwnedSchema { + type Error = Error; + + fn try_from(schema: Schema) -> AvroResult { + let names = HashMap::new(); + let mut rs = ResolvedOwnedSchema { + names, + root_schema: schema, + }; + Self::from_internal(&rs.root_schema, &mut rs.names, &None)?; + Ok(rs) + } +} + +impl ResolvedOwnedSchema { + pub(crate) fn get_root_schema(&self) -> &Schema { + &self.root_schema + } + pub(crate) fn get_names(&self) -> &Names { + &self.names + } + + fn from_internal( + schema: &Schema, + names: &mut Names, + enclosing_namespace: &Namespace, + ) -> AvroResult<()> { + match schema { + Schema::Array(schema) | Schema::Map(schema) => { + Self::from_internal(schema, names, enclosing_namespace) + } + Schema::Union(UnionSchema { schemas, .. }) => { + for schema in schemas { + Self::from_internal(schema, names, enclosing_namespace)? + } + Ok(()) + } + Schema::Enum { name, .. } | Schema::Fixed { name, .. } => { + let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); + if names + .insert(fully_qualified_name.clone(), schema.clone()) + .is_some() + { + Err(Error::AmbiguousSchemaDefinition(fully_qualified_name)) + } else { + Ok(()) + } + } + Schema::Record { name, fields, .. } => { + let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); + if names + .insert(fully_qualified_name.clone(), schema.clone()) + .is_some() + { + Err(Error::AmbiguousSchemaDefinition(fully_qualified_name)) + } else { + let record_namespace = fully_qualified_name.namespace; + for field in fields { + Self::from_internal(&field.schema, names, &record_namespace)? + } + Ok(()) + } + } + Schema::Ref { name } => { + let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); + names + .get(&fully_qualified_name) + .map(|_| ()) + .ok_or(Error::SchemaResolutionError(fully_qualified_name)) + } + _ => Ok(()), + } + } +} + /// Represents a `field` in a `record` Avro schema. #[derive(Clone, Debug, PartialEq)] pub struct RecordField { diff --git a/lang/rust/avro/src/ser.rs b/lang/rust/avro/src/ser.rs index 5cff13e5666..9520ef69cbd 100644 --- a/lang/rust/avro/src/ser.rs +++ b/lang/rust/avro/src/ser.rs @@ -178,7 +178,7 @@ impl<'b> ser::Serializer for &'b mut Serializer { } fn serialize_none(self) -> Result { - Ok(Value::from(None::)) + Ok(Value::Null) } fn serialize_some(self, value: &T) -> Result @@ -186,7 +186,7 @@ impl<'b> ser::Serializer for &'b mut Serializer { T: Serialize, { let v = value.serialize(&mut Serializer::default())?; - Ok(Value::from(Some(v))) + Ok(v) } fn serialize_unit(self) -> Result { diff --git a/lang/rust/avro/src/types.rs b/lang/rust/avro/src/types.rs index 25d9681209d..f24fb33d1a8 100644 --- a/lang/rust/avro/src/types.rs +++ b/lang/rust/avro/src/types.rs @@ -20,12 +20,12 @@ use crate::{ decimal::Decimal, duration::Duration, schema::{ - NamesRef, Precision, RecordField, ResolvedSchema, Scale, Schema, SchemaKind, UnionSchema, + NamesRef, Precision, RecordField, ResolvedSchema, Scale, Schema, SchemaKind, UnionSchema, Name, }, AvroResult, Error, }; use serde_json::{Number, Value as JsonValue}; -use std::{collections::HashMap, convert::TryFrom, hash::BuildHasher, str::FromStr, u8}; +use std::{collections::HashMap, convert::TryFrom, hash::BuildHasher, str::FromStr, u8, borrow::Borrow}; use uuid::Uuid; /// Compute the maximum decimal value precision of a byte array of length `len` could hold. @@ -355,7 +355,7 @@ impl Value { } } - pub(crate) fn validate_internal(&self, schema: &Schema, names: &NamesRef) -> Option { + pub(crate) fn validate_internal>(&self, schema: &Schema, names: &HashMap) -> Option { match (self, schema) { (_, &Schema::Ref { ref name }) => names.get(name).map_or_else( || { @@ -365,7 +365,7 @@ impl Value { names.keys() )); }, - |s| self.validate_internal(s, names), + |s| self.validate_internal(s.borrow(), names), ), (&Value::Null, &Schema::Null) => None, (&Value::Boolean(_), &Schema::Boolean) => None, @@ -1080,7 +1080,7 @@ mod tests { ]; for (value, schema, valid, expected_err_message) in value_schema_valid.into_iter() { - let err_message = value.validate_internal(&schema, &HashMap::default()); + let err_message = value.validate_internal::(&schema, &HashMap::default()); assert_eq!(valid, err_message.is_none()); if !valid { let full_err_message = format!( diff --git a/lang/rust/avro/src/writer.rs b/lang/rust/avro/src/writer.rs index e815f517f7c..4bbd8c1de60 100644 --- a/lang/rust/avro/src/writer.rs +++ b/lang/rust/avro/src/writer.rs @@ -18,14 +18,14 @@ //! Logic handling writing in Avro format at user level. use crate::{ encode::{encode, encode_internal, encode_to_vec}, - schema::{ResolvedSchema, Schema}, + schema::{ResolvedSchema, Schema, AvroSchema, ResolvedOwnedSchema}, ser::Serializer, types::Value, - AvroResult, Codec, Error, + AvroResult, Codec, Error, rabin::Rabin, }; use rand::random; use serde::Serialize; -use std::{collections::HashMap, convert::TryFrom, io::Write}; +use std::{collections::HashMap, convert::TryFrom, io::Write, marker::PhantomData}; const DEFAULT_BLOCK_SIZE: usize = 16000; const AVRO_OBJECT_HEADER: &[u8] = b"Obj\x01"; @@ -352,6 +352,76 @@ fn write_avro_datum>( Ok(()) } +/// Writer that encodes messages according to the single object encoding v1 spec +pub struct SingleObjectWriter +where + T: AvroSchema +{ + buffer: Vec, + resolved: ResolvedOwnedSchema, + header: [u8; 10], + _model: PhantomData, +} + +impl SingleObjectWriter +where + T: AvroSchema +{ + pub fn with_capacity(buffer_cap: usize) -> AvroResult> { + let schema = T::get_schema(); + let fingerprint = schema.fingerprint::(); + Ok(SingleObjectWriter{ + buffer: Vec::with_capacity(buffer_cap), + resolved: ResolvedOwnedSchema::try_from(schema)?, + header: [0xC3, 0x01, + fingerprint.bytes[0], + fingerprint.bytes[1], + fingerprint.bytes[2], + fingerprint.bytes[3], + fingerprint.bytes[4], + fingerprint.bytes[5], + fingerprint.bytes[6], + fingerprint.bytes[7]], + _model: PhantomData, + }) + } + + fn write_internal(&mut self, v: &Value, writer: &mut W) -> AvroResult { + if ! self.buffer.is_empty() { + Err(Error::IllegalSingleObjectWriterState) + } else { + self.buffer.extend_from_slice(&self.header); + write_value_ref_owned_resolved( &self.resolved, v, &mut self.buffer)?; + writer.write_all(&self.buffer).map_err(Error::WriteBytes)?; + let len = self.buffer.len(); + self.buffer.clear(); + Ok(len) + } + } +} + + +impl SingleObjectWriter +where + T: AvroSchema + Into +{ + pub fn write_value(&mut self, data: T, writer: &mut W) -> AvroResult { + let v: Value = data.into(); + self.write_internal(&v, writer) + } +} + +impl SingleObjectWriter +where + T: AvroSchema + Serialize +{ + pub fn write(&mut self, data: T, writer: &mut W) -> AvroResult { + let mut serializer = Serializer::default(); + let v = data.serialize(&mut serializer)?; + self.write_internal(&v, writer) + } +} + fn write_value_ref_resolved( resolved_schema: &ResolvedSchema, value: &Value, @@ -373,6 +443,27 @@ fn write_value_ref_resolved( Ok(()) } +fn write_value_ref_owned_resolved( + resolved_schema: &ResolvedOwnedSchema, + value: &Value, + buffer: &mut Vec, +) -> AvroResult<()> { + if let Some(err) = value.validate_internal( + resolved_schema.get_root_schema(), + resolved_schema.get_names(), + ) { + return Err(Error::ValidationWithReason(err)); + } + encode_internal( + value, + resolved_schema.get_root_schema(), + resolved_schema.get_names(), + &None, + buffer, + )?; + Ok(()) +} + /// Encode a compatible value (implementing the `ToAvro` trait) into Avro format, also /// performing schema validation. /// diff --git a/share/test/data/messageV1/README.md b/share/test/data/messageV1/README.md new file mode 100644 index 00000000000..048815e78fd --- /dev/null +++ b/share/test/data/messageV1/README.md @@ -0,0 +1,36 @@ +BinaryMessage data in single object encoding https://avro.apache.org/docs/current/spec.html#single_object_encoding + +Ground truth data generated with Java Code + +The binary data will be the V1 sincle object encoding with the schema of +``` +{ + "type":"record", + "namespace":"org.apache.avro", + "name":"TestMessage", + "fields":[ + { + "name":"id", + "type":"long" + }, + { + "name":"name", + "type":"string" + }, + { + "name":"tags", + "type":{ + "type":"array", + "items":"string" + } + }, + { + "name":"scores", + "type":{ + "type":"map", + "values":"double" + } + } + ] +} +``` \ No newline at end of file diff --git a/share/test/data/messageV1/test_message.bin b/share/test/data/messageV1/test_message.bin new file mode 100644 index 0000000000000000000000000000000000000000..025d46151de381840b61ddc0679cccda9d98b9ef GIT binary patch literal 38 tcmX@ixKj7wk3~*jZMis{GIMfRgi`X;<8$)MQj3I=6HDSV5=&Bx7yvWB4zmCN literal 0 HcmV?d00001 diff --git a/share/test/data/messageV1/test_schema.json b/share/test/data/messageV1/test_schema.json new file mode 100644 index 00000000000..cd49b72a456 --- /dev/null +++ b/share/test/data/messageV1/test_schema.json @@ -0,0 +1,22 @@ +{ + "type":"record", + "namespace":"org.apache.avro", + "name":"TestMessage", + "fields":[ + { + "name":"id", + "type":"long" + }, + { + "name":"name", + "type":"string" + }, + { + "name":"tags", + "type":{ + "type":"array", + "items":"string" + } + } + ] +} \ No newline at end of file From afee9fdb62a65b1a6e27e798018e191c1e1317e4 Mon Sep 17 00:00:00 2001 From: Jack Klamer Date: Sat, 23 Apr 2022 16:11:37 -0500 Subject: [PATCH 02/66] unit tested --- .../examples/test_interop_message_data.rs | 5 + lang/rust/avro/src/types.rs | 4 +- lang/rust/avro/src/writer.rs | 180 +++++++++++++++--- 3 files changed, 157 insertions(+), 32 deletions(-) create mode 100644 lang/rust/avro/examples/test_interop_message_data.rs diff --git a/lang/rust/avro/examples/test_interop_message_data.rs b/lang/rust/avro/examples/test_interop_message_data.rs new file mode 100644 index 00000000000..0b50d0284f8 --- /dev/null +++ b/lang/rust/avro/examples/test_interop_message_data.rs @@ -0,0 +1,5 @@ + + +fn main() { + +} \ No newline at end of file diff --git a/lang/rust/avro/src/types.rs b/lang/rust/avro/src/types.rs index f24fb33d1a8..251b44c426d 100644 --- a/lang/rust/avro/src/types.rs +++ b/lang/rust/avro/src/types.rs @@ -25,7 +25,7 @@ use crate::{ AvroResult, Error, }; use serde_json::{Number, Value as JsonValue}; -use std::{collections::HashMap, convert::TryFrom, hash::BuildHasher, str::FromStr, u8, borrow::Borrow}; +use std::{collections::HashMap, convert::TryFrom, hash::BuildHasher, str::FromStr, u8}; use uuid::Uuid; /// Compute the maximum decimal value precision of a byte array of length `len` could hold. @@ -355,7 +355,7 @@ impl Value { } } - pub(crate) fn validate_internal>(&self, schema: &Schema, names: &HashMap) -> Option { + pub(crate) fn validate_internal>(&self, schema: &Schema, names: &HashMap) -> Option { match (self, schema) { (_, &Schema::Ref { ref name }) => names.get(name).map_or_else( || { diff --git a/lang/rust/avro/src/writer.rs b/lang/rust/avro/src/writer.rs index 4bbd8c1de60..99c3a8a1690 100644 --- a/lang/rust/avro/src/writer.rs +++ b/lang/rust/avro/src/writer.rs @@ -353,51 +353,75 @@ fn write_avro_datum>( } /// Writer that encodes messages according to the single object encoding v1 spec -pub struct SingleObjectWriter -where - T: AvroSchema +/// Uses an API similar to the current File Writer +/// Writes all object bytes at once, and drains internal buffer +pub struct GenericSingleObjectWriter { buffer: Vec, resolved: ResolvedOwnedSchema, - header: [u8; 10], - _model: PhantomData, } -impl SingleObjectWriter -where - T: AvroSchema +impl GenericSingleObjectWriter { - pub fn with_capacity(buffer_cap: usize) -> AvroResult> { - let schema = T::get_schema(); + pub fn new_with_capacity(schema: &Schema, initial_buffer_cap: usize) -> AvroResult { let fingerprint = schema.fingerprint::(); - Ok(SingleObjectWriter{ - buffer: Vec::with_capacity(buffer_cap), - resolved: ResolvedOwnedSchema::try_from(schema)?, - header: [0xC3, 0x01, - fingerprint.bytes[0], - fingerprint.bytes[1], - fingerprint.bytes[2], - fingerprint.bytes[3], - fingerprint.bytes[4], - fingerprint.bytes[5], - fingerprint.bytes[6], - fingerprint.bytes[7]], - _model: PhantomData, + let mut buffer = Vec::with_capacity(initial_buffer_cap); + let header =[0xC3, 0x01, + fingerprint.bytes[0], + fingerprint.bytes[1], + fingerprint.bytes[2], + fingerprint.bytes[3], + fingerprint.bytes[4], + fingerprint.bytes[5], + fingerprint.bytes[6], + fingerprint.bytes[7]]; + buffer.extend_from_slice(&header); + + Ok(GenericSingleObjectWriter{ + buffer, + resolved: ResolvedOwnedSchema::try_from(schema.clone())?, }) } - fn write_internal(&mut self, v: &Value, writer: &mut W) -> AvroResult { - if ! self.buffer.is_empty() { + /// Wrtite the referenced Value to the provided Write object. Returns a result with the number of bytes writtern including the header + pub fn write_value_ref(&mut self, v: &Value, writer: &mut W) -> AvroResult{ + if self.buffer.len() != 10 { Err(Error::IllegalSingleObjectWriterState) } else { - self.buffer.extend_from_slice(&self.header); write_value_ref_owned_resolved( &self.resolved, v, &mut self.buffer)?; writer.write_all(&self.buffer).map_err(Error::WriteBytes)?; let len = self.buffer.len(); - self.buffer.clear(); + self.buffer.truncate(10); Ok(len) } } + + /// Wrtite the Value to the provided Write object. Returns a result with the number of bytes writtern including the header + pub fn write_value(&mut self, v: Value, writer: &mut W) -> AvroResult{ + self.write_value_ref(&v, writer) + } +} + +/// Writer that encodes messages according to the single object encoding v1 spec +pub struct SingleObjectWriter +where + T: AvroSchema +{ + inner: GenericSingleObjectWriter, + _model: PhantomData, +} + +impl SingleObjectWriter +where + T: AvroSchema +{ + pub fn with_capacity(buffer_cap: usize) -> AvroResult> { + let schema = T::get_schema(); + Ok(SingleObjectWriter{ + inner: GenericSingleObjectWriter::new_with_capacity(&schema, buffer_cap)?, + _model: PhantomData, + }) + } } @@ -405,9 +429,10 @@ impl SingleObjectWriter where T: AvroSchema + Into { + /// Wrtite the Into to the provided Write object. Returns a result with the number of bytes writtern including the header pub fn write_value(&mut self, data: T, writer: &mut W) -> AvroResult { let v: Value = data.into(); - self.write_internal(&v, writer) + self.inner.write_value_ref(&v, writer) } } @@ -415,10 +440,16 @@ impl SingleObjectWriter where T: AvroSchema + Serialize { - pub fn write(&mut self, data: T, writer: &mut W) -> AvroResult { + /// Wrtite the Serialize object to the provided Write object. Returns a result with the number of bytes writtern including the header + pub fn write_ref(&mut self, data: &T, writer: &mut W) -> AvroResult { let mut serializer = Serializer::default(); let v = data.serialize(&mut serializer)?; - self.write_internal(&v, writer) + self.inner.write_value_ref(&v, writer) + } + + /// Wrtite the Serialize object to the provided Write object. Returns a result with the number of bytes writtern including the header + pub fn write(&mut self, data: T, writer: &mut W) -> AvroResult { + self.write_ref(&data, writer) } } @@ -1034,4 +1065,93 @@ mod tests { assert_eq!(writer.user_metadata, user_meta_data); } + + #[derive(Serialize, Clone)] + struct TestSingleObjectWriter { + a: i64, + b: f64, + c: Vec + } + + impl AvroSchema for TestSingleObjectWriter { + fn get_schema() -> Schema { + let schema = r#" + { + "type":"record", + "name":"TestSingleObjectWrtierSerialize", + "fields":[ + { + "name":"a", + "type":"long" + }, + { + "name":"b", + "type":"double" + }, + { + "name":"c", + "type":{ + "type":"array", + "items":"string" + } + } + ] + } + "#; + Schema::parse_str(schema).unwrap() + } + } + + impl Into for TestSingleObjectWriter { + fn into(self) -> Value { + Value::Record(vec![ + ("a".into(), self.a.into()), + ("b".into(), self.b.into()), + ("c".into(), Value::Array(self.c.into_iter().map(|s| s.into()).collect())) + ]) + } + } + + #[test] + fn test_single_object_writer() { + let mut buf: Vec = Vec::new(); + let obj = TestSingleObjectWriter{ + a: 300, + b: 34.555, + c: vec!["cat".into(),"dog".into()] + }; + let mut writer = GenericSingleObjectWriter::new_with_capacity(&TestSingleObjectWriter::get_schema(), 1024).expect("Should resolve schema"); + let value = obj.into(); + let written_bytes = writer.write_value_ref(&value, &mut buf).expect("Error serializing properly"); + + assert!(buf.len() > 10, "no bytes written"); + assert_eq!(buf.len(), written_bytes); + assert_eq!(buf[0] , 0xC3); + assert_eq!(buf[1] , 0x01); + assert_eq!(&buf[2..10], &TestSingleObjectWriter::get_schema().fingerprint::().bytes[..]); + let mut msg_binary = Vec::new(); + encode(&value, &TestSingleObjectWriter::get_schema(), &mut msg_binary).expect("encode should have failed by here as a depndency of any writing"); + assert_eq!(&buf[10..], &msg_binary[..]) + } + + #[test] + fn test_writer_parity() { + let obj1 = TestSingleObjectWriter{ + a: 300, + b: 34.555, + c: vec!["cat".into(),"dog".into()] + }; + + let mut buf1: Vec = Vec::new(); + let mut buf2: Vec = Vec::new(); + let mut buf3: Vec = Vec::new(); + + let mut generic_writer = GenericSingleObjectWriter::new_with_capacity(&TestSingleObjectWriter::get_schema(), 1024).expect("Should resolve schema"); + let mut specifc_writer = SingleObjectWriter::::with_capacity(1024).expect("Resolved should pass"); + specifc_writer.write(obj1.clone(), &mut buf1).expect("Serialization expected"); + specifc_writer.write_value(obj1.clone(), &mut buf2).expect("Serialization expected"); + generic_writer.write_value(obj1.clone().into(), &mut buf3).expect("Serialization expected"); + assert_eq!(buf1, buf2); + assert_eq!(buf1, buf3); + } } From a89010df95e3ba13020c6d2470152fa8a25443c9 Mon Sep 17 00:00:00 2001 From: Jack Klamer Date: Sat, 23 Apr 2022 16:20:09 -0500 Subject: [PATCH 03/66] fmt --- .../examples/test_interop_message_data.rs | 6 +- lang/rust/avro/src/encode.rs | 12 +- lang/rust/avro/src/types.rs | 9 +- lang/rust/avro/src/writer.rs | 146 +++++++++++------- 4 files changed, 106 insertions(+), 67 deletions(-) diff --git a/lang/rust/avro/examples/test_interop_message_data.rs b/lang/rust/avro/examples/test_interop_message_data.rs index 0b50d0284f8..f328e4d9d04 100644 --- a/lang/rust/avro/examples/test_interop_message_data.rs +++ b/lang/rust/avro/examples/test_interop_message_data.rs @@ -1,5 +1 @@ - - -fn main() { - -} \ No newline at end of file +fn main() {} diff --git a/lang/rust/avro/src/encode.rs b/lang/rust/avro/src/encode.rs index 84e665f41c1..bd69ba50100 100644 --- a/lang/rust/avro/src/encode.rs +++ b/lang/rust/avro/src/encode.rs @@ -16,12 +16,16 @@ // under the License. use crate::{ - schema::{Namespace, ResolvedSchema, Schema, SchemaKind, Name}, + schema::{Name, Namespace, ResolvedSchema, Schema, SchemaKind}, types::{Value, ValueKind}, util::{zig_i32, zig_i64}, AvroResult, Error, }; -use std::{convert::{TryFrom, TryInto}, borrow::Borrow, collections::HashMap}; +use std::{ + borrow::Borrow, + collections::HashMap, + convert::{TryFrom, TryInto}, +}; /// Encode a `Value` into avro format. /// @@ -47,10 +51,10 @@ fn encode_int(i: i32, buffer: &mut Vec) { zig_i32(i, buffer) } -pub(crate) fn encode_internal< S :Borrow>( +pub(crate) fn encode_internal>( value: &Value, schema: &Schema, - names: &HashMap, + names: &HashMap, enclosing_namespace: &Namespace, buffer: &mut Vec, ) -> AvroResult<()> { diff --git a/lang/rust/avro/src/types.rs b/lang/rust/avro/src/types.rs index 251b44c426d..4f7f13f8254 100644 --- a/lang/rust/avro/src/types.rs +++ b/lang/rust/avro/src/types.rs @@ -20,7 +20,8 @@ use crate::{ decimal::Decimal, duration::Duration, schema::{ - NamesRef, Precision, RecordField, ResolvedSchema, Scale, Schema, SchemaKind, UnionSchema, Name, + Name, NamesRef, Precision, RecordField, ResolvedSchema, Scale, Schema, SchemaKind, + UnionSchema, }, AvroResult, Error, }; @@ -355,7 +356,11 @@ impl Value { } } - pub(crate) fn validate_internal>(&self, schema: &Schema, names: &HashMap) -> Option { + pub(crate) fn validate_internal>( + &self, + schema: &Schema, + names: &HashMap, + ) -> Option { match (self, schema) { (_, &Schema::Ref { ref name }) => names.get(name).map_or_else( || { diff --git a/lang/rust/avro/src/writer.rs b/lang/rust/avro/src/writer.rs index 99c3a8a1690..4b8604fd7fc 100644 --- a/lang/rust/avro/src/writer.rs +++ b/lang/rust/avro/src/writer.rs @@ -18,10 +18,11 @@ //! Logic handling writing in Avro format at user level. use crate::{ encode::{encode, encode_internal, encode_to_vec}, - schema::{ResolvedSchema, Schema, AvroSchema, ResolvedOwnedSchema}, + rabin::Rabin, + schema::{AvroSchema, ResolvedOwnedSchema, ResolvedSchema, Schema}, ser::Serializer, types::Value, - AvroResult, Codec, Error, rabin::Rabin, + AvroResult, Codec, Error, }; use rand::random; use serde::Serialize; @@ -355,40 +356,44 @@ fn write_avro_datum>( /// Writer that encodes messages according to the single object encoding v1 spec /// Uses an API similar to the current File Writer /// Writes all object bytes at once, and drains internal buffer -pub struct GenericSingleObjectWriter -{ +pub struct GenericSingleObjectWriter { buffer: Vec, resolved: ResolvedOwnedSchema, } -impl GenericSingleObjectWriter -{ - pub fn new_with_capacity(schema: &Schema, initial_buffer_cap: usize) -> AvroResult { +impl GenericSingleObjectWriter { + pub fn new_with_capacity( + schema: &Schema, + initial_buffer_cap: usize, + ) -> AvroResult { let fingerprint = schema.fingerprint::(); - let mut buffer = Vec::with_capacity(initial_buffer_cap); - let header =[0xC3, 0x01, - fingerprint.bytes[0], - fingerprint.bytes[1], - fingerprint.bytes[2], - fingerprint.bytes[3], - fingerprint.bytes[4], - fingerprint.bytes[5], - fingerprint.bytes[6], - fingerprint.bytes[7]]; + let mut buffer = Vec::with_capacity(initial_buffer_cap); + let header = [ + 0xC3, + 0x01, + fingerprint.bytes[0], + fingerprint.bytes[1], + fingerprint.bytes[2], + fingerprint.bytes[3], + fingerprint.bytes[4], + fingerprint.bytes[5], + fingerprint.bytes[6], + fingerprint.bytes[7], + ]; buffer.extend_from_slice(&header); - Ok(GenericSingleObjectWriter{ + Ok(GenericSingleObjectWriter { buffer, resolved: ResolvedOwnedSchema::try_from(schema.clone())?, }) } /// Wrtite the referenced Value to the provided Write object. Returns a result with the number of bytes writtern including the header - pub fn write_value_ref(&mut self, v: &Value, writer: &mut W) -> AvroResult{ + pub fn write_value_ref(&mut self, v: &Value, writer: &mut W) -> AvroResult { if self.buffer.len() != 10 { Err(Error::IllegalSingleObjectWriterState) } else { - write_value_ref_owned_resolved( &self.resolved, v, &mut self.buffer)?; + write_value_ref_owned_resolved(&self.resolved, v, &mut self.buffer)?; writer.write_all(&self.buffer).map_err(Error::WriteBytes)?; let len = self.buffer.len(); self.buffer.truncate(10); @@ -397,58 +402,57 @@ impl GenericSingleObjectWriter } /// Wrtite the Value to the provided Write object. Returns a result with the number of bytes writtern including the header - pub fn write_value(&mut self, v: Value, writer: &mut W) -> AvroResult{ + pub fn write_value(&mut self, v: Value, writer: &mut W) -> AvroResult { self.write_value_ref(&v, writer) } } /// Writer that encodes messages according to the single object encoding v1 spec -pub struct SingleObjectWriter +pub struct SingleObjectWriter where - T: AvroSchema -{ + T: AvroSchema, +{ inner: GenericSingleObjectWriter, _model: PhantomData, } -impl SingleObjectWriter +impl SingleObjectWriter where - T: AvroSchema + T: AvroSchema, { pub fn with_capacity(buffer_cap: usize) -> AvroResult> { let schema = T::get_schema(); - Ok(SingleObjectWriter{ - inner: GenericSingleObjectWriter::new_with_capacity(&schema, buffer_cap)?, + Ok(SingleObjectWriter { + inner: GenericSingleObjectWriter::new_with_capacity(&schema, buffer_cap)?, _model: PhantomData, }) } } - -impl SingleObjectWriter +impl SingleObjectWriter where - T: AvroSchema + Into + T: AvroSchema + Into, { /// Wrtite the Into to the provided Write object. Returns a result with the number of bytes writtern including the header - pub fn write_value(&mut self, data: T, writer: &mut W) -> AvroResult { + pub fn write_value(&mut self, data: T, writer: &mut W) -> AvroResult { let v: Value = data.into(); self.inner.write_value_ref(&v, writer) } } -impl SingleObjectWriter +impl SingleObjectWriter where - T: AvroSchema + Serialize + T: AvroSchema + Serialize, { - /// Wrtite the Serialize object to the provided Write object. Returns a result with the number of bytes writtern including the header - pub fn write_ref(&mut self, data: &T, writer: &mut W) -> AvroResult { + /// Wrtite the referenced Serialize object to the provided Write object. Returns a result with the number of bytes writtern including the header + pub fn write_ref(&mut self, data: &T, writer: &mut W) -> AvroResult { let mut serializer = Serializer::default(); let v = data.serialize(&mut serializer)?; - self.inner.write_value_ref(&v, writer) + self.inner.write_value_ref(&v, writer) } /// Wrtite the Serialize object to the provided Write object. Returns a result with the number of bytes writtern including the header - pub fn write(&mut self, data: T, writer: &mut W) -> AvroResult { + pub fn write(&mut self, data: T, writer: &mut W) -> AvroResult { self.write_ref(&data, writer) } } @@ -1070,7 +1074,7 @@ mod tests { struct TestSingleObjectWriter { a: i64, b: f64, - c: Vec + c: Vec, } impl AvroSchema for TestSingleObjectWriter { @@ -1107,7 +1111,10 @@ mod tests { Value::Record(vec![ ("a".into(), self.a.into()), ("b".into(), self.b.into()), - ("c".into(), Value::Array(self.c.into_iter().map(|s| s.into()).collect())) + ( + "c".into(), + Value::Array(self.c.into_iter().map(|s| s.into()).collect()), + ), ]) } } @@ -1115,42 +1122,69 @@ mod tests { #[test] fn test_single_object_writer() { let mut buf: Vec = Vec::new(); - let obj = TestSingleObjectWriter{ + let obj = TestSingleObjectWriter { a: 300, b: 34.555, - c: vec!["cat".into(),"dog".into()] + c: vec!["cat".into(), "dog".into()], }; - let mut writer = GenericSingleObjectWriter::new_with_capacity(&TestSingleObjectWriter::get_schema(), 1024).expect("Should resolve schema"); + let mut writer = GenericSingleObjectWriter::new_with_capacity( + &TestSingleObjectWriter::get_schema(), + 1024, + ) + .expect("Should resolve schema"); let value = obj.into(); - let written_bytes = writer.write_value_ref(&value, &mut buf).expect("Error serializing properly"); - + let written_bytes = writer + .write_value_ref(&value, &mut buf) + .expect("Error serializing properly"); + assert!(buf.len() > 10, "no bytes written"); assert_eq!(buf.len(), written_bytes); - assert_eq!(buf[0] , 0xC3); - assert_eq!(buf[1] , 0x01); - assert_eq!(&buf[2..10], &TestSingleObjectWriter::get_schema().fingerprint::().bytes[..]); + assert_eq!(buf[0], 0xC3); + assert_eq!(buf[1], 0x01); + assert_eq!( + &buf[2..10], + &TestSingleObjectWriter::get_schema() + .fingerprint::() + .bytes[..] + ); let mut msg_binary = Vec::new(); - encode(&value, &TestSingleObjectWriter::get_schema(), &mut msg_binary).expect("encode should have failed by here as a depndency of any writing"); + encode( + &value, + &TestSingleObjectWriter::get_schema(), + &mut msg_binary, + ) + .expect("encode should have failed by here as a depndency of any writing"); assert_eq!(&buf[10..], &msg_binary[..]) } #[test] fn test_writer_parity() { - let obj1 = TestSingleObjectWriter{ + let obj1 = TestSingleObjectWriter { a: 300, b: 34.555, - c: vec!["cat".into(),"dog".into()] + c: vec!["cat".into(), "dog".into()], }; let mut buf1: Vec = Vec::new(); let mut buf2: Vec = Vec::new(); let mut buf3: Vec = Vec::new(); - let mut generic_writer = GenericSingleObjectWriter::new_with_capacity(&TestSingleObjectWriter::get_schema(), 1024).expect("Should resolve schema"); - let mut specifc_writer = SingleObjectWriter::::with_capacity(1024).expect("Resolved should pass"); - specifc_writer.write(obj1.clone(), &mut buf1).expect("Serialization expected"); - specifc_writer.write_value(obj1.clone(), &mut buf2).expect("Serialization expected"); - generic_writer.write_value(obj1.clone().into(), &mut buf3).expect("Serialization expected"); + let mut generic_writer = GenericSingleObjectWriter::new_with_capacity( + &TestSingleObjectWriter::get_schema(), + 1024, + ) + .expect("Should resolve schema"); + let mut specifc_writer = SingleObjectWriter::::with_capacity(1024) + .expect("Resolved should pass"); + specifc_writer + .write(obj1.clone(), &mut buf1) + .expect("Serialization expected"); + specifc_writer + .write_value(obj1.clone(), &mut buf2) + .expect("Serialization expected"); + generic_writer + .write_value(obj1.clone().into(), &mut buf3) + .expect("Serialization expected"); assert_eq!(buf1, buf2); assert_eq!(buf1, buf3); } From a67a35c6b1da4ab9d5de2de173a988a2e6e2e71c Mon Sep 17 00:00:00 2001 From: Jack Klamer Date: Sat, 23 Apr 2022 17:37:52 -0500 Subject: [PATCH 04/66] Interop tested --- ...pData.java => TestInteropMessageData.java} | 16 ++++--- .../examples/test_interop_message_data.rs | 31 +++++++++++- lang/rust/avro/src/lib.rs | 2 +- share/test/data/README.md | 45 ++++++++++++++++++ share/test/data/messageV1/test_message.bin | Bin 38 -> 38 bytes share/test/data/messageV1/test_message.json | 5 ++ share/test/data/test_schema.json | 22 +++++++++ 7 files changed, 113 insertions(+), 8 deletions(-) rename lang/java/avro/src/test/java/org/apache/avro/message/{TestInteropData.java => TestInteropMessageData.java} (69%) create mode 100644 share/test/data/README.md create mode 100644 share/test/data/messageV1/test_message.json create mode 100644 share/test/data/test_schema.json diff --git a/lang/java/avro/src/test/java/org/apache/avro/message/TestInteropData.java b/lang/java/avro/src/test/java/org/apache/avro/message/TestInteropMessageData.java similarity index 69% rename from lang/java/avro/src/test/java/org/apache/avro/message/TestInteropData.java rename to lang/java/avro/src/test/java/org/apache/avro/message/TestInteropMessageData.java index c7489c1a4ec..98f5e96aed9 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/message/TestInteropData.java +++ b/lang/java/avro/src/test/java/org/apache/avro/message/TestInteropMessageData.java @@ -3,16 +3,20 @@ import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecordBuilder; +import org.junit.Assert; import org.junit.Test; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; +import java.io.FileReader; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.file.Files; import java.util.Arrays; +import java.util.logging.Logger; -public class TestInteropData { +public class TestInteropMessageData { private static String inDir = System.getProperty("share.dir", "../../../share") + "/test/data/messageV1"; private static File SCHEMA_FILE = new File(inDir + "/test_schema.json"); private static File MESSAGE_FILE = new File(inDir + "/test_message.bin"); @@ -29,11 +33,11 @@ public class TestInteropData { } @Test - public void generateData() throws IOException { + public void sanity_check() throws IOException { MessageEncoder encoder = new BinaryMessageEncoder<>(GenericData.get(), SCHEMA); - BUILDER.set("id", 5L).set("name", "Bill").set("tags", Arrays.asList("dog_lover", "cat_hater")).build(); - ByteBuffer buffer = encoder - .encode(BUILDER.set("id", 5L).set("name", "Bill").set("tags", Arrays.asList("dog_lover", "cat_hater")).build()); - new FileOutputStream(MESSAGE_FILE).write(buffer.array()); + ByteBuffer buffer = encoder.encode( + BUILDER.set("id", 42L).set("name", "Bill").set("tags", Arrays.asList("dog_lover", "cat_hater")).build()); + byte[] fileBuffer = Files.readAllBytes(MESSAGE_FILE.toPath()); + Assert.assertArrayEquals(fileBuffer, buffer.array()); } } diff --git a/lang/rust/avro/examples/test_interop_message_data.rs b/lang/rust/avro/examples/test_interop_message_data.rs index f328e4d9d04..9840af63f66 100644 --- a/lang/rust/avro/examples/test_interop_message_data.rs +++ b/lang/rust/avro/examples/test_interop_message_data.rs @@ -1 +1,30 @@ -fn main() {} +use apache_avro::{schema::AvroSchema, types::Value}; + + + +struct InteropMessage; + + +impl AvroSchema for InteropMessage { + fn get_schema() -> apache_avro::Schema { + let schema = std::fs::read_to_string("../../share/test/data/messageV1/test_schema.json").expect("File should exist with schema inside"); + apache_avro::Schema::parse_str(schema.as_str()).expect("File should exist with schema inside") + } +} + +impl Into for InteropMessage { + fn into(self) -> Value { + Value::Record(vec![ + ("id".into(), 42i64.into()), + ("name".into(), "Bill".into()), + ("tags".into(), Value::Array(vec!["dog_lover", "cat_hater"].into_iter().map(|s| s.into()).collect())) + ]) + } +} + +fn main() { + let file_message = std::fs::read("../../share/test/data/messageV1/test_message.bin").expect("File not found or error reading"); + let mut generated_encoding: Vec = Vec::new(); + apache_avro::SingleObjectWriter::::with_capacity(1024).expect("resolve expected").write_value(InteropMessage, &mut generated_encoding).expect("Should encode"); + assert_eq!(file_message, generated_encoding) +} diff --git a/lang/rust/avro/src/lib.rs b/lang/rust/avro/src/lib.rs index 8b271d60337..83525ed2b8c 100644 --- a/lang/rust/avro/src/lib.rs +++ b/lang/rust/avro/src/lib.rs @@ -746,7 +746,7 @@ pub use reader::{from_avro_datum, Reader}; pub use schema::Schema; pub use ser::to_value; pub use util::max_allocation_bytes; -pub use writer::{to_avro_datum, Writer}; +pub use writer::{to_avro_datum, Writer, SingleObjectWriter, GenericSingleObjectWriter}; #[macro_use] extern crate log; diff --git a/share/test/data/README.md b/share/test/data/README.md new file mode 100644 index 00000000000..74f8806f597 --- /dev/null +++ b/share/test/data/README.md @@ -0,0 +1,45 @@ +BinaryMessage data in single object encoding https://avro.apache.org/docs/current/spec.html#single_object_encoding + +Ground truth data generated with Java Code + +The binary data will be the V1 sincle object encoding with the schema of +``` +{ + "type":"record", + "namespace":"org.apache.avro", + "name":"TestMessage", + "fields":[ + { + "name":"id", + "type":"long" + }, + { + "name":"name", + "type":"string" + }, + { + "name":"tags", + "type":{ + "type":"array", + "items":"string" + } + }, + { + "name":"scores", + "type":{ + "type":"map", + "values":"double" + } + } + ] +} +``` + +The sample binary message will have the values equal to the json serialized version of the record shown below +``` +{ + "id": 42, + "name": "Bill", + "tags": ["dog_lover", "cat_hater"] +} +``` \ No newline at end of file diff --git a/share/test/data/messageV1/test_message.bin b/share/test/data/messageV1/test_message.bin index 025d46151de381840b61ddc0679cccda9d98b9ef..609337eb91464ddf773f25c182a092b72e0ab08e 100644 GIT binary patch delta 17 ZcmY#W<37x|QupDHMNVIBLnd-d0{}d+2J8R; delta 17 ZcmY#W<37x|QupDHMNVIBxh8T;0{}aT2BH7} diff --git a/share/test/data/messageV1/test_message.json b/share/test/data/messageV1/test_message.json new file mode 100644 index 00000000000..e0c2b0d81ed --- /dev/null +++ b/share/test/data/messageV1/test_message.json @@ -0,0 +1,5 @@ +{ + "id": 42, + "name": "Bill", + "tags": ["dog_lover", "cat_hater"] +} \ No newline at end of file diff --git a/share/test/data/test_schema.json b/share/test/data/test_schema.json new file mode 100644 index 00000000000..cd49b72a456 --- /dev/null +++ b/share/test/data/test_schema.json @@ -0,0 +1,22 @@ +{ + "type":"record", + "namespace":"org.apache.avro", + "name":"TestMessage", + "fields":[ + { + "name":"id", + "type":"long" + }, + { + "name":"name", + "type":"string" + }, + { + "name":"tags", + "type":{ + "type":"array", + "items":"string" + } + } + ] +} \ No newline at end of file From d955204353861b4e380de4f1d8048cbbc7dbfd1d Mon Sep 17 00:00:00 2001 From: Jack Klamer Date: Sat, 23 Apr 2022 17:38:30 -0500 Subject: [PATCH 05/66] uneed file --- share/test/data/test_schema.json | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 share/test/data/test_schema.json diff --git a/share/test/data/test_schema.json b/share/test/data/test_schema.json deleted file mode 100644 index cd49b72a456..00000000000 --- a/share/test/data/test_schema.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "type":"record", - "namespace":"org.apache.avro", - "name":"TestMessage", - "fields":[ - { - "name":"id", - "type":"long" - }, - { - "name":"name", - "type":"string" - }, - { - "name":"tags", - "type":{ - "type":"array", - "items":"string" - } - } - ] -} \ No newline at end of file From 8141d2e720605be21503859176bc0aaf95e02fb6 Mon Sep 17 00:00:00 2001 From: Jack Klamer Date: Sat, 23 Apr 2022 17:44:24 -0500 Subject: [PATCH 06/66] remove bugs --- lang/rust/avro/src/ser.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lang/rust/avro/src/ser.rs b/lang/rust/avro/src/ser.rs index 9520ef69cbd..5cff13e5666 100644 --- a/lang/rust/avro/src/ser.rs +++ b/lang/rust/avro/src/ser.rs @@ -178,7 +178,7 @@ impl<'b> ser::Serializer for &'b mut Serializer { } fn serialize_none(self) -> Result { - Ok(Value::Null) + Ok(Value::from(None::)) } fn serialize_some(self, value: &T) -> Result @@ -186,7 +186,7 @@ impl<'b> ser::Serializer for &'b mut Serializer { T: Serialize, { let v = value.serialize(&mut Serializer::default())?; - Ok(v) + Ok(Value::from(Some(v))) } fn serialize_unit(self) -> Result { From 6821a024fef1593fd5d7b7de0179d23863249dc3 Mon Sep 17 00:00:00 2001 From: Jack Klamer Date: Sat, 23 Apr 2022 17:49:42 -0500 Subject: [PATCH 07/66] clippy --- .../examples/test_interop_message_data.rs | 33 ++++++++++++------- lang/rust/avro/src/lib.rs | 2 +- lang/rust/avro/src/writer.rs | 12 +++---- 3 files changed, 29 insertions(+), 18 deletions(-) diff --git a/lang/rust/avro/examples/test_interop_message_data.rs b/lang/rust/avro/examples/test_interop_message_data.rs index 9840af63f66..e3949214ff2 100644 --- a/lang/rust/avro/examples/test_interop_message_data.rs +++ b/lang/rust/avro/examples/test_interop_message_data.rs @@ -1,30 +1,41 @@ use apache_avro::{schema::AvroSchema, types::Value}; - - struct InteropMessage; - impl AvroSchema for InteropMessage { - fn get_schema() -> apache_avro::Schema { - let schema = std::fs::read_to_string("../../share/test/data/messageV1/test_schema.json").expect("File should exist with schema inside"); - apache_avro::Schema::parse_str(schema.as_str()).expect("File should exist with schema inside") + fn get_schema() -> apache_avro::Schema { + let schema = std::fs::read_to_string("../../share/test/data/messageV1/test_schema.json") + .expect("File should exist with schema inside"); + apache_avro::Schema::parse_str(schema.as_str()) + .expect("File should exist with schema inside") } } -impl Into for InteropMessage { - fn into(self) -> Value { +impl From for Value { + fn from(_: InteropMessage) -> Value { Value::Record(vec![ ("id".into(), 42i64.into()), ("name".into(), "Bill".into()), - ("tags".into(), Value::Array(vec!["dog_lover", "cat_hater"].into_iter().map(|s| s.into()).collect())) + ( + "tags".into(), + Value::Array( + vec!["dog_lover", "cat_hater"] + .into_iter() + .map(|s| s.into()) + .collect(), + ), + ), ]) } } fn main() { - let file_message = std::fs::read("../../share/test/data/messageV1/test_message.bin").expect("File not found or error reading"); + let file_message = std::fs::read("../../share/test/data/messageV1/test_message.bin") + .expect("File not found or error reading"); let mut generated_encoding: Vec = Vec::new(); - apache_avro::SingleObjectWriter::::with_capacity(1024).expect("resolve expected").write_value(InteropMessage, &mut generated_encoding).expect("Should encode"); + apache_avro::SingleObjectWriter::::with_capacity(1024) + .expect("resolve expected") + .write_value(InteropMessage, &mut generated_encoding) + .expect("Should encode"); assert_eq!(file_message, generated_encoding) } diff --git a/lang/rust/avro/src/lib.rs b/lang/rust/avro/src/lib.rs index 83525ed2b8c..ba9ec3a3194 100644 --- a/lang/rust/avro/src/lib.rs +++ b/lang/rust/avro/src/lib.rs @@ -746,7 +746,7 @@ pub use reader::{from_avro_datum, Reader}; pub use schema::Schema; pub use ser::to_value; pub use util::max_allocation_bytes; -pub use writer::{to_avro_datum, Writer, SingleObjectWriter, GenericSingleObjectWriter}; +pub use writer::{to_avro_datum, GenericSingleObjectWriter, SingleObjectWriter, Writer}; #[macro_use] extern crate log; diff --git a/lang/rust/avro/src/writer.rs b/lang/rust/avro/src/writer.rs index 4b8604fd7fc..e3389fe4aaa 100644 --- a/lang/rust/avro/src/writer.rs +++ b/lang/rust/avro/src/writer.rs @@ -1106,14 +1106,14 @@ mod tests { } } - impl Into for TestSingleObjectWriter { - fn into(self) -> Value { + impl From for Value { + fn from(obj: TestSingleObjectWriter) -> Value { Value::Record(vec![ - ("a".into(), self.a.into()), - ("b".into(), self.b.into()), + ("a".into(), obj.a.into()), + ("b".into(), obj.b.into()), ( "c".into(), - Value::Array(self.c.into_iter().map(|s| s.into()).collect()), + Value::Array(obj.c.into_iter().map(|s| s.into()).collect()), ), ]) } @@ -1183,7 +1183,7 @@ mod tests { .write_value(obj1.clone(), &mut buf2) .expect("Serialization expected"); generic_writer - .write_value(obj1.clone().into(), &mut buf3) + .write_value(obj1.into(), &mut buf3) .expect("Serialization expected"); assert_eq!(buf1, buf2); assert_eq!(buf1, buf3); From de03d26ac588bc9c7fe184640d677e2cb2794d93 Mon Sep 17 00:00:00 2001 From: Jack Klamer Date: Mon, 25 Apr 2022 21:43:05 -0500 Subject: [PATCH 08/66] fix README --- share/test/data/README.md | 45 ----------------------------- share/test/data/messageV1/README.md | 9 ++++++ 2 files changed, 9 insertions(+), 45 deletions(-) delete mode 100644 share/test/data/README.md diff --git a/share/test/data/README.md b/share/test/data/README.md deleted file mode 100644 index 74f8806f597..00000000000 --- a/share/test/data/README.md +++ /dev/null @@ -1,45 +0,0 @@ -BinaryMessage data in single object encoding https://avro.apache.org/docs/current/spec.html#single_object_encoding - -Ground truth data generated with Java Code - -The binary data will be the V1 sincle object encoding with the schema of -``` -{ - "type":"record", - "namespace":"org.apache.avro", - "name":"TestMessage", - "fields":[ - { - "name":"id", - "type":"long" - }, - { - "name":"name", - "type":"string" - }, - { - "name":"tags", - "type":{ - "type":"array", - "items":"string" - } - }, - { - "name":"scores", - "type":{ - "type":"map", - "values":"double" - } - } - ] -} -``` - -The sample binary message will have the values equal to the json serialized version of the record shown below -``` -{ - "id": 42, - "name": "Bill", - "tags": ["dog_lover", "cat_hater"] -} -``` \ No newline at end of file diff --git a/share/test/data/messageV1/README.md b/share/test/data/messageV1/README.md index 048815e78fd..0d0ec2b20fc 100644 --- a/share/test/data/messageV1/README.md +++ b/share/test/data/messageV1/README.md @@ -33,4 +33,13 @@ The binary data will be the V1 sincle object encoding with the schema of } ] } +``` + +The sample binary message will have the values equal to the json serialized version of the record shown below +``` +{ + "id": 42, + "name": "Bill", + "tags": ["dog_lover", "cat_hater"] +} ``` \ No newline at end of file From b0e3e224d489333e2e6e07aa5eed48c992e6b61c Mon Sep 17 00:00:00 2001 From: Jack Klamer Date: Mon, 25 Apr 2022 21:46:27 -0500 Subject: [PATCH 09/66] rat fix --- .../avro/message/TestInteropMessageData.java | 19 +++++++++++++++++++ .../examples/test_interop_message_data.rs | 17 +++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/lang/java/avro/src/test/java/org/apache/avro/message/TestInteropMessageData.java b/lang/java/avro/src/test/java/org/apache/avro/message/TestInteropMessageData.java index 98f5e96aed9..db2b3ace129 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/message/TestInteropMessageData.java +++ b/lang/java/avro/src/test/java/org/apache/avro/message/TestInteropMessageData.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.avro.message; import org.apache.avro.Schema; diff --git a/lang/rust/avro/examples/test_interop_message_data.rs b/lang/rust/avro/examples/test_interop_message_data.rs index e3949214ff2..9fc45c89d3e 100644 --- a/lang/rust/avro/examples/test_interop_message_data.rs +++ b/lang/rust/avro/examples/test_interop_message_data.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use apache_avro::{schema::AvroSchema, types::Value}; struct InteropMessage; From 15b91c5f7405b8771494819bc32ee54ef86c9a31 Mon Sep 17 00:00:00 2001 From: Jack Klamer Date: Sun, 1 May 2022 11:59:07 -0500 Subject: [PATCH 10/66] Update lang/rust/avro/src/writer.rs Co-authored-by: Martin Grigorov --- lang/rust/avro/src/writer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lang/rust/avro/src/writer.rs b/lang/rust/avro/src/writer.rs index e3389fe4aaa..a3c14493fff 100644 --- a/lang/rust/avro/src/writer.rs +++ b/lang/rust/avro/src/writer.rs @@ -388,7 +388,7 @@ impl GenericSingleObjectWriter { }) } - /// Wrtite the referenced Value to the provided Write object. Returns a result with the number of bytes writtern including the header + /// Write the referenced Value to the provided Write object. Returns a result with the number of bytes written including the header pub fn write_value_ref(&mut self, v: &Value, writer: &mut W) -> AvroResult { if self.buffer.len() != 10 { Err(Error::IllegalSingleObjectWriterState) From cf9f39472efc17ec2ba1bfc6a4b11f42e3273a3f Mon Sep 17 00:00:00 2001 From: Jack Klamer Date: Sun, 1 May 2022 11:59:14 -0500 Subject: [PATCH 11/66] Update lang/rust/avro/src/writer.rs Co-authored-by: Martin Grigorov --- lang/rust/avro/src/writer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lang/rust/avro/src/writer.rs b/lang/rust/avro/src/writer.rs index a3c14493fff..e18246d630e 100644 --- a/lang/rust/avro/src/writer.rs +++ b/lang/rust/avro/src/writer.rs @@ -433,7 +433,7 @@ impl SingleObjectWriter where T: AvroSchema + Into, { - /// Wrtite the Into to the provided Write object. Returns a result with the number of bytes writtern including the header + /// Write the Into to the provided Write object. Returns a result with the number of bytes written including the header pub fn write_value(&mut self, data: T, writer: &mut W) -> AvroResult { let v: Value = data.into(); self.inner.write_value_ref(&v, writer) From b4663b33d066491c8a2924c55339e0e749a0beed Mon Sep 17 00:00:00 2001 From: Jack Klamer Date: Sun, 1 May 2022 11:59:31 -0500 Subject: [PATCH 12/66] Update lang/rust/avro/src/writer.rs Co-authored-by: Martin Grigorov --- lang/rust/avro/src/writer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lang/rust/avro/src/writer.rs b/lang/rust/avro/src/writer.rs index e18246d630e..1172bce5a80 100644 --- a/lang/rust/avro/src/writer.rs +++ b/lang/rust/avro/src/writer.rs @@ -401,7 +401,7 @@ impl GenericSingleObjectWriter { } } - /// Wrtite the Value to the provided Write object. Returns a result with the number of bytes writtern including the header + /// Write the Value to the provided Write object. Returns a result with the number of bytes written including the header pub fn write_value(&mut self, v: Value, writer: &mut W) -> AvroResult { self.write_value_ref(&v, writer) } From 2b91fcf2e44a91603b38d7e3df9efce79996ecdd Mon Sep 17 00:00:00 2001 From: Jack Klamer Date: Sun, 1 May 2022 11:59:42 -0500 Subject: [PATCH 13/66] Update lang/rust/avro/src/writer.rs Co-authored-by: Martin Grigorov --- lang/rust/avro/src/writer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lang/rust/avro/src/writer.rs b/lang/rust/avro/src/writer.rs index 1172bce5a80..8f023f90880 100644 --- a/lang/rust/avro/src/writer.rs +++ b/lang/rust/avro/src/writer.rs @@ -444,7 +444,7 @@ impl SingleObjectWriter where T: AvroSchema + Serialize, { - /// Wrtite the referenced Serialize object to the provided Write object. Returns a result with the number of bytes writtern including the header + /// Write the referenced Serialize object to the provided Write object. Returns a result with the number of bytes written including the header pub fn write_ref(&mut self, data: &T, writer: &mut W) -> AvroResult { let mut serializer = Serializer::default(); let v = data.serialize(&mut serializer)?; From 061b89d1aec2f00ede45a7cf669eab32f076971e Mon Sep 17 00:00:00 2001 From: Jack Klamer Date: Sun, 1 May 2022 11:59:51 -0500 Subject: [PATCH 14/66] Update lang/rust/avro/src/writer.rs Co-authored-by: Martin Grigorov --- lang/rust/avro/src/writer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lang/rust/avro/src/writer.rs b/lang/rust/avro/src/writer.rs index 8f023f90880..c313e1667d5 100644 --- a/lang/rust/avro/src/writer.rs +++ b/lang/rust/avro/src/writer.rs @@ -1153,7 +1153,7 @@ mod tests { &TestSingleObjectWriter::get_schema(), &mut msg_binary, ) - .expect("encode should have failed by here as a depndency of any writing"); + .expect("encode should have failed by here as a dependency of any writing"); assert_eq!(&buf[10..], &msg_binary[..]) } From 8a971d21ff54f7d99e262104e28be79f8dfa90cb Mon Sep 17 00:00:00 2001 From: Jack Klamer Date: Sun, 1 May 2022 12:32:56 -0500 Subject: [PATCH 15/66] PR changes --- .../avro/message/TestInteropMessageData.java | 26 ++++++++++--------- .../examples/test_interop_message_data.rs | 2 +- .../{test_schema.json => test_schema.avsc} | 0 3 files changed, 15 insertions(+), 13 deletions(-) rename share/test/data/messageV1/{test_schema.json => test_schema.avsc} (100%) diff --git a/lang/java/avro/src/test/java/org/apache/avro/message/TestInteropMessageData.java b/lang/java/avro/src/test/java/org/apache/avro/message/TestInteropMessageData.java index db2b3ace129..6a658462d54 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/message/TestInteropMessageData.java +++ b/lang/java/avro/src/test/java/org/apache/avro/message/TestInteropMessageData.java @@ -22,7 +22,10 @@ import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecordBuilder; +import org.junit.AfterClass; import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import java.io.File; @@ -36,19 +39,18 @@ import java.util.logging.Logger; public class TestInteropMessageData { - private static String inDir = System.getProperty("share.dir", "../../../share") + "/test/data/messageV1"; - private static File SCHEMA_FILE = new File(inDir + "/test_schema.json"); - private static File MESSAGE_FILE = new File(inDir + "/test_message.bin"); - private static final Schema SCHEMA; - private static final GenericRecordBuilder BUILDER; + private static final String inDir = System.getProperty("share.dir", "../../../share") + "/test/data/messageV1"; + private static final File SCHEMA_FILE = new File(inDir + "/test_schema.avsc"); + private static final File MESSAGE_FILE = new File(inDir + "/test_message.bin"); + private static Schema SCHEMA; + private static GenericRecordBuilder BUILDER; - static { - try { - SCHEMA = new Schema.Parser().parse(new FileInputStream(SCHEMA_FILE)); - BUILDER = new GenericRecordBuilder(SCHEMA); - } catch (IOException e) { - throw new RuntimeException("Interop Message Data Schema not found"); - } + @BeforeClass + public void setup() throws IOException { + final FileInputStream fileInputStream = new FileInputStream(SCHEMA_FILE); + SCHEMA = new Schema.Parser().parse(fileInputStream); + BUILDER = new GenericRecordBuilder(SCHEMA); + fileInputStream.close(); } @Test diff --git a/lang/rust/avro/examples/test_interop_message_data.rs b/lang/rust/avro/examples/test_interop_message_data.rs index 9fc45c89d3e..0a72b25e47c 100644 --- a/lang/rust/avro/examples/test_interop_message_data.rs +++ b/lang/rust/avro/examples/test_interop_message_data.rs @@ -21,7 +21,7 @@ struct InteropMessage; impl AvroSchema for InteropMessage { fn get_schema() -> apache_avro::Schema { - let schema = std::fs::read_to_string("../../share/test/data/messageV1/test_schema.json") + let schema = std::fs::read_to_string("../../share/test/data/messageV1/test_schema.avsc") .expect("File should exist with schema inside"); apache_avro::Schema::parse_str(schema.as_str()) .expect("File should exist with schema inside") diff --git a/share/test/data/messageV1/test_schema.json b/share/test/data/messageV1/test_schema.avsc similarity index 100% rename from share/test/data/messageV1/test_schema.json rename to share/test/data/messageV1/test_schema.avsc From 75884332368394bf3aa556a825bae1900b5ff58e Mon Sep 17 00:00:00 2001 From: Jack Klamer Date: Sun, 1 May 2022 12:46:49 -0500 Subject: [PATCH 16/66] static setup --- .../java/org/apache/avro/message/TestInteropMessageData.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lang/java/avro/src/test/java/org/apache/avro/message/TestInteropMessageData.java b/lang/java/avro/src/test/java/org/apache/avro/message/TestInteropMessageData.java index 6a658462d54..0a8335e7eeb 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/message/TestInteropMessageData.java +++ b/lang/java/avro/src/test/java/org/apache/avro/message/TestInteropMessageData.java @@ -46,7 +46,7 @@ public class TestInteropMessageData { private static GenericRecordBuilder BUILDER; @BeforeClass - public void setup() throws IOException { + public static void setup() throws IOException { final FileInputStream fileInputStream = new FileInputStream(SCHEMA_FILE); SCHEMA = new Schema.Parser().parse(fileInputStream); BUILDER = new GenericRecordBuilder(SCHEMA); From 8041112ab28e2ce3dd7134d48e02d1626039ef99 Mon Sep 17 00:00:00 2001 From: Jack Klamer Date: Tue, 3 May 2022 21:31:44 -0500 Subject: [PATCH 17/66] Specific rename and interop test in script --- lang/rust/build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lang/rust/build.sh b/lang/rust/build.sh index 7b78acd5d5c..a1429b7d97f 100755 --- a/lang/rust/build.sh +++ b/lang/rust/build.sh @@ -60,11 +60,13 @@ do export RUST_LOG=apache_avro=debug export RUST_BACKTRACE=1 cargo run --all-features --example generate_interop_data + cargo run --all-features --example generate_interop_data ;; interop-data-test) prepare_build cargo run --all-features --example test_interop_data + cargo run --all-features --example test_interop_message_data ;; *) echo "Usage: $0 {lint|test|dist|clean|interop-data-generate|interop-data-test}" >&2 From 3aeceae3c4419513821aa668af4369ee085f6e36 Mon Sep 17 00:00:00 2001 From: Jack Klamer Date: Tue, 3 May 2022 21:57:33 -0500 Subject: [PATCH 18/66] typo --- share/test/data/messageV1/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/test/data/messageV1/README.md b/share/test/data/messageV1/README.md index 0d0ec2b20fc..e79f334c7e6 100644 --- a/share/test/data/messageV1/README.md +++ b/share/test/data/messageV1/README.md @@ -2,7 +2,7 @@ BinaryMessage data in single object encoding https://avro.apache.org/docs/curren Ground truth data generated with Java Code -The binary data will be the V1 sincle object encoding with the schema of +The binary data will be the V1 single object encoding with the schema of ``` { "type":"record", From c6ac5350bdee8b18198a6c22159082013450c6c6 Mon Sep 17 00:00:00 2001 From: Martin Grigorov Date: Mon, 18 Apr 2022 08:37:16 +0300 Subject: [PATCH 19/66] AVRO-3492: Add support for deriving Schema::Record aliases (#1647) * AVRO-3492: Add support for deriving Schema::Record aliases Uses Darling's 'multiple' attribute feature. Signed-off-by: Martin Tzvetanov Grigorov * AVRO-3492: Add a test case with multiple attributes with different values for 'alias' key Signed-off-by: Martin Tzvetanov Grigorov --- lang/rust/avro_derive/src/lib.rs | 16 +++++- lang/rust/avro_derive/tests/derive.rs | 74 +++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 1 deletion(-) diff --git a/lang/rust/avro_derive/src/lib.rs b/lang/rust/avro_derive/src/lib.rs index 96575e0901b..11ab7a61661 100644 --- a/lang/rust/avro_derive/src/lib.rs +++ b/lang/rust/avro_derive/src/lib.rs @@ -39,6 +39,8 @@ struct NamedTypeOptions { namespace: Option, #[darling(default)] doc: Option, + #[darling(multiple)] + alias: Vec, } #[proc_macro_derive(AvroSchema, attributes(avro))] @@ -64,6 +66,7 @@ fn derive_avro_schema(input: &mut DeriveInput) -> Result Result, + aliases: Vec, s: &syn::DataStruct, error_span: Span, ) -> Result> { @@ -143,6 +147,7 @@ fn get_data_struct_schema_def( } } let record_doc = preserve_optional(record_doc); + let record_aliases = preserve_vec(aliases); Ok(quote! { let schema_fields = vec![#(#record_field_exprs),*]; let name = apache_avro::schema::Name::new(#full_schema_name).expect(&format!("Unable to parse struct name for schema {}", #full_schema_name)[..]); @@ -152,7 +157,7 @@ fn get_data_struct_schema_def( .collect(); apache_avro::schema::Schema::Record { name, - aliases: None, + aliases: #record_aliases, doc: #record_doc, fields: schema_fields, lookup, @@ -277,6 +282,15 @@ fn preserve_optional(op: Option) -> TokenStream { } } +fn preserve_vec(op: Vec) -> TokenStream { + let items: Vec = op.iter().map(|tt| quote! {#tt.into()}).collect(); + if items.is_empty() { + quote! {None} + } else { + quote! {Some(vec![#(#items),*])} + } +} + fn darling_to_syn(e: darling::Error) -> Vec { let msg = format!("{}", e); let token_errors = e.write_errors(); diff --git a/lang/rust/avro_derive/tests/derive.rs b/lang/rust/avro_derive/tests/derive.rs index 8ac95755f4a..c058c44c10d 100644 --- a/lang/rust/avro_derive/tests/derive.rs +++ b/lang/rust/avro_derive/tests/derive.rs @@ -1063,4 +1063,78 @@ mod test_derive { serde_assert(TestBasicWithU32 { a: u32::MIN }); serde_assert(TestBasicWithU32 { a: 1_u32 }); } + + #[derive(Debug, Serialize, Deserialize, AvroSchema, Clone, PartialEq)] + #[avro(alias = "a", alias = "b", alias = "c")] + struct TestBasicWithAliases { + a: i32, + } + + #[test] + fn test_basic_with_aliases() { + let schema = r#" + { + "type":"record", + "name":"TestBasicWithAliases", + "aliases":["a", "b", "c"], + "fields":[ + { + "name":"a", + "type":"int" + } + ] + } + "#; + let schema = Schema::parse_str(schema).unwrap(); + if let Schema::Record { name, aliases, .. } = TestBasicWithAliases::get_schema() { + assert_eq!("TestBasicWithAliases", name.fullname(None)); + assert_eq!( + Some(vec!["a".to_owned(), "b".to_owned(), "c".to_owned()]), + aliases + ); + } else { + panic!("TestBasicWithAliases schema must be a record schema") + } + assert_eq!(schema, TestBasicWithAliases::get_schema()); + + serde_assert(TestBasicWithAliases { a: i32::MAX }); + } + + #[derive(Debug, Serialize, Deserialize, AvroSchema, Clone, PartialEq)] + #[avro(alias = "d")] + #[avro(alias = "e")] + #[avro(alias = "f")] + struct TestBasicWithAliases2 { + a: i32, + } + + #[test] + fn test_basic_with_aliases2() { + let schema = r#" + { + "type":"record", + "name":"TestBasicWithAliases2", + "aliases":["d", "e", "f"], + "fields":[ + { + "name":"a", + "type":"int" + } + ] + } + "#; + let schema = Schema::parse_str(schema).unwrap(); + if let Schema::Record { name, aliases, .. } = TestBasicWithAliases2::get_schema() { + assert_eq!("TestBasicWithAliases2", name.fullname(None)); + assert_eq!( + Some(vec!["d".to_owned(), "e".to_owned(), "f".to_owned()]), + aliases + ); + } else { + panic!("TestBasicWithAliases2 schema must be a record schema") + } + assert_eq!(schema, TestBasicWithAliases2::get_schema()); + + serde_assert(TestBasicWithAliases2 { a: i32::MAX }); + } } From 65f8d0ecbede8c6c33b6c9f61b2abfb8a93ddaae Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Mon, 18 Apr 2022 21:59:21 +0300 Subject: [PATCH 20/66] AVRO-3494: Rust: uncomment some tests which actually pass Signed-off-by: Martin Tzvetanov Grigorov --- lang/rust/avro/tests/schema.rs | 41 +++++++++++++--------------------- 1 file changed, 16 insertions(+), 25 deletions(-) diff --git a/lang/rust/avro/tests/schema.rs b/lang/rust/avro/tests/schema.rs index 292d0aaa64c..1410424baab 100644 --- a/lang/rust/avro/tests/schema.rs +++ b/lang/rust/avro/tests/schema.rs @@ -150,18 +150,14 @@ const RECORD_EXAMPLES: &[(&str, bool)] = &[ }"#, true, ), - /* - // TODO: (#91) figure out why "type": "error" seems to be valid (search in spec) and uncomment ( r#"{ "type": "error", "name": "Test", "fields": [{"name": "f", "type": "long"}] }"#, - true + false, ), - */ - /* // TODO: (#92) properly support recursive types and uncomment ( r#"{ @@ -172,7 +168,7 @@ const RECORD_EXAMPLES: &[(&str, bool)] = &[ {"name": "children", "type": {"type": "array", "items": "Node"}} ] }"#, - true + true, ), ( r#"{ @@ -195,7 +191,7 @@ const RECORD_EXAMPLES: &[(&str, bool)] = &[ } ] }"#, - true + true, ), ( r#"{ @@ -208,9 +204,9 @@ const RECORD_EXAMPLES: &[(&str, bool)] = &[ {"name": "serverHash", "type": "MD5"}, {"name": "meta", "type": ["null", {"type": "map", "values": "bytes"}]} ] - }"#, true + }"#, + true, ), - */ ( r#"{ "type":"record", @@ -262,9 +258,6 @@ const RECORD_EXAMPLES: &[(&str, bool)] = &[ }"#, true, ), - /* - // TODO: (#95) support same types but with different names in unions and uncomment (below the explanation) - // Unions may not contain more than one schema with the same type, except for the named // types record, fixed and enum. For example, unions containing two array types or two map // types are not permitted, but two types with different names are permitted. @@ -283,9 +276,8 @@ const RECORD_EXAMPLES: &[(&str, bool)] = &[ } ] }"#, - true + true, ), - */ ( r#"{ "type": "record", @@ -376,8 +368,7 @@ const OTHER_ATTRIBUTES_EXAMPLES: &[(&str, bool)] = &[ ]; const DECIMAL_LOGICAL_TYPE: &[(&str, bool)] = &[ - /* - // TODO: (#93) support logical types and uncomment + /* ( r#"{ "type": "fixed", @@ -387,7 +378,7 @@ const DECIMAL_LOGICAL_TYPE: &[(&str, bool)] = &[ "size": 10, "scale": 2 }"#, - true + true, ), ( r#"{ @@ -396,7 +387,7 @@ const DECIMAL_LOGICAL_TYPE: &[(&str, bool)] = &[ "precision": 4, "scale": 2 }"#, - true + true, ), ( r#"{ @@ -405,7 +396,7 @@ const DECIMAL_LOGICAL_TYPE: &[(&str, bool)] = &[ "precision": 2, "scale": -2 }"#, - false + false, ), ( r#"{ @@ -414,7 +405,7 @@ const DECIMAL_LOGICAL_TYPE: &[(&str, bool)] = &[ "precision": -2, "scale": 2 }"#, - false + false, ), ( r#"{ @@ -423,7 +414,7 @@ const DECIMAL_LOGICAL_TYPE: &[(&str, bool)] = &[ "precision": 2, "scale": 3 }"#, - false + false, ), ( r#"{ @@ -434,7 +425,7 @@ const DECIMAL_LOGICAL_TYPE: &[(&str, bool)] = &[ "scale": 2, "size": 5 }"#, - false + false, ), ( r#"{ @@ -445,7 +436,7 @@ const DECIMAL_LOGICAL_TYPE: &[(&str, bool)] = &[ "scale": 3, "size": 2 }"#, - false + false, ), ( r#"{ @@ -456,9 +447,9 @@ const DECIMAL_LOGICAL_TYPE: &[(&str, bool)] = &[ "scale": 2, "size": -2 }"#, - false + false, ), - */ + */ ]; const DECIMAL_LOGICAL_TYPE_ATTRIBUTES: &[(&str, bool)] = &[ From 3aa2c3759672ef1757e14eb73406fb6538442b8a Mon Sep 17 00:00:00 2001 From: Martin Grigorov Date: Mon, 18 Apr 2022 22:43:57 +0300 Subject: [PATCH 21/66] AVRO-3494: Uncomment a test for recursive types (#1648) Signed-off-by: Martin Tzvetanov Grigorov --- lang/rust/avro/tests/schema.rs | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/lang/rust/avro/tests/schema.rs b/lang/rust/avro/tests/schema.rs index 1410424baab..f58b62f9e7e 100644 --- a/lang/rust/avro/tests/schema.rs +++ b/lang/rust/avro/tests/schema.rs @@ -565,12 +565,6 @@ lazy_static! { EXAMPLES.iter().copied().filter(|s| s.1).collect(); } -/* -// TODO: (#92) properly support recursive types and uncomment - -This test is failing unwrapping the outer schema with ParseSchemaError("Unknown type: X"). It seems -that recursive types are not properly supported. - #[test] fn test_correct_recursive_extraction() { init(); @@ -594,21 +588,31 @@ fn test_correct_recursive_extraction() { ] }"#; let outer_schema = Schema::parse_str(raw_outer_schema).unwrap(); - if let Schema::Record { fields: outer_fields, .. } = outer_schema { - let raw_inner_schema = outer_fields[0].schema.canonical_form(); - let inner_schema = Schema::parse_str(raw_inner_schema.as_str()).unwrap(); - if let Schema::Record { fields: inner_fields, .. } = inner_schema { - if let Schema::Record {name: recursive_type, .. } = &inner_fields[0].schema { + if let Schema::Record { + fields: outer_fields, + .. + } = outer_schema + { + let inner_schema = &outer_fields[0].schema; + if let Schema::Record { + fields: inner_fields, + .. + } = inner_schema + { + if let Schema::Record { + name: recursive_type, + .. + } = &inner_fields[0].schema + { assert_eq!("X", recursive_type.name.as_str()); } } else { - panic!("inner schema {} should have been a record", raw_inner_schema) + panic!("inner schema {:?} should have been a record", inner_schema) } } else { - panic!("outer schema {} should have been a record", raw_outer_schema) + panic!("outer schema {:?} should have been a record", outer_schema) } } -*/ #[test] fn test_parse() { From db855f6970f0a080dc4d0d4e3ccd75192fe5c541 Mon Sep 17 00:00:00 2001 From: Martin Grigorov Date: Tue, 19 Apr 2022 00:06:13 +0300 Subject: [PATCH 22/66] AVRO-3492: Add logic to derive the aliases for Schema::Enum (#1649) Signed-off-by: Martin Tzvetanov Grigorov --- lang/rust/avro_derive/src/lib.rs | 5 +- lang/rust/avro_derive/tests/derive.rs | 104 ++++++++++++++++++++++---- 2 files changed, 92 insertions(+), 17 deletions(-) diff --git a/lang/rust/avro_derive/src/lib.rs b/lang/rust/avro_derive/src/lib.rs index 11ab7a61661..0055249cb3b 100644 --- a/lang/rust/avro_derive/src/lib.rs +++ b/lang/rust/avro_derive/src/lib.rs @@ -75,6 +75,7 @@ fn derive_avro_schema(input: &mut DeriveInput) -> Result, + aliases: Vec, e: &syn::DataEnum, error_span: Span, ) -> Result> { let doc = preserve_optional(doc); + let enum_aliases = preserve_vec(aliases); if e.variants.iter().all(|v| syn::Fields::Unit == v.fields) { let symbols: Vec = e .variants @@ -181,7 +184,7 @@ fn get_data_enum_schema_def( Ok(quote! { apache_avro::schema::Schema::Enum { name: apache_avro::schema::Name::new(#full_schema_name).expect(&format!("Unable to parse enum name for schema {}", #full_schema_name)[..]), - aliases: None, + aliases: #enum_aliases, doc: #doc, symbols: vec![#(#symbols.to_owned()),*] } diff --git a/lang/rust/avro_derive/tests/derive.rs b/lang/rust/avro_derive/tests/derive.rs index c058c44c10d..284323a2af4 100644 --- a/lang/rust/avro_derive/tests/derive.rs +++ b/lang/rust/avro_derive/tests/derive.rs @@ -1066,16 +1066,16 @@ mod test_derive { #[derive(Debug, Serialize, Deserialize, AvroSchema, Clone, PartialEq)] #[avro(alias = "a", alias = "b", alias = "c")] - struct TestBasicWithAliases { + struct TestBasicStructWithAliases { a: i32, } #[test] - fn test_basic_with_aliases() { + fn test_basic_struct_with_aliases() { let schema = r#" { "type":"record", - "name":"TestBasicWithAliases", + "name":"TestBasicStructWithAliases", "aliases":["a", "b", "c"], "fields":[ { @@ -1086,34 +1086,34 @@ mod test_derive { } "#; let schema = Schema::parse_str(schema).unwrap(); - if let Schema::Record { name, aliases, .. } = TestBasicWithAliases::get_schema() { - assert_eq!("TestBasicWithAliases", name.fullname(None)); + if let Schema::Record { name, aliases, .. } = TestBasicStructWithAliases::get_schema() { + assert_eq!("TestBasicStructWithAliases", name.fullname(None)); assert_eq!( Some(vec!["a".to_owned(), "b".to_owned(), "c".to_owned()]), aliases ); } else { - panic!("TestBasicWithAliases schema must be a record schema") + panic!("TestBasicStructWithAliases schema must be a record schema") } - assert_eq!(schema, TestBasicWithAliases::get_schema()); + assert_eq!(schema, TestBasicStructWithAliases::get_schema()); - serde_assert(TestBasicWithAliases { a: i32::MAX }); + serde_assert(TestBasicStructWithAliases { a: i32::MAX }); } #[derive(Debug, Serialize, Deserialize, AvroSchema, Clone, PartialEq)] #[avro(alias = "d")] #[avro(alias = "e")] #[avro(alias = "f")] - struct TestBasicWithAliases2 { + struct TestBasicStructWithAliases2 { a: i32, } #[test] - fn test_basic_with_aliases2() { + fn test_basic_struct_with_aliases2() { let schema = r#" { "type":"record", - "name":"TestBasicWithAliases2", + "name":"TestBasicStructWithAliases2", "aliases":["d", "e", "f"], "fields":[ { @@ -1124,17 +1124,89 @@ mod test_derive { } "#; let schema = Schema::parse_str(schema).unwrap(); - if let Schema::Record { name, aliases, .. } = TestBasicWithAliases2::get_schema() { - assert_eq!("TestBasicWithAliases2", name.fullname(None)); + if let Schema::Record { name, aliases, .. } = TestBasicStructWithAliases2::get_schema() { + assert_eq!("TestBasicStructWithAliases2", name.fullname(None)); assert_eq!( Some(vec!["d".to_owned(), "e".to_owned(), "f".to_owned()]), aliases ); } else { - panic!("TestBasicWithAliases2 schema must be a record schema") + panic!("TestBasicStructWithAliases2 schema must be a record schema") } - assert_eq!(schema, TestBasicWithAliases2::get_schema()); + assert_eq!(schema, TestBasicStructWithAliases2::get_schema()); - serde_assert(TestBasicWithAliases2 { a: i32::MAX }); + serde_assert(TestBasicStructWithAliases2 { a: i32::MAX }); + } + + #[derive(Debug, Serialize, Deserialize, AvroSchema, Clone, PartialEq)] + #[avro(alias = "a", alias = "b", alias = "c")] + enum TestBasicEnumWithAliases { + A, + B, + } + + #[test] + fn test_basic_enum_with_aliases() { + let schema = r#" + { + "type":"enum", + "name":"TestBasicEnumWithAliases", + "aliases":["a", "b", "c"], + "symbols":[ + "A", + "B" + ] + } + "#; + let schema = Schema::parse_str(schema).unwrap(); + if let Schema::Enum { name, aliases, .. } = TestBasicEnumWithAliases::get_schema() { + assert_eq!("TestBasicEnumWithAliases", name.fullname(None)); + assert_eq!( + Some(vec!["a".to_owned(), "b".to_owned(), "c".to_owned()]), + aliases + ); + } else { + panic!("TestBasicEnumWithAliases schema must be an enum schema") + } + assert_eq!(schema, TestBasicEnumWithAliases::get_schema()); + + serde_assert(TestBasicEnumWithAliases::A); + } + + #[derive(Debug, Serialize, Deserialize, AvroSchema, Clone, PartialEq)] + #[avro(alias = "d")] + #[avro(alias = "e")] + #[avro(alias = "f")] + enum TestBasicEnumWithAliases2 { + A, + B, + } + + #[test] + fn test_basic_enum_with_aliases2() { + let schema = r#" + { + "type":"enum", + "name":"TestBasicEnumWithAliases2", + "aliases":["d", "e", "f"], + "symbols":[ + "A", + "B" + ] + } + "#; + let schema = Schema::parse_str(schema).unwrap(); + if let Schema::Enum { name, aliases, .. } = TestBasicEnumWithAliases2::get_schema() { + assert_eq!("TestBasicEnumWithAliases2", name.fullname(None)); + assert_eq!( + Some(vec!["d".to_owned(), "e".to_owned(), "f".to_owned()]), + aliases + ); + } else { + panic!("TestBasicEnumWithAliases2 schema must be an enum schema") + } + assert_eq!(schema, TestBasicEnumWithAliases2::get_schema()); + + serde_assert(TestBasicEnumWithAliases2::B); } } From 767922862be02c69c6ba60ddef1cd1a510030c0b Mon Sep 17 00:00:00 2001 From: Kyle Schoonover Date: Tue, 19 Apr 2022 00:30:09 -0700 Subject: [PATCH 23/66] AVRO-3415: Add code coverage report support for csharp (#1565) * AVRO-3360 Updated XML documentation * Revert "AVRO-3360 Updated XML documentation" This reverts commit b8601c072a5083380d30b580804dd0908b8cf4cc. * AVRO-3415 Add code coverage report support for csharp * Ignore Updates and package references * Updated names * Sorted packages alphabetically * Mode ReportGenerator instructions for global. * Update versions.props * Remove path * Updated tabbing * Cleanup version.props * Add missing settings from version.props * Updated from tabs to 2 space indents * Added command in code block * Fix carriage return * force carriage return * Another carriage return * Added longer path to report Co-authored-by: Kyle T. Schoonover --- lang/csharp/.gitignore | 2 ++ lang/csharp/CODECOVERAGE.md | 31 ++++++++++++++++++++ lang/csharp/src/apache/test/Avro.test.csproj | 8 +++++ lang/csharp/versions.props | 3 ++ 4 files changed, 44 insertions(+) create mode 100644 lang/csharp/CODECOVERAGE.md diff --git a/lang/csharp/.gitignore b/lang/csharp/.gitignore index 80304575bd8..4218bd59d51 100644 --- a/lang/csharp/.gitignore +++ b/lang/csharp/.gitignore @@ -52,5 +52,7 @@ obj/ #Test results TestResult.xml +Coverage +TestResults .vs/ diff --git a/lang/csharp/CODECOVERAGE.md b/lang/csharp/CODECOVERAGE.md new file mode 100644 index 00000000000..b5411cc937a --- /dev/null +++ b/lang/csharp/CODECOVERAGE.md @@ -0,0 +1,31 @@ + +# C# Avro Code Coverage + +The following instructions should be followed in order to create a code coverage report locally. + +1. Open a command prompt +2. Install ReportGenerator globally\ + a. Run the following command line: `dotnet tool install --global dotnet-reportgenerator-globaltool --version 5.1.4 --add-source https://www.nuget.org/packages/`\ + b. The latest version can be found at [Nuget ReportGenerator](https://www.nuget.org/packages/dotnet-reportgenerator-globaltool/) +3. Navigate to the test project `avro\lang\csharp\src\apache\test` +4. Run the following test command `dotnet test --results-directory ./TestResults --collect:"XPlat Code Coverage"` +5. Generate the report with the following command `ReportGenerator "-reports:./TestResults/*/coverage.cobertura.xml" "-targetdir:./Coverage/" -reporttypes:HTML` +6. Open Report under `avro\lang\csharp\src\apache\test\Coverage\index.html` diff --git a/lang/csharp/src/apache/test/Avro.test.csproj b/lang/csharp/src/apache/test/Avro.test.csproj index 18b5dd8a31f..24fa4b31fc6 100644 --- a/lang/csharp/src/apache/test/Avro.test.csproj +++ b/lang/csharp/src/apache/test/Avro.test.csproj @@ -32,6 +32,14 @@ + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/lang/csharp/versions.props b/lang/csharp/versions.props index d1c39164f5e..170e609db36 100644 --- a/lang/csharp/versions.props +++ b/lang/csharp/versions.props @@ -52,9 +52,12 @@ 0.13.1 + 3.1.2 + 3.1.2 17.1.0 17.1.0 4.1.0 From 0d5a02299d07011c4430172d8965f5bf081acdcf Mon Sep 17 00:00:00 2001 From: Kyle Schoonover Date: Tue, 19 Apr 2022 11:17:46 -0700 Subject: [PATCH 24/66] AVRO-3384: Define C# Coding Style Guidelines (#1534) * AVRO-3360 Updated XML documentation * Revert "AVRO-3360 Updated XML documentation" This reverts commit b8601c072a5083380d30b580804dd0908b8cf4cc. * AVRO-3384 Initial check in * Formatting fix * Additional formatting * More formatting * Added additional rule * Completed new line rules * Indentation preferences complete * Updated header * Additional formatting * More formatting changes * Added spacing options * Updated wrap options * Additional documentation for styling * Updated notes * Updated more * Added var preferences and Expression-bodied member preferences * Initial styling rules documented * Updated naming rules to reflect Roslyn naming rules * Added other styling rule callouts. * Updated Readme * Updated rule * Add header template * Microsoft has a bug for semicolon which makes this not work. * Added license * Added note about IDE0055 Co-authored-by: Kyle T. Schoonover --- .editorconfig | 84 ++- lang/csharp/README.md | 4 + lang/csharp/STYLING.md | 1595 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 1655 insertions(+), 28 deletions(-) create mode 100644 lang/csharp/STYLING.md diff --git a/.editorconfig b/.editorconfig index b24d76fe62a..da154efb8a0 100644 --- a/.editorconfig +++ b/.editorconfig @@ -81,32 +81,60 @@ csharp_style_var_elsewhere = false:suggestion dotnet_style_predefined_type_for_locals_parameters_members = true:suggestion dotnet_style_predefined_type_for_member_access = true:suggestion -# name all constant fields using PascalCase -dotnet_naming_rule.constant_fields_should_be_pascal_case.severity = suggestion -dotnet_naming_rule.constant_fields_should_be_pascal_case.symbols = constant_fields -dotnet_naming_rule.constant_fields_should_be_pascal_case.style = pascal_case_style -dotnet_naming_symbols.constant_fields.applicable_kinds = field -dotnet_naming_symbols.constant_fields.required_modifiers = const -dotnet_naming_style.pascal_case_style.capitalization = pascal_case - -# static fields should have s_ prefix -dotnet_naming_rule.static_fields_should_have_prefix.severity = suggestion -dotnet_naming_rule.static_fields_should_have_prefix.symbols = static_fields -dotnet_naming_rule.static_fields_should_have_prefix.style = static_prefix_style -dotnet_naming_symbols.static_fields.applicable_kinds = field +# Non-private static fields are PascalCase +dotnet_naming_rule.non_private_static_fields_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.non_private_static_fields_should_be_pascal_case.symbols = non_private_static_fields +dotnet_naming_rule.non_private_static_fields_should_be_pascal_case.style = non_private_static_field_style +dotnet_naming_symbols.non_private_static_fields.applicable_kinds = field +dotnet_naming_symbols.non_private_static_fields.applicable_accessibilities = public, protected, internal, protected_internal, private_protected +dotnet_naming_symbols.non_private_static_fields.required_modifiers = static +dotnet_naming_style.non_private_static_field_style.capitalization = pascal_case + +# Constants are PascalCase +dotnet_naming_rule.constants_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.constants_should_be_pascal_case.symbols = constants +dotnet_naming_rule.constants_should_be_pascal_case.style = constant_style +dotnet_naming_symbols.constants.applicable_kinds = field, local +dotnet_naming_symbols.constants.required_modifiers = const +dotnet_naming_style.constant_style.capitalization = pascal_case + +# Static fields are camelCase and start with s_ +dotnet_naming_rule.static_fields_should_be_camel_case.severity = suggestion +dotnet_naming_rule.static_fields_should_be_camel_case.symbols = static_fields +dotnet_naming_rule.static_fields_should_be_camel_case.style = static_field_style +dotnet_naming_symbols.static_fields.applicable_kinds = field dotnet_naming_symbols.static_fields.required_modifiers = static -dotnet_naming_symbols.static_fields.applicable_accessibilities = private, internal, private_protected -dotnet_naming_style.static_prefix_style.required_prefix = s_ -dotnet_naming_style.static_prefix_style.capitalization = camel_case - -# internal and private fields should be _camelCase -dotnet_naming_rule.camel_case_for_private_internal_fields.severity = suggestion -dotnet_naming_rule.camel_case_for_private_internal_fields.symbols = private_internal_fields -dotnet_naming_rule.camel_case_for_private_internal_fields.style = camel_case_underscore_style -dotnet_naming_symbols.private_internal_fields.applicable_kinds = field -dotnet_naming_symbols.private_internal_fields.applicable_accessibilities = private, internal -dotnet_naming_style.camel_case_underscore_style.required_prefix = _ -dotnet_naming_style.camel_case_underscore_style.capitalization = camel_case +dotnet_naming_style.static_field_style.capitalization = camel_case +dotnet_naming_style.static_field_style.required_prefix = s_ + +# Instance fields are camelCase and start with _ +dotnet_naming_rule.instance_fields_should_be_camel_case.severity = suggestion +dotnet_naming_rule.instance_fields_should_be_camel_case.symbols = instance_fields +dotnet_naming_rule.instance_fields_should_be_camel_case.style = instance_field_style +dotnet_naming_symbols.instance_fields.applicable_kinds = field +dotnet_naming_style.instance_field_style.capitalization = camel_case +dotnet_naming_style.instance_field_style.required_prefix = _ + +# Locals and parameters are camelCase +dotnet_naming_rule.locals_should_be_camel_case.severity = suggestion +dotnet_naming_rule.locals_should_be_camel_case.symbols = locals_and_parameters +dotnet_naming_rule.locals_should_be_camel_case.style = camel_case_style +dotnet_naming_symbols.locals_and_parameters.applicable_kinds = parameter, local +dotnet_naming_style.camel_case_style.capitalization = camel_case + +# Local functions are PascalCase +dotnet_naming_rule.local_functions_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.local_functions_should_be_pascal_case.symbols = local_functions +dotnet_naming_rule.local_functions_should_be_pascal_case.style = local_function_style +dotnet_naming_symbols.local_functions.applicable_kinds = local_function +dotnet_naming_style.local_function_style.capitalization = pascal_case + +# By default, name items with PascalCase +dotnet_naming_rule.members_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.members_should_be_pascal_case.symbols = all_members +dotnet_naming_rule.members_should_be_pascal_case.style = pascal_case_style +dotnet_naming_symbols.all_members.applicable_kinds = * +dotnet_naming_style.pascal_case_style.capitalization = pascal_case # Code style defaults csharp_using_directive_placement = outside_namespace:suggestion @@ -116,7 +144,7 @@ csharp_preserve_single_line_blocks = true:none csharp_preserve_single_line_statements = false:none csharp_prefer_static_local_function = true:suggestion csharp_prefer_simple_using_statement = false:none -csharp_style_prefer_switch_expression = true:suggestion +csharp_style_prefer_switch_expression = false:none dotnet_style_readonly_field = true:suggestion # Expression-level preferences @@ -165,7 +193,7 @@ csharp_space_after_dot = false csharp_space_after_keywords_in_control_flow_statements = true csharp_space_after_semicolon_in_for_statement = true csharp_space_around_binary_operators = before_and_after -csharp_space_around_declaration_statements = do_not_ignore +csharp_space_around_declaration_statements = false csharp_space_before_colon_in_inheritance_clause = true csharp_space_before_comma = false csharp_space_before_dot = false @@ -179,4 +207,4 @@ csharp_space_between_method_declaration_empty_parameter_list_parentheses = false csharp_space_between_method_declaration_name_and_open_parenthesis = false csharp_space_between_method_declaration_parameter_list_parentheses = false csharp_space_between_parentheses = false -csharp_space_between_square_brackets = false \ No newline at end of file +csharp_space_between_square_brackets = false diff --git a/lang/csharp/README.md b/lang/csharp/README.md index b035f3c5841..70fc90e86da 100644 --- a/lang/csharp/README.md +++ b/lang/csharp/README.md @@ -42,3 +42,7 @@ In short, we should only update the version of the dependencies in our libraries ## Notes The [LICENSE](./LICENSE) and [NOTICE](./NOTICE) files in the lang/csharp source directory are used to build the binary distribution. The [LICENSE.txt](../../LICENSE.txt) and [NOTICE.txt](../../NOTICE.txt) information for the Avro C# source distribution is in the root directory. + +## Styling Guidelines + +Can be found in [STYLING](./STYLING.MD). diff --git a/lang/csharp/STYLING.md b/lang/csharp/STYLING.md new file mode 100644 index 00000000000..948eddaba8e --- /dev/null +++ b/lang/csharp/STYLING.md @@ -0,0 +1,1595 @@ + +# C# Styling Rules for Apache.Avro + +The following rules are currently used within the .editorconfig of the Avro solution. Any changes to this documentation should be reflected in the .editorconfig file and vice versa. + +Notes + - The examples shown are based on the current settings in .editorconfig + - :exclamation: Not defined :exclamation: means we have not set a preference + - There are cases where it is not explicitly defined in the .editorconfig, but there is a default option + - The project currently targets a framework that uses C# 7.3 + - When violating a formatting rule it may show up as an IDE0055 Fix formatting violation. + +## New line preferences + +### csharp_new_line_before_open_brace +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/formatting-rules#csharp_new_line_before_open_brace) + +This rule concerns whether an open brace { should be placed on the same line as the preceding code, or on a new line. + +**Example** +``` +void MyMethod() +{ + if (...) + { + ... + } +} +``` +--- +### csharp_new_line_before_else +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/formatting-rules#csharp_new_line_before_else) + +**Example** +``` +if (...) { + ... +} +else { + ... +} +``` +--- +### csharp_new_line_before_catch +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/formatting-rules#csharp_new_line_before_catch) + +**Example** +``` +try { + ... +} +catch (Exception e) { + ... +} +``` +--- +### csharp_new_line_before_finally +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/formatting-rules#csharp_new_line_before_finally) + +**Example** +``` +try { + ... +} +catch (Exception e) { + ... +} +finally { + ... +} +``` +--- +### csharp_new_line_before_members_in_object_initializers +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/formatting-rules#csharp_new_line_before_members_in_object_initializers) + +**Example** +``` +var z = new B() +{ + A = 3, + B = 4 +} +``` +--- +### csharp_new_line_before_members_in_anonymous_types +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/formatting-rules#csharp_new_line_before_members_in_anonymous_types) + +**Example** +``` +var z = new +{ + A = 3, + B = 4 +} +``` +--- +### csharp_new_line_between_query_expression_clauses +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/formatting-rules#csharp_new_line_between_query_expression_clauses) + +**Example** +``` +var q = from a in e + from b in e + select a * b; +``` +--- + +## Indentation preferences + +### csharp_indent_case_contents +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/formatting-rules#csharp_indent_case_contents) + +**Example** +``` +switch(c) { + case Color.Red: + Console.WriteLine("The color is red"); + break; + case Color.Blue: + Console.WriteLine("The color is blue"); + break; + default: + Console.WriteLine("The color is unknown."); + break; +} +``` +--- +### csharp_indent_switch_labels +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/formatting-rules#csharp_indent_switch_labels) + +**Example** +``` +switch(c) { + case Color.Red: + Console.WriteLine("The color is red"); + break; + case Color.Blue: + Console.WriteLine("The color is blue"); + break; + default: + Console.WriteLine("The color is unknown."); + break; +} +``` +--- +### csharp_indent_labels +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/formatting-rules#csharp_indent_labels) + +Labels are placed at one less indent to the current context + +**Example** +``` +class C +{ + private string MyMethod(...) + { + if (...) { + goto error; + } + error: + throw new Exception(...); + } +} +``` +--- +### csharp_indent_block_contents +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/formatting-rules#csharp_indent_block_contents) + +**Example** +``` +static void Hello() +{ + Console.WriteLine("Hello"); +} +``` +--- +### csharp_indent_braces +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/formatting-rules#csharp_indent_braces) + +**Example** +``` +static void Hello() +{ + Console.WriteLine("Hello"); +} +``` +--- +### csharp_indent_case_contents_when_block +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/formatting-rules#csharp_indent_case_contents_when_block) + +**Example** +``` +case 0: + { + Console.WriteLine("Hello"); + break; + } +``` +--- + +## Spacing Preferences + +### csharp_space_after_cast +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/formatting-rules#csharp_space_after_cast) + +**Example** +``` +int y = (int)x; +``` +--- +### csharp_space_after_keywords_in_control_flow_statements +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/formatting-rules#csharp_space_after_keywords_in_control_flow_statements) + +**Example** +``` +for (int i;i>, &, ^, |) precedence + +Default is always_for_clarity + +**Example** +``` +var v = a + (b * c); +``` +--- +### dotnet_style_parentheses_in_relational_binary_operators +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0047-ide0048#dotnet_style_parentheses_in_relational_binary_operators) + +Prefer parentheses to clarify relational operator (>, <, <=, >=, is, as, ==, !=) precedence + +Default is always_for_clarity + +**Example** +``` +var v = (a < b) == (c > d); +``` +--- +### dotnet_style_parentheses_in_other_binary_operators +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0047-ide0048#dotnet_style_parentheses_in_other_binary_operators) + +Prefer parentheses to clarify other binary operator (&&, ||, ??) precedence + +Default is always_for_clarity + +**Example** +``` +var v = a || (b && c); +``` +--- +### dotnet_style_parentheses_in_other_operators +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0047-ide0048#dotnet_style_parentheses_in_other_operators) + +Prefer to not have parentheses when operator precedence is obvious + +Default is never_if_unnecessary + +**Example** +``` +var v = a.b.Length; +``` +--- + +## Expression-level preferences + +### dotnet_style_object_initializer +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0017#dotnet_style_object_initializer) + +Prefer objects to be initialized using object initializers when possible + +default is true + +**Example** +``` +var c = new Customer() { Age = 21 }; +``` +--- +### csharp_style_inlined_variable_declaration +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0018#csharp_style_inlined_variable_declaration) + +Prefer out variables to be declared inline in the argument list of a method call when possible + +**Example** +``` +if (int.TryParse(value, out int i) {...} +``` +--- +### dotnet_style_collection_initializer +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0028#dotnet_style_collection_initializer) + +Prefer collections to be initialized using collection initializers when possible + +**Example** +``` +var list = new List { 1, 2, 3 }; +``` +--- +### dotnet_style_prefer_auto_properties +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0032#dotnet_style_prefer_auto_properties) + +Prefer auto properties over properties with private backing fields + +**Example** +``` +private int Age { get; } +``` +--- +### dotnet_style_explicit_tuple_names +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0033#dotnet_style_explicit_tuple_names) + +Prefer tuple names to ItemX properties + +**Example** +``` +(string name, int age) customer = GetCustomer(); +var name = customer.name; +``` +--- +### csharp_prefer_simple_default_expression +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0034#csharp_prefer_simple_default_expression) + +Prefer default over default(T) + +**Example** +``` +void DoWork(CancellationToken cancellationToken = default) { ... } +``` +--- +### dotnet_style_prefer_inferred_tuple_names +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0037#dotnet_style_prefer_inferred_tuple_names) + +Prefer inferred tuple element names + +**Example** +``` +var tuple = (age, name); +``` +--- +### dotnet_style_prefer_inferred_anonymous_type_member_names +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0037#dotnet_style_prefer_inferred_anonymous_type_member_names) + +Prefer inferred anonymous type member names + +**Example** +``` +var anon = new { age, name }; +``` +--- +### csharp_style_pattern_local_over_anonymous_function +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0039#csharp_style_pattern_local_over_anonymous_function) + +Prefer anonymous functions over local functions + +**Example** +``` +Func fibonacci = null; +fibonacci = (int n) => +{ + return n <= 1 ? 1 : fibonacci(n - 1) + fibonacci(n - 2); +}; +``` +--- +### csharp_style_deconstructed_variable_declaration +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0042#csharp_style_deconstructed_variable_declaration) + +Prefer deconstructed variable declaration + +default is true + +**Example** +``` +var (name, age) = GetPersonTuple(); +Console.WriteLine($"{name} {age}"); + +(int x, int y) = GetPointTuple(); +Console.WriteLine($"{x} {y}"); +``` +--- +### dotnet_style_prefer_conditional_expression_over_assignment +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0045#dotnet_style_prefer_conditional_expression_over_assignment) + +Prefer assignments with a ternary conditional over an if-else statement + +**Example** +``` +string s = expr ? "hello" : "world"; +``` +--- +### dotnet_style_prefer_conditional_expression_over_return +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0046#dotnet_style_prefer_conditional_expression_over_return) + +Prefer return statements to use a ternary conditional over an if-else statement + +**Example** +``` +return expr ? "hello" : "world" +``` +--- +### dotnet_style_prefer_compound_assignment +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0054-ide0074#dotnet_style_prefer_compound_assignment) + +Prefer compound assignment expressions + +default is true + +**Example** +``` +x += 1; +``` +--- +### dotnet_style_prefer_simplified_boolean_expressions +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0075#dotnet_style_prefer_simplified_boolean_expressions) + +Prefer simplified conditional expressions + +default is true + +**Example** +``` +var result1 = M1() && M2(); +var result2 = M1() || M2(); +``` +--- +### csharp_style_implicit_object_creation_when_type_is_apparent +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0090#csharp_style_implicit_object_creation_when_type_is_apparent) + +Prefer target-typed new expressions when created type is apparent + +default is true + +**Example** +``` +C c = new(); +C c2 = new() { Field = 0 }; +``` +--- + +## Null-checking Preferences + +### csharp_style_throw_expression +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0016#csharp_style_throw_expression) + +Prefer to use throw expressions instead of throw statements + +**Example** +``` +_s = s ?? throw new ArgumentNullException(nameof(s)); +``` +--- +### dotnet_style_coalesce_expression +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0029-ide0030#dotnet_style_coalesce_expression) + +Prefer null coalescing expressions to ternary operator checking + +**Example** +``` +var v = x ?? y; +``` +--- +### dotnet_style_coalesce_expression +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0031#dotnet_style_null_propagation) + +Prefer to use null-conditional operator when possible + +**Example** +``` +string v = o?.ToString(); +``` +--- +### dotnet_style_prefer_is_null_check_over_reference_equality_method +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0041#dotnet_style_prefer_is_null_check_over_reference_equality_method) + +Prefer is null check over reference equality method + +**Example** +``` +if (value is null) + return; +``` +--- +### csharp_style_conditional_delegate_call +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide1005#csharp_style_conditional_delegate_call) + +Prefer to use the conditional coalescing operator (?.) when invoking a lambda expression, instead of performing a null check + +**Example** +``` +func?.Invoke(args); +``` +--- + +## var Preferences + +### csharp_style_var_for_built_in_types +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0007-ide0008#csharp_style_var_for_built_in_types) + +Prefer explicit type over var to declare variables with built-in system types such as int + +**Example** +``` +int x = 5; +``` +--- +### csharp_style_var_when_type_is_apparent +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0007-ide0008#csharp_style_var_when_type_is_apparent) + +Prefer explicit type over var when the type is already mentioned on the right-hand side of a declaration expression + +**Example** +``` +Customer obj = new Customer(); +``` +--- +### csharp_style_var_elsewhere +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0007-ide0008#csharp_style_var_elsewhere) + +Prefer explicit type over var in all cases, unless overridden by another code style rule + +**Example** +``` +bool f = this.Init(); +``` +--- + +## Expression-bodied member Preferences + +### csharp_style_expression_bodied_constructors +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0021#csharp_style_expression_bodied_constructors) + +Prefer expression bodies for constructors + +**Example** +``` +public Customer(int age) => Age = age; +``` +--- +### csharp_style_expression_bodied_methods +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0022#csharp_style_expression_bodied_methods) + +Prefer expression bodies for methods + +**Example** +``` +public int GetAge() => this.Age; +``` +--- +### csharp_style_expression_bodied_operators +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0023-ide0024#csharp_style_expression_bodied_operators) + +Prefer expression bodies for operators + +**Example** +``` +public static ComplexNumber operator + (ComplexNumber c1, ComplexNumber c2) + => new ComplexNumber(c1.Real + c2.Real, c1.Imaginary + c2.Imaginary); +``` +--- +### csharp_style_expression_bodied_properties +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0025#csharp_style_expression_bodied_properties) + +Prefer expression bodies for properties + +**Example** +``` +public int Age => _age; +``` +--- +### csharp_style_expression_bodied_indexers +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0026#csharp_style_expression_bodied_indexers) + +Prefer expression bodies for indexers + +**Example** +``` +public T this[int i] => _values[i]; +``` +--- +### csharp_style_expression_bodied_accessors +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0027#csharp_style_expression_bodied_accessors) + +Prefer expression bodies for accessors + +**Example** +``` +public int Age { get => _age; set => _age = value; } +``` +--- +### csharp_style_expression_bodied_lambdas +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0053#csharp_style_expression_bodied_lambdas) + +Prefer expression bodies for lambdas + +**Example** +``` +Func square = x => x * x; +``` +--- +### csharp_style_expression_bodied_local_functions +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0061#csharp_style_expression_bodied_local_functions) + +Prefer expression bodies for local functions + +**Example** +``` +void M() +{ + Hello(); + void Hello() => Console.WriteLine("Hello"); +} +``` +--- + +## Pattern matching Preferences + +### csharp_style_pattern_matching_over_as_with_null_check +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0019#csharp_style_pattern_matching_over_as_with_null_check) + +Prefer pattern matching instead of as expressions with null checks to determine if something is of a particular type + +**Example** +``` +if (o is string s) {...} +``` +--- +### csharp_style_pattern_matching_over_is_with_cast_check +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0020-ide0038#csharp_style_pattern_matching_over_is_with_cast_check) + +Prefer pattern matching instead of is expressions with type casts + +**Example** +``` +if (o is int i) {...} +``` +--- +### csharp_style_prefer_switch_expression +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0066#csharp_style_prefer_switch_expression) + +Prefer to use a switch expression (introduced with C# 8.0) + +**Example** +``` +return x switch +{ + 1 => 1 * 1, + 2 => 2 * 2, + _ => 0, +}; +``` +--- +### csharp_style_prefer_not_pattern +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0083#csharp_style_prefer_not_pattern) + +Prefer to use 'not' pattern, when possible (introduced with C# 9.0) + +Default is true + +**Example** +``` +var y = o is not C c; +``` +--- + +## Code block Prerferences + +### csharp_prefer_braces +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0011#csharp_prefer_braces) + +Prefer curly braces even for one line of code + +**Example** +``` +if (test) { this.Display(); } +``` +--- +### csharp_prefer_simple_using_statement +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0063#csharp_prefer_simple_using_statement) + +Don't prefer to use a simple using statement + +**Example** +``` +using (var a = b) { } +``` +--- + +## File Header Preferences + +### file_header_template +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0073#file_header_template) + +unset or empty string - Do not require file header. + +Default is unset + +**Example** +``` +namespace N2 +{ + class C2 { } +} +``` +--- + +## Naming Rules + +### Non-private static fields are PascalCase +**Example** +``` +public static MyString = "value"; +protected static MyString = "value"; +internal static MyString = "value"; +protected_internal static MyString = "value"; +private_protected static MyString = "value";; +``` +--- +### Constants are PascalCase +**Example** +``` +public const string MyConstant = "value"; +``` +--- +### Static fields are camelCase and start with s_ +**Example** +``` +private static int s_myInt; +``` +--- +# Instance fields are camelCase and start with _ +**Example** +``` +private int _myInt; + +internal string _myString; +``` +--- +# Locals and parameters are camelCase +**Example** +``` +private static string GetText(string path, string filename) +{ + var reader = File.OpenText($"{AppendPathSeparator(path)}{filename}"); + var text = reader.ReadToEnd(); + return text; + + string AppendPathSeparator(string filepath) + { + return filepath.EndsWith(@"\") ? filepath : filepath + @"\"; + } +} +``` +--- +# Local functions are PascalCase +**Example** +``` +private static string GetText(string path, string filename) +{ + var reader = File.OpenText($"{AppendPathSeparator(path)}{filename}"); + var text = reader.ReadToEnd(); + return text; + + string AppendPathSeparator(string filepath) + { + return filepath.EndsWith(@"\") ? filepath : filepath + @"\"; + } +} +``` +--- +# By default, name items with PascalCase +**Example** +``` +public void MyMethod() { }; +``` +--- + +## Formatting Rules + +### dotnet_sort_system_directives_first +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/formatting-rules#dotnet_sort_system_directives_first) + +Sort System.* using directives alphabetically, and place them before other using directives. + +**Example** +``` +using System.Collections.Generic; +using System.Threading.Tasks; +using Avro; +``` +--- +### dotnet_separate_import_directive_groups +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/formatting-rules#dotnet_separate_import_directive_groups) + +:exclamation: Not defined :exclamation: + +**Example** +``` +// dotnet_separate_import_directive_groups = true +using System.Collections.Generic; +using System.Threading.Tasks; + +using Avro; + +// dotnet_separate_import_directive_groups = false +using System.Collections.Generic; +using System.Threading.Tasks; +using Avro; +``` +--- +### dotnet_style_namespace_match_folder +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/formatting-rules#dotnet_style_namespace_match_folder) + +:exclamation: Not defined :exclamation: + +**Example** +``` +// dotnet_style_namespace_match_folder = true +// file path: Example/Convention/C.cs +using System; + +namespace Example.Convention +{ + class C + { + } +} + +// dotnet_style_namespace_match_folder = false +// file path: Example/Convention/C.cs +using System; + +namespace Example +{ + class C + { + } +} +``` +--- + +## Unnecessary Code Rules + +### Simplify name (IDE0001) +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0001) +**Example** +``` +using System.IO; +class C +{ + // IDE0001: 'System.IO.FileInfo' can be simplified to 'FileInfo' + System.IO.FileInfo file; + + // Fixed code + FileInfo file; +} +``` +--- +### Simplify member access (IDE0002) +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0002) +**Example** +``` +static void M1() { } +static void M2() +{ + // IDE0002: 'C.M1' can be simplified to 'M1' + C.M1(); + + // Fixed code + M1(); +} +``` +--- +### Remove unnecessary cast (IDE0004) +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0004) +**Example** +``` +// Code with violations +int v = (int)0; + +// Fixed code +int v = 0; +``` +--- +### Remove unnecessary import (IDE0005) +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0005) +**Example** +``` +// Code with violations +using System; +using System.IO; // IDE0005: Using directive is unnecessary +class C +{ + public static void M() + { + Console.WriteLine("Hello"); + } +} + +// Fixed code +using System; +class C +{ + public static void M() + { + Console.WriteLine("Hello"); + } +} +``` +--- +### Remove unreachable code (IDE0035) +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0035) +**Example** +``` +// Code with violations +void M() +{ + throw new System.Exception(); + + // IDE0035: Remove unreachable code + int v = 0; +} + +// Fixed code +void M() +{ + throw new System.Exception(); +} +``` +--- +### Remove unused private member (IDE0051) +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0051) +**Example** +``` +// Code with violations +class C +{ + // IDE0051: Remove unused private members + private readonly int _fieldPrivate; + private int PropertyPrivate => 1; + private int GetNumPrivate() => 1; + + // No IDE0051 + internal readonly int FieldInternal; + private readonly int _fieldPrivateUsed; + public int PropertyPublic => _fieldPrivateUsed; + private int GetNumPrivateUsed() => 1; + internal int GetNumInternal() => GetNumPrivateUsed(); + public int GetNumPublic() => GetNumPrivateUsed(); +} + +// Fixed code +class C +{ + // No IDE0051 + internal readonly int FieldInternal; + private readonly int _fieldPrivateUsed; + public int PropertyPublic => _fieldPrivateUsed; + private int GetNumPrivateUsed() => 1; + internal int GetNumInternal() => GetNumPrivateUsed(); + public int GetNumPublic() => GetNumPrivateUsed(); +} +``` +--- +### Remove unread private member (IDE0052) +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0052) +**Example** +``` +class C +{ + // IDE0052: Remove unread private members + private readonly int _field1; + private int _field2; + private int Property { get; set; } + + public C() + { + _field1 = 0; + } + + public void SetMethod() + { + _field2 = 0; + Property = 0; + } +} + +// Fixed code +class C +{ + public C() + { + } + + public void SetMethod() + { + } +} +``` +--- +### csharp_style_unused_value_expression_statement_preference +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0058#csharp_style_unused_value_expression_statement_preference) + +Prefer to assign an unused expression to a discard + +Default is discard_variable + +**Example** +``` +_ = System.Convert.ToInt32("35"); +``` +--- +### csharp_style_unused_value_assignment_preference +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0059#csharp_style_unused_value_assignment_preference) + +Prefer to use a discard when assigning a value that's not used + +Default is discard_variable + +**Example** +``` +int GetCount(Dictionary wordCount, string searchWord) +{ + _ = wordCount.TryGetValue(searchWord, out var count); + return count; +} +``` +--- +### dotnet_code_quality_unused_parameters +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0060#dotnet_code_quality_unused_parameters) + +Flag methods with any accessibility that contain unused parameters + +Default is all + +**Example** +``` +public int GetNum1(int unusedParam) { return 1; } +internal int GetNum2(int unusedParam) { return 1; } +private int GetNum3(int unusedParam) { return 1; } +``` +--- +### dotnet_remove_unnecessary_suppression_exclusions +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0079#dotnet_remove_unnecessary_suppression_exclusions) + + enables the rule for all rule IDs and rule categories + +Default is none + +**Example** +``` +using System.Diagnostics.CodeAnalysis; + +class C1 +{ + // 'dotnet_remove_unnecessary_suppression_exclusions = IDE0051' + + // Unnecessary pragma suppression, but not flagged by IDE0079 +#pragma warning disable IDE0051 // IDE0051: Remove unused member + private int UsedMethod() => 0; +#pragma warning restore IDE0051 + + public int PublicMethod() => UsedMethod(); +} +``` +--- +### Remove unnecessary suppression operator (IDE0080) +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0080) +**Example** +``` +// Code with violations +if (o !is string) { } + +// Potential fixes: +// 1. +if (o is not string) { } + +// 2. +if (!(o is string)) { } + +// 3. +if (o is string) { } +``` +--- +### Remove unnecessary equality operator (IDE0100) +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0100) +**Example** +``` +// Code with violations +if (x == true) { } +if (M() != false) { } + +// Fixed code +if (x) { } +if (M()) { } +``` +--- +### Remove unnecessary discard (IDE0110) +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0110) +**Example** +``` +// Code with violations +switch (o) +{ + case int _: + Console.WriteLine("Value was an int"); + break; + case string _: + Console.WriteLine("Value was a string"); + break; +} + +// Fixed code +switch (o) +{ + case int: + Console.WriteLine("Value was an int"); + break; + case string: + Console.WriteLine("Value was a string"); + break; +} +``` +--- + +## Miscellaneous Rules + +### Remove invalid global 'SuppressMessageAttribute' (IDE0076) +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0076) +**Example** +``` +// IDE0076: Invalid target '~F:N.C.F2' - no matching field named 'F2' +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Category", "Id: Title", Scope = "member", Target = "~F:N.C.F2")] +// IDE0076: Invalid scope 'property' +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Category", "Id: Title", Scope = "property", Target = "~P:N.C.P")] + +// Fixed code +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Category", "Id: Title", Scope = "member", Target = "~F:N.C.F")] +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Category", "Id: Title", Scope = "member", Target = "~P:N.C.P")] + +namespace N +{ + class C + { + public int F; + public int P { get; } + } +} +``` +--- +### Avoid legacy format target in global 'SuppressMessageAttribute' (IDE0077) +[Reference](https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0077) +**Example** +``` +// IDE0077: Legacy format target 'N.C.#F' +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Category", "Id: Title", Scope = "member", Target = "N.C.#F")] + +// Fixed code +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Category", "Id: Title", Scope = "member", Target = "~F:N.C.F")] + +namespace N +{ + class C + { + public int F; + } +} +``` +--- From b465b39795f4d289dcaa741677691d4ac45c9d68 Mon Sep 17 00:00:00 2001 From: Kyle Schoonover Date: Tue, 19 Apr 2022 11:19:01 -0700 Subject: [PATCH 25/66] AVRO-3424: Added support to parse string into Schema.Type (#1571) * AVRO-3360 Updated XML documentation * Revert "AVRO-3360 Updated XML documentation" This reverts commit b8601c072a5083380d30b580804dd0908b8cf4cc. * AVRO-3424 Created extension method for converting string into a Schema.Type enumeration * Updated functionality * Removed breaking code * Updated remove quotes * Removed if from tests Co-authored-by: Kyle T. Schoonover --- lang/csharp/src/apache/main/Schema/Schema.cs | 89 ++++++++++++++++++- .../src/apache/test/Schema/SchemaTests.cs | 43 ++++++++- 2 files changed, 127 insertions(+), 5 deletions(-) diff --git a/lang/csharp/src/apache/main/Schema/Schema.cs b/lang/csharp/src/apache/main/Schema/Schema.cs index 69de388c35c..3e54653f015 100644 --- a/lang/csharp/src/apache/main/Schema/Schema.cs +++ b/lang/csharp/src/apache/main/Schema/Schema.cs @@ -196,7 +196,8 @@ internal static Schema ParseJson(JToken jtok, SchemaNames names, string encspace return LogicalSchema.NewInstance(jtok, props, names, encspace); Schema schema = PrimitiveSchema.NewInstance((string)type, props); - if (null != schema) return schema; + if (null != schema) + return schema; return NamedSchema.NewInstance(jo, props, names, encspace); } @@ -380,5 +381,91 @@ protected static int getHashCode(object obj) { return obj == null ? 0 : obj.GetHashCode(); } + + /// + /// Parses the Schema.Type from a string. + /// + /// The type to convert. + /// if set to true [remove quotes]. + /// A Schema.Type unless it could not parse then null + /// + /// usage ParseType("string") returns Schema.Type.String + /// + public static Schema.Type? ParseType(string type, bool removeQuotes = false) + { + string newValue = removeQuotes ? RemoveQuotes(type) : type; + + switch (newValue) + { + case "null": + return Schema.Type.Null; + + case "boolean": + return Schema.Type.Boolean; + + case "int": + return Schema.Type.Int; + + case "long": + return Schema.Type.Long; + + case "float": + return Schema.Type.Float; + + case "double": + return Schema.Type.Double; + + case "bytes": + return Schema.Type.Bytes; + + case "string": + return Schema.Type.String; + + case "record": + return Schema.Type.Record; + + case "enumeration": + return Schema.Type.Enumeration; + + case "array": + return Schema.Type.Array; + + case "map": + return Schema.Type.Map; + + case "union": + return Schema.Type.Union; + + case "fixed": + return Schema.Type.Fixed; + + case "error": + return Schema.Type.Error; + + case "logical": + return Schema.Type.Logical; + + default: + return null; + } + } + + /// + /// Removes the quotes from the first position and last position of the string. + /// + /// The value. + /// + /// If string has a quote at the beginning and the end it removes them, + /// otherwise it returns the original string + /// + private static string RemoveQuotes(string value) + { + if(value.StartsWith("\"") && value.EndsWith("\"")) + { + return value.Substring(1, value.Length - 2); + } + + return value; + } } } diff --git a/lang/csharp/src/apache/test/Schema/SchemaTests.cs b/lang/csharp/src/apache/test/Schema/SchemaTests.cs index 59b4a90f38f..a061d53731e 100644 --- a/lang/csharp/src/apache/test/Schema/SchemaTests.cs +++ b/lang/csharp/src/apache/test/Schema/SchemaTests.cs @@ -16,10 +16,7 @@ * limitations under the License. */ using System; -using System.Collections.Generic; -using System.Text; using NUnit.Framework; -using Avro; namespace Avro.Test { @@ -226,7 +223,7 @@ public void TestPrimitiveWithMetadata(string rawSchema, Schema.Type type) Assert.True(recordSchema["f1"].Schema.ToString().Contains("metafield")); Assert.True(definedSchema.Equals(recordSchema["f1"].Schema)); - Assert.AreEqual(definedSchema.GetHashCode(),recordSchema["f1"].Schema.GetHashCode()); + Assert.AreEqual(definedSchema.GetHashCode(), recordSchema["f1"].Schema.GetHashCode()); } [TestCase("{\"type\":\"record\",\"name\":\"LongList\"," + @@ -440,5 +437,43 @@ public void TestUnionSchemaWithoutTypeProperty(string schemaJson, string expecte var schema = Schema.Parse(schemaJson); Assert.AreEqual(schema.ToString(), expectedSchemaJson); } + + [TestFixture] + public class SchemaTypeExtensionsTests + { + [TestCase("null", Schema.Type.Null)] + [TestCase("boolean", Schema.Type.Boolean)] + [TestCase("int", Schema.Type.Int)] + [TestCase("long", Schema.Type.Long)] + [TestCase("float", Schema.Type.Float)] + [TestCase("double", Schema.Type.Double)] + [TestCase("bytes", Schema.Type.Bytes)] + [TestCase("string", Schema.Type.String)] + [TestCase("record", Schema.Type.Record)] + [TestCase("enumeration", Schema.Type.Enumeration)] + [TestCase("array", Schema.Type.Array)] + [TestCase("map", Schema.Type.Map)] + [TestCase("union", Schema.Type.Union)] + [TestCase("fixed", Schema.Type.Fixed)] + [TestCase("error", Schema.Type.Error)] + [TestCase("logical", Schema.Type.Logical)] + [TestCase("Logical", null)] + [TestCase("InvalidValue", null)] + [TestCase("\"null\"", null)] + [TestCase("", null)] + [TestCase(null, null)] + public void ParseTypeTest(string value, object expectedResult) + { + Assert.AreEqual(Schema.ParseType(value), expectedResult); + } + + [TestCase("\"null\"", Schema.Type.Null)] + [TestCase("\"nu\"ll\"", null)] + [TestCase("\"\"", null)] + public void ParseTypeRemoveQuotesTest(string value, object expectedResult) + { + Assert.AreEqual(Schema.ParseType(value, true), expectedResult); + } + } } } From 7f82a8185be70548723e8d7e0dad7a990e40c8cb Mon Sep 17 00:00:00 2001 From: Jose Massada Date: Tue, 19 Apr 2022 19:20:03 +0100 Subject: [PATCH 26/66] AVRO-3003: Fully qualify enum default value in C# code gen (#1596) --- .../csharp/src/apache/main/CodeGen/CodeGen.cs | 2 +- .../src/apache/test/AvroGen/AvroGenTests.cs | 31 ++++++++++++++++--- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/lang/csharp/src/apache/main/CodeGen/CodeGen.cs b/lang/csharp/src/apache/main/CodeGen/CodeGen.cs index e2438d980aa..85ba07da6df 100644 --- a/lang/csharp/src/apache/main/CodeGen/CodeGen.cs +++ b/lang/csharp/src/apache/main/CodeGen/CodeGen.cs @@ -784,7 +784,7 @@ protected virtual CodeTypeDeclaration processRecord(Schema schema) codeField.Attributes = MemberAttributes.Private; if (field.Schema is EnumSchema es && es.Default != null) { - codeField.InitExpression = new CodeTypeReferenceExpression($"{es.Name}.{es.Default}"); + codeField.InitExpression = new CodeTypeReferenceExpression($"{es.Namespace}.{es.Name}.{es.Default}"); } // Process field documentation if it exist and add to the field diff --git a/lang/csharp/src/apache/test/AvroGen/AvroGenTests.cs b/lang/csharp/src/apache/test/AvroGen/AvroGenTests.cs index 9b6a3a024c8..93e453190ad 100644 --- a/lang/csharp/src/apache/test/AvroGen/AvroGenTests.cs +++ b/lang/csharp/src/apache/test/AvroGen/AvroGenTests.cs @@ -421,7 +421,7 @@ public void GenerateSchema(string schema, IEnumerable typeNamesToCheck, { TestSchema(schema, typeNamesToCheck, generatedFilesToCheck: generatedFilesToCheck); } - + [TestCase( _nullableLogicalTypesArray, "org.apache.avro.codegentest.testdata", "org.apache.csharp.codegentest.testdata", @@ -552,7 +552,7 @@ public void NotSupportedSchema(string schema, Type expectedException) ""name"" : ""ClassKeywords"", ""namespace"" : ""com.base"", ""fields"" : - [ + [ { ""name"" : ""int"", ""type"" : ""int"" }, { ""name"" : ""base"", ""type"" : ""long"" }, { ""name"" : ""event"", ""type"" : ""boolean"" }, @@ -583,7 +583,7 @@ public void NotSupportedSchema(string schema, Type expectedException) ""name"" : ""SchemaObject"", ""namespace"" : ""schematest"", ""fields"" : - [ + [ { ""name"" : ""myobject"", ""type"" : [ ""null"", @@ -605,7 +605,7 @@ public void NotSupportedSchema(string schema, Type expectedException) ""name"" : ""LogicalTypes"", ""namespace"" : ""schematest"", ""fields"" : - [ + [ { ""name"" : ""nullibleguid"", ""type"" : [""null"", {""type"": ""string"", ""logicalType"": ""uuid"" } ]}, { ""name"" : ""guid"", ""type"" : {""type"": ""string"", ""logicalType"": ""uuid"" } }, { ""name"" : ""nullibletimestampmillis"", ""type"" : [""null"", {""type"": ""long"", ""logicalType"": ""timestamp-millis""}] }, @@ -621,6 +621,29 @@ public void NotSupportedSchema(string schema, Type expectedException) ] }", new object[] { "schematest.LogicalTypes", typeof(Guid?), typeof(Guid), typeof(DateTime?), typeof(DateTime), typeof(DateTime?), typeof(DateTime), typeof(TimeSpan?), typeof(TimeSpan), typeof(TimeSpan?), typeof(TimeSpan), typeof(AvroDecimal?), typeof(AvroDecimal) })] + [TestCase(@" +{ + ""namespace"": ""enum.base"", + ""type"": ""record"", + ""name"": ""EnumInDifferentNamespace"", + ""doc"": ""Test enum with a default value in a different namespace"", + ""fields"": [ + { + ""name"": ""anEnum"", + ""type"": { + ""namespace"": ""enum.base.other"", + ""type"": ""enum"", + ""name"": ""AnEnum"", + ""symbols"": [ + ""A"", + ""B"" + ], + ""default"": ""A"" + } + } + ] +}", + new object[] { "enum.base.EnumInDifferentNamespace", "enum.base.other.AnEnum" })] public void GenerateSchemaCheckFields(string schema, object[] result) { Assembly assembly = TestSchema(schema); From 328f59063a990d1fbc2e952c5d81d67b77925787 Mon Sep 17 00:00:00 2001 From: Kyle Schoonover Date: Tue, 19 Apr 2022 11:23:05 -0700 Subject: [PATCH 27/66] AVRO-3458: Added tests for GenericRecord (#1606) * AVRO-3360 Updated XML documentation * Revert "AVRO-3360 Updated XML documentation" This reverts commit b8601c072a5083380d30b580804dd0908b8cf4cc. * AVRO-3458 Added tests for GenericRecord * Moved Schema to const * using discard * Empty * Add license Co-authored-by: Kyle T. Schoonover --- .../apache/test/Generic/GenericRecordTests.cs | 220 ++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 lang/csharp/src/apache/test/Generic/GenericRecordTests.cs diff --git a/lang/csharp/src/apache/test/Generic/GenericRecordTests.cs b/lang/csharp/src/apache/test/Generic/GenericRecordTests.cs new file mode 100644 index 00000000000..242ed60a121 --- /dev/null +++ b/lang/csharp/src/apache/test/Generic/GenericRecordTests.cs @@ -0,0 +1,220 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using Avro.Generic; +using NUnit.Framework; + +namespace Avro.test.Generic +{ + [TestFixture] + public class GenericRecordTests + { + private const string baseSchema = "{\"type\":\"record\",\"name\":\"r\",\"fields\":" + + "[{\"name\":\"f2\",\"type\":\"int\"},{\"name\":\"f1\",\"type\":\"boolean\"}]}"; + + [Test] + public void TestAddByFieldNameThrows() + { + GenericRecord genericRecord = GetBaseGenericRecord(); + + // Field does not exist + Assert.Throws(() => { genericRecord.Add("badField", "test"); }); + } + + [Test] + public void TestAddByPosition() + { + GenericRecord genericRecord = GetBaseGenericRecord(); + + genericRecord.Add(0, 2); + + object value = genericRecord.GetValue(0); + + Assert.IsNotNull(value); + Assert.IsTrue(value is int); + Assert.AreEqual(2, (int)value); + } + + [Test] + public void TestAddByPositionThrows() + { + GenericRecord genericRecord = GetBaseGenericRecord(); + + Assert.Throws(() => { genericRecord.Add(2, 2); }); + } + + [Test] + public void TestEquals() + { + GenericRecord genericRecord = GetBaseGenericRecord(); + GenericRecord genericRecord2 = GetBaseGenericRecord(); + + Assert.IsTrue(genericRecord.Equals(genericRecord2)); + } + + [Test] + public void TestEqualsNotEqual() + { + GenericRecord genericRecord = GetBaseGenericRecord(); + GenericRecord genericRecord2 = GetBaseGenericRecord(); + genericRecord2.Add(0, 2); + + Assert.IsFalse(genericRecord.Equals(genericRecord2)); + } + + [Test] + public void TestEqualsObject() + { + GenericRecord genericRecord = GetBaseGenericRecord(); + object genericRecord2 = genericRecord; + + Assert.IsTrue(genericRecord.Equals(genericRecord2)); + } + + [Test] + public void TestEqualsObjectNotEqual() + { + GenericRecord genericRecord = GetBaseGenericRecord(); + GenericRecord genericRecord2 = GetBaseGenericRecord(); + genericRecord2.Add(0, 2); + + Assert.IsFalse(genericRecord.Equals((object)genericRecord2)); + } + + [Test] + public void TestEqualsObjectNullObject() + { + GenericRecord genericRecord = GetBaseGenericRecord(); + + Assert.IsFalse(genericRecord.Equals((object)null)); + } + + [Test] + public void TestGetHashCode() + { + int hashCode = GetBaseGenericRecord().GetHashCode(); + Assert.IsTrue(hashCode > 0); + } + + [Test] + public void TestGetValue() + { + GenericRecord genericRecord = GetBaseGenericRecord(); + + object value = genericRecord.GetValue(0); + + Assert.IsNotNull(value); + Assert.IsTrue(value is int); + Assert.AreEqual(1, (int)value); + } + + [Test] + public void TestKeyValueLookup() + { + GenericRecord genericRecord = GetBaseGenericRecord(); + + // Key Exists + object existingKey = genericRecord["f2"]; + Assert.IsNotNull(existingKey); + Assert.IsTrue(existingKey is int); + } + + [Test] + public void TestKeyValueLookupThrows() + { + GenericRecord genericRecord = GetBaseGenericRecord(); + + // Key does not exist + Assert.Throws(() => { _ = genericRecord["badField"]; }); + } + + [Test] + public void TestToString() + { + GenericRecord genericRecord = GetBaseGenericRecord(); + string str = genericRecord.ToString(); + string expectedValue = "Schema: {\"type\":\"record\",\"name\":\"r\",\"fields\":" + + "[{\"name\":\"f2\",\"type\":\"int\"},{\"name\":\"f1\",\"type\":" + + "\"boolean\"}]}, contents: { f2: 1, f1: True, }"; + + Assert.AreEqual(expectedValue, str); + } + + [Test] + public void TestTryGetValue() + { + GenericRecord genericRecord = GetBaseGenericRecord(); + + // Value exists + bool returnResult = genericRecord.TryGetValue("f2", out object result); + + Assert.IsTrue(returnResult); + Assert.IsNotNull(result); + Assert.IsTrue(result is int); + Assert.AreEqual(1, (int)result); + } + + [Test] + public void TestTryGetValueByPosition() + { + GenericRecord genericRecord = GetBaseGenericRecord(); + + bool returnResult = genericRecord.TryGetValue(0, out object value); + + Assert.IsTrue(returnResult); + Assert.IsNotNull(value); + Assert.IsTrue(value is int); + Assert.AreEqual(1, (int)value); + } + + [Test] + public void TestTryGetValueByPositionNotFound() + { + GenericRecord genericRecord = GetBaseGenericRecord(); + + bool returnResult = genericRecord.TryGetValue(3, out object value); + + Assert.IsFalse(returnResult); + Assert.IsNull(value); + } + + [Test] + public void TestTryGetValueNotFound() + { + GenericRecord genericRecord = GetBaseGenericRecord(); + + // Value exists + bool returnResult = genericRecord.TryGetValue("badField", out object result); + + Assert.IsFalse(returnResult); + Assert.IsNull(result); + } + + private GenericRecord GetBaseGenericRecord() + { + RecordSchema testSchema = Schema.Parse(baseSchema) as RecordSchema; + GenericRecord genericRecord = new GenericRecord(testSchema); + genericRecord.Add("f2", 1); + genericRecord.Add("f1", true); + + return genericRecord; + } + } +} From 16efcc36c4602f956403267daecbf3d7c979a01c Mon Sep 17 00:00:00 2001 From: Zoltan Csizmadia Date: Tue, 19 Apr 2022 13:24:05 -0500 Subject: [PATCH 28/66] AVRO-2883: Fix namespace mapping (#1610) * Remove unused package references * Replace namespace in text schema * Remove namespace mapping * Add unit tests * Match namespace mapping used in ticket * Make ReplaceMappedNamespacesInSchema private * Mark NamespaceMapping obsolete Co-authored-by: Zoltan Csizmadia --- .../src/apache/codegen/Avro.codegen.csproj | 9 -- lang/csharp/src/apache/codegen/AvroGen.cs | 13 +- .../csharp/src/apache/main/CodeGen/CodeGen.cs | 76 ++++++++++- .../src/apache/test/AvroGen/AvroGenTests.cs | 123 ++++++++++++++---- 4 files changed, 170 insertions(+), 51 deletions(-) diff --git a/lang/csharp/src/apache/codegen/Avro.codegen.csproj b/lang/csharp/src/apache/codegen/Avro.codegen.csproj index e0e2b8e68b3..dfb438d37a4 100644 --- a/lang/csharp/src/apache/codegen/Avro.codegen.csproj +++ b/lang/csharp/src/apache/codegen/Avro.codegen.csproj @@ -54,15 +54,6 @@ - - - - - - - - - diff --git a/lang/csharp/src/apache/codegen/AvroGen.cs b/lang/csharp/src/apache/codegen/AvroGen.cs index f1572c32f19..d1169027269 100644 --- a/lang/csharp/src/apache/codegen/AvroGen.cs +++ b/lang/csharp/src/apache/codegen/AvroGen.cs @@ -159,13 +159,9 @@ public static int GenProtocol(string infile, string outdir, try { string text = System.IO.File.ReadAllText(infile); - Protocol protocol = Protocol.Parse(text); CodeGen codegen = new CodeGen(); - codegen.AddProtocol(protocol); - - foreach (var entry in namespaceMapping) - codegen.NamespaceMapping[entry.Key] = entry.Value; + codegen.AddProtocol(text, namespaceMapping); codegen.GenerateCode(); codegen.WriteTypes(outdir); @@ -185,13 +181,8 @@ public static int GenSchema(string infile, string outdir, try { string text = System.IO.File.ReadAllText(infile); - Schema schema = Schema.Parse(text); - CodeGen codegen = new CodeGen(); - codegen.AddSchema(schema); - - foreach (var entry in namespaceMapping) - codegen.NamespaceMapping[entry.Key] = entry.Value; + codegen.AddSchema(text, namespaceMapping); codegen.GenerateCode(); codegen.WriteTypes(outdir); diff --git a/lang/csharp/src/apache/main/CodeGen/CodeGen.cs b/lang/csharp/src/apache/main/CodeGen/CodeGen.cs index 85ba07da6df..72776036e2d 100644 --- a/lang/csharp/src/apache/main/CodeGen/CodeGen.cs +++ b/lang/csharp/src/apache/main/CodeGen/CodeGen.cs @@ -24,6 +24,7 @@ using System.Linq; using System.Reflection; using System.Text; +using System.Text.RegularExpressions; using Microsoft.CSharp; namespace Avro @@ -63,6 +64,7 @@ public class CodeGen /// /// The namespace mapping. /// + [Obsolete("NamespaceMapping is not used, use AddProtocol(string ...) or AddSchema(string ...) instead!")] public IDictionary NamespaceMapping { get; private set; } /// @@ -80,7 +82,6 @@ public CodeGen() { Schemas = new List(); Protocols = new List(); - NamespaceMapping = new Dictionary(); NamespaceLookup = new Dictionary(StringComparer.Ordinal); } @@ -103,6 +104,19 @@ public virtual void AddProtocol(Protocol protocol) Protocols.Add(protocol); } + /// + /// Parses and adds a protocol object to generate code for. + /// + /// The protocol. + /// namespace mapping key value pairs. + public virtual void AddProtocol(string protocolText, IEnumerable> namespaceMapping = null) + { + // Map namespaces + protocolText = ReplaceMappedNamespacesInSchema(protocolText, namespaceMapping); + Protocol protocol = Protocol.Parse(protocolText); + Protocols.Add(protocol); + } + /// /// Adds a schema object to generate code for. /// @@ -112,6 +126,19 @@ public virtual void AddSchema(Schema schema) Schemas.Add(schema); } + /// + /// Parses and adds a schema object to generate code for. + /// + /// schema object. + /// namespace mapping key value pairs. + public virtual void AddSchema(string schemaText, IEnumerable> namespaceMapping = null) + { + // Map namespaces + schemaText = ReplaceMappedNamespacesInSchema(schemaText, namespaceMapping); + Schema schema = Schema.Parse(schemaText); + Schemas.Add(schema); + } + /// /// Adds a namespace object for the given name into the dictionary if it doesn't exist yet. /// @@ -129,9 +156,7 @@ protected virtual CodeNamespace AddNamespace(string name) if (!NamespaceLookup.TryGetValue(name, out CodeNamespace ns)) { - ns = NamespaceMapping.TryGetValue(name, out string csharpNamespace) - ? new CodeNamespace(csharpNamespace) - : new CodeNamespace(CodeGenUtil.Instance.Mangle(name)); + ns = new CodeNamespace(CodeGenUtil.Instance.Mangle(name)); foreach (CodeNamespaceImport nci in CodeGenUtil.Instance.NamespaceImports) { @@ -1158,5 +1183,48 @@ public virtual void WriteTypes(string outputdir) } } } + + /// + /// Replace namespace(s) in schema or protocol definition. + /// + /// input schema or protocol definition. + /// namespace mappings object. + private static string ReplaceMappedNamespacesInSchema(string input, IEnumerable> namespaceMapping) + { + if (namespaceMapping == null || input == null) + return input; + + // Replace namespace in "namespace" definitions: + // "namespace": "originalnamespace" -> "namespace": "mappednamespace" + // "namespace": "originalnamespace.whatever" -> "namespace": "mappednamespace.whatever" + // Note: It keeps the original whitespaces + return Regex.Replace(input, @"""namespace""(\s*):(\s*)""([^""]*)""", m => + { + // m.Groups[1]: whitespaces before ':' + // m.Groups[2]: whitespaces after ':' + // m.Groups[3]: the namespace + + string ns = m.Groups[3].Value; + + foreach (var mapping in namespaceMapping) + { + // Full match + if (mapping.Key == ns) + { + ns = mapping.Value; + break; + } + else + // Partial match + if (ns.StartsWith($"{mapping.Key}.")) + { + ns = $"{mapping.Value}.{ns.Substring(mapping.Key.Length + 1)}"; + break; + } + } + + return $@"""namespace""{m.Groups[1].Value}:{m.Groups[2].Value}""{ns}"""; + }); + } } } diff --git a/lang/csharp/src/apache/test/AvroGen/AvroGenTests.cs b/lang/csharp/src/apache/test/AvroGen/AvroGenTests.cs index 93e453190ad..2426bc27519 100644 --- a/lang/csharp/src/apache/test/AvroGen/AvroGenTests.cs +++ b/lang/csharp/src/apache/test/AvroGen/AvroGenTests.cs @@ -261,6 +261,51 @@ class AvroGenTests ] }"; + // https://issues.apache.org/jira/browse/AVRO-2883 + private const string _schema_avro_2883 = @" +{ + ""type"" : ""record"", + ""name"" : ""TestModel"", + ""namespace"" : ""my.avro.ns"", + ""fields"" : [ { + ""name"" : ""eventType"", + ""type"" : { + ""type"" : ""enum"", + ""name"" : ""EventType"", + ""symbols"" : [ ""CREATE"", ""UPDATE"", ""DELETE"" ] + } +} ] +}"; + + // https://issues.apache.org/jira/browse/AVRO-3046 + private const string _schema_avro_3046 = @" +{ + ""type"": ""record"", + ""name"": ""ExampleRecord"", + ""namespace"": ""com.example"", + ""fields"": [ + { + ""name"": ""Id"", + ""type"": ""string"", + ""logicalType"": ""UUID"" + }, + { + ""name"": ""InnerRecord"", + ""type"": { + ""type"": ""record"", + ""name"": ""InnerRecord"", + ""fields"": [ + { + ""name"": ""Id"", + ""type"": ""string"", + ""logicalType"": ""UUID"" + } + ] + } + } + ] +}"; + private Assembly TestSchema( string schema, IEnumerable typeNamesToCheck = null, @@ -417,6 +462,18 @@ private Assembly TestSchema( { "org/apache/avro/codegentest/testdata/NullableLogicalTypesArray.cs" })] + [TestCase( + _schema_avro_2883, + new string[] + { + "my.avro.ns.TestModel", + "my.avro.ns.EventType", + }, + new string[] + { + "my/avro/ns/TestModel.cs", + "my/avro/ns/EventType.cs" + })] public void GenerateSchema(string schema, IEnumerable typeNamesToCheck, IEnumerable generatedFilesToCheck) { TestSchema(schema, typeNamesToCheck, generatedFilesToCheck: generatedFilesToCheck); @@ -433,6 +490,45 @@ public void GenerateSchema(string schema, IEnumerable typeNamesToCheck, { "org/apache/csharp/codegentest/testdata/NullableLogicalTypesArray.cs" })] + [TestCase( + _nestedLogicalTypesUnion, + "org.apache.avro.codegentest.testdata", "org.apache.csharp.codegentest.testdata", + new string[] + { + "org.apache.csharp.codegentest.testdata.NestedLogicalTypesUnion", + "org.apache.csharp.codegentest.testdata.RecordInUnion" + }, + new string[] + { + "org/apache/csharp/codegentest/testdata/NestedLogicalTypesUnion.cs", + "org/apache/csharp/codegentest/testdata/RecordInUnion.cs" + })] + [TestCase( + _schema_avro_2883, + "my.avro.ns", "my.csharp.ns", + new string[] + { + "my.csharp.ns.TestModel", + "my.csharp.ns.EventType", + }, + new string[] + { + "my/csharp/ns/TestModel.cs", + "my/csharp/ns/EventType.cs" + })] + [TestCase( + _schema_avro_3046, + "com.example", "Example", + new string[] + { + "Example.ExampleRecord", + "Example.InnerRecord", + }, + new string[] + { + "Example/ExampleRecord.cs", + "Example/InnerRecord.cs" + })] [TestCase( _nullableLogicalTypesArray, "org.apache.avro.codegentest.testdata", "org.apache.@return.@int", // Reserved keywords in namespace @@ -497,33 +593,6 @@ public void GenerateSchemaWithNamespaceMapping( TestSchema(schema, typeNamesToCheck, new Dictionary { { namespaceMappingFrom, namespaceMappingTo } }, generatedFilesToCheck); } - [TestCase( - _nestedLogicalTypesUnion, - "org.apache.avro.codegentest.testdata", "org.apache.csharp.codegentest.testdata", - new string[] - { - "org.apache.avro.codegentest.testdata.NestedLogicalTypesUnion", - "org.apache.avro.codegentest.testdata.RecordInUnion" - }, - new string[] - { - "org/apache/csharp/codegentest/testdata/NestedLogicalTypesUnion.cs", - "org/apache/csharp/codegentest/testdata/RecordInUnion.cs" - })] - public void GenerateSchemaWithNamespaceMapping_Bug_AVRO_2883( - string schema, - string namespaceMappingFrom, - string namespaceMappingTo, - IEnumerable typeNamesToCheck, - IEnumerable generatedFilesToCheck) - { - // !!! This is a bug which must be fixed - // !!! Once it is fixed, this test will fail and this test can be removed - // https://issues.apache.org/jira/browse/AVRO-2883 - // https://issues.apache.org/jira/browse/AVRO-3046 - Assert.Throws(() => TestSchema(schema, typeNamesToCheck, new Dictionary { { namespaceMappingFrom, namespaceMappingTo } }, generatedFilesToCheck)); - } - [TestCase(_logicalTypesWithCustomConversion, typeof(AvroTypeException))] [TestCase(_customConversionWithLogicalTypes, typeof(SchemaParseException))] [TestCase(_nestedLogicalTypesUnionFixedDecimal, typeof(SchemaParseException))] From 9556eb461e03219998fe39246a1cdc3f0d861e1d Mon Sep 17 00:00:00 2001 From: yanivru Date: Tue, 19 Apr 2022 21:27:30 +0300 Subject: [PATCH 29/66] AVRO-2211: SchemaBuilder equivalent or other means of schema creation (#1597) * AVRO-2211: Support schema creation * Add license info to new files * Fix documentation for FixedSchema ctor * Remove and sort using * Add missing brackets and replace var with explicit type * Fix exception type in case of parsing * Rename field to follow conventions * AVRO 2211: Inlining temporary variable in linq * AVRO-2211: Change exception type and add missing documentations * AVRO-2211: Fix RecordSchema to set the positions of it's fields, instead of verifying it * AVRO-2211: Fix RecordSchema fields assignment when creation new RecordSchema * AVRO-2211: Change constructors of schema classes to factory method * AVRO-2211: Add unit tests for RecordSchema and EnumSchema * :AVRO-2211: Remove whitespace * :AVRO-2211: Add symbol names verification for EnumSchema * AVRO-2211: Fix enum name validation * AVRO-2211: Throw AvroException consistently * AVRO-2211: Throw AvroException in RecrodSchema consistently * AVRO-2211: Remove duplicate factory methods on MapSchema * AVRO-2211: Remove redundant parameter doc * AVRO-2211: Add Schema creation tests * AVRO-2211: Change ValidateSymbol to throw exception * AVRO-2211: Fix typo * AVRO-2211: Fix code QL issues * AVRO-2211: Fix typo Co-authored-by: Martin Grigorov --- lang/csharp/src/apache/main/AssemblyInfo.cs | 20 ++ lang/csharp/src/apache/main/Schema/Aliases.cs | 36 +++ .../src/apache/main/Schema/ArraySchema.cs | 22 +- .../src/apache/main/Schema/EnumSchema.cs | 75 ++++- lang/csharp/src/apache/main/Schema/Field.cs | 34 ++- .../src/apache/main/Schema/FixedSchema.cs | 14 + .../src/apache/main/Schema/MapSchema.cs | 12 +- .../src/apache/main/Schema/PrimitiveSchema.cs | 16 +- .../src/apache/main/Schema/RecordSchema.cs | 144 +++++++++- .../src/apache/main/Schema/UnionSchema.cs | 37 ++- lang/csharp/src/apache/test/Avro.test.csproj | 2 + .../src/apache/test/Schema/AliasesTests.cs | 49 ++++ .../src/apache/test/Schema/SchemaTests.cs | 259 +++++++++++++++++- 13 files changed, 675 insertions(+), 45 deletions(-) create mode 100644 lang/csharp/src/apache/main/AssemblyInfo.cs create mode 100644 lang/csharp/src/apache/main/Schema/Aliases.cs create mode 100644 lang/csharp/src/apache/test/Schema/AliasesTests.cs diff --git a/lang/csharp/src/apache/main/AssemblyInfo.cs b/lang/csharp/src/apache/main/AssemblyInfo.cs new file mode 100644 index 00000000000..53eacc1f9df --- /dev/null +++ b/lang/csharp/src/apache/main/AssemblyInfo.cs @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("Avro.test, PublicKey=00240000048000009400000006020000002400005253413100040000010001001145636d1b96168c2781abfd60478f45d010fe83dd0f318404cbf67252bca8cd827f24648d47ff682f35e60307c05d3cd89f0b063729cf8d2ebe6510b9e7d295dec6707ec91719d859458981f7ca1cbbea79b702b2fb64d1dbf0881887315345b70fa50fcf91b59e6a937c8d23919d409ee2f1f234cc4c8dbf5a29d3d670f3c9")] diff --git a/lang/csharp/src/apache/main/Schema/Aliases.cs b/lang/csharp/src/apache/main/Schema/Aliases.cs new file mode 100644 index 00000000000..6574e3163d6 --- /dev/null +++ b/lang/csharp/src/apache/main/Schema/Aliases.cs @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +using System.Collections.Generic; +using System.Linq; + +namespace Avro +{ + internal static class Aliases + { + internal static IList GetSchemaNames(IEnumerable aliases, string enclosingTypeName, string enclosingTypeNamespace) + { + if (aliases == null) + { + return null; + } + + SchemaName enclosingSchemaName = new SchemaName(enclosingTypeName, enclosingTypeNamespace, null, null); + return aliases.Select(alias => new SchemaName(alias, enclosingSchemaName.Namespace, null, null)).ToList(); + } + } +} diff --git a/lang/csharp/src/apache/main/Schema/ArraySchema.cs b/lang/csharp/src/apache/main/Schema/ArraySchema.cs index 5b4e6a434e0..03a331289aa 100644 --- a/lang/csharp/src/apache/main/Schema/ArraySchema.cs +++ b/lang/csharp/src/apache/main/Schema/ArraySchema.cs @@ -29,7 +29,7 @@ public class ArraySchema : UnnamedSchema /// /// Schema for the array 'type' attribute /// - public Schema ItemSchema { get; set; } + public Schema ItemSchema { get; set; } /// /// Static class to return a new instance of ArraySchema @@ -48,11 +48,23 @@ internal static ArraySchema NewInstance(JToken jtok, PropertyMap props, SchemaNa } /// - /// Constructor + /// Creates a new /// - /// schema for the array items type - /// dictionary that provides access to custom properties - private ArraySchema(Schema items, PropertyMap props) : base(Type.Array, props) + /// Schema for the array items type + /// Dictionary that provides access to custom properties + /// + public static ArraySchema Create(Schema items, PropertyMap customAttributes = null) + { + return new ArraySchema(items, customAttributes); + } + + /// + /// Initializes a new instance of the class. + /// + /// Schema for the array items type + /// Dictionary that provides access to custom properties + private ArraySchema(Schema items, PropertyMap customAttributes) + : base(Type.Array, customAttributes) { if (null == items) throw new ArgumentNullException(nameof(items)); this.ItemSchema = items; diff --git a/lang/csharp/src/apache/main/Schema/EnumSchema.cs b/lang/csharp/src/apache/main/Schema/EnumSchema.cs index f128790d1f7..295b73ebdc6 100644 --- a/lang/csharp/src/apache/main/Schema/EnumSchema.cs +++ b/lang/csharp/src/apache/main/Schema/EnumSchema.cs @@ -17,7 +17,8 @@ */ using System; using System.Collections.Generic; -using System.Text; +using System.Linq; +using System.Text.RegularExpressions; using Newtonsoft.Json.Linq; namespace Avro @@ -30,7 +31,7 @@ public class EnumSchema : NamedSchema /// /// List of strings representing the enum symbols /// - public IList Symbols { get; private set; } + public IList Symbols { get; private set; } /// /// The default token to use when deserializing an enum when the provided token is not found @@ -47,6 +48,34 @@ public class EnumSchema : NamedSchema /// public int Count { get { return Symbols.Count; } } + /// + /// Initializes a new instance of the class. + /// + /// Name of enum + /// Namespace of enum + /// List of aliases for the name + /// List of enum symbols + /// Custom properties on this schema + /// Documentation for this named schema + /// + public static EnumSchema Create(string name, + IEnumerable symbols, + string space = null, + IEnumerable aliases = null, + PropertyMap customProperties = null, + string doc = null, + string defaultSymbol = null) + { + return new EnumSchema(new SchemaName(name, space, null, doc), + Aliases.GetSchemaNames(aliases, name, space), + symbols.ToList(), + CreateSymbolsMap(symbols), + customProperties, + new SchemaNames(), + doc, + defaultSymbol); + } + /// /// Static function to return new instance of EnumSchema /// @@ -81,7 +110,7 @@ internal static EnumSchema NewInstance(JToken jtok, PropertyMap props, SchemaNam return new EnumSchema(name, aliases, symbols, symbolMap, props, names, JsonHelper.GetOptionalString(jtok, "doc"), JsonHelper.GetOptionalString(jtok, "default")); } - catch (SchemaParseException e) + catch (AvroException e) { throw new SchemaParseException($"{e.Message} at '{jtok.Path}'", e); } @@ -103,15 +132,49 @@ private EnumSchema(SchemaName name, IList aliases, List symb string doc, string defaultSymbol) : base(Type.Enumeration, name, aliases, props, names, doc) { - if (null == name.Name) throw new SchemaParseException("name cannot be null for enum schema."); + if (null == name.Name) throw new AvroException("name cannot be null for enum schema."); this.Symbols = symbols; this.symbolMap = symbolMap; if (null != defaultSymbol && !symbolMap.ContainsKey(defaultSymbol)) - throw new SchemaParseException($"Default symbol: {defaultSymbol} not found in symbols"); + throw new AvroException($"Default symbol: {defaultSymbol} not found in symbols"); Default = defaultSymbol; } + /// + /// Creates symbols map from specified list of symbols. + /// Symbol map contains the names of the symbols and their index. + /// + /// List of symbols + /// Symbol map + /// Is thrown if the symbols list contains invalid symbol name or duplicate symbols + private static IDictionary CreateSymbolsMap(IEnumerable symbols) + { + IDictionary symbolMap = new Dictionary(); + int i = 0; + foreach (var symbol in symbols) + { + ValidateSymbolName(symbol); + + if (symbolMap.ContainsKey(symbol)) + { + throw new AvroException($"Duplicate symbol: {symbol}"); + } + + symbolMap[symbol] = i++; + } + + return symbolMap; + } + + private static void ValidateSymbolName(string symbol) + { + if(string.IsNullOrEmpty(symbol) || !Regex.IsMatch(symbol, "^([A-Za-z_][A-Za-z0-9_]*)$")) + { + throw new AvroException($"Invalid symbol name: {symbol}"); + } + } + /// /// Writes enum schema in JSON format /// @@ -127,7 +190,7 @@ protected internal override void WriteJsonFields(Newtonsoft.Json.JsonTextWriter foreach (string s in this.Symbols) writer.WriteValue(s); writer.WriteEndArray(); - if (null != Default) + if (null != Default) { writer.WritePropertyName("default"); writer.WriteValue(Default); diff --git a/lang/csharp/src/apache/main/Schema/Field.cs b/lang/csharp/src/apache/main/Schema/Field.cs index bdfe9282cb3..08ea03305d7 100644 --- a/lang/csharp/src/apache/main/Schema/Field.cs +++ b/lang/csharp/src/apache/main/Schema/Field.cs @@ -103,7 +103,39 @@ public enum SortOrder /// /// Static comparer object for JSON objects such as the fields default value /// - internal static JTokenEqualityComparer JtokenEqual = new JTokenEqualityComparer(); + internal readonly static JTokenEqualityComparer JtokenEqual = new JTokenEqualityComparer(); + + /// + /// Initializes a new instance of the class. + /// + /// schema for the field type. + /// name of the field. + /// list of aliases for the name of the field. + /// position of the field. + /// documentation for the field. + /// field's default value if it exists. + /// sort order of the field. + /// dictionary that provides access to custom properties. + public Field(Schema schema, + string name, + int pos, + IList aliases = null, + string doc = null, + JToken defaultValue = null, + SortOrder sortorder = SortOrder.ignore, + PropertyMap customProperties = null) + : this(schema, name, aliases, pos, doc, defaultValue, sortorder, customProperties) + { + } + + /// + /// Creates a new field based on the specified field, with a different position. + /// + /// A clone of this field with new position. + internal Field ChangePosition(int newPosition) + { + return new Field(Schema, Name, newPosition, Aliases, Documentation, DefaultValue, Ordering ?? SortOrder.ignore, Props); + } /// /// A flag to indicate if reader schema has a field that is missing from writer schema and has a default value diff --git a/lang/csharp/src/apache/main/Schema/FixedSchema.cs b/lang/csharp/src/apache/main/Schema/FixedSchema.cs index b16c1ff1dcb..2b24e6b8689 100644 --- a/lang/csharp/src/apache/main/Schema/FixedSchema.cs +++ b/lang/csharp/src/apache/main/Schema/FixedSchema.cs @@ -32,6 +32,20 @@ public class FixedSchema : NamedSchema /// public int Size { get; set; } + /// + /// Initializes a new instance of the class. + /// + /// Name of the fixed schema + /// List of aliases for the name + /// Fixed size + /// Namespace of fixed + /// Custom properties on this schema + /// Documentation for this named schema + public static FixedSchema Create(string name, int size, string space = null, IEnumerable aliases = null, PropertyMap customProperties = null, string doc = null) + { + return new FixedSchema(new SchemaName(name, space, null, doc), Aliases.GetSchemaNames(aliases, name, space), size, customProperties, new SchemaNames(), doc); + } + /// /// Static function to return new instance of the fixed schema class /// diff --git a/lang/csharp/src/apache/main/Schema/MapSchema.cs b/lang/csharp/src/apache/main/Schema/MapSchema.cs index 54bc05a8d31..d9f4995abf6 100644 --- a/lang/csharp/src/apache/main/Schema/MapSchema.cs +++ b/lang/csharp/src/apache/main/Schema/MapSchema.cs @@ -36,10 +36,11 @@ public class MapSchema : UnnamedSchema /// Creates a new from the given schema. /// /// Schema to create the map schema from. + /// Dictionary that provides access to custom properties /// A new . - public static MapSchema CreateMap(Schema type) + public static MapSchema CreateMap(Schema type, PropertyMap customProperties = null) { - return new MapSchema(type,null); + return new MapSchema(type, customProperties); } /// @@ -67,9 +68,10 @@ internal static MapSchema NewInstance(JToken jtok, PropertyMap props, SchemaName /// /// Constructor for map schema class /// - /// schema for map values type - /// dictionary that provides access to custom properties - private MapSchema(Schema valueSchema, PropertyMap props) : base(Type.Map, props) + /// Schema for map values type + /// Dictionary that provides access to custom properties + private MapSchema(Schema valueSchema, PropertyMap cutsomProperties) + : base(Type.Map, cutsomProperties) { if (null == valueSchema) throw new ArgumentNullException(nameof(valueSchema), "valueSchema cannot be null."); this.ValueSchema = valueSchema; diff --git a/lang/csharp/src/apache/main/Schema/PrimitiveSchema.cs b/lang/csharp/src/apache/main/Schema/PrimitiveSchema.cs index 23c2cee7cc2..db5db2cb03e 100644 --- a/lang/csharp/src/apache/main/Schema/PrimitiveSchema.cs +++ b/lang/csharp/src/apache/main/Schema/PrimitiveSchema.cs @@ -30,9 +30,21 @@ public sealed class PrimitiveSchema : UnnamedSchema /// Constructor for primitive schema /// /// - /// dictionary that provides access to custom properties - private PrimitiveSchema(Type type, PropertyMap props) : base(type, props) + /// dictionary that provides access to custom properties + private PrimitiveSchema(Type type, PropertyMap customProperties) + : base(type, customProperties) + { + } + + /// + /// Creates a new instance of + /// + /// The primitive type to create + /// Dictionary that provides access to custom properties + /// + public static PrimitiveSchema Create(Type type, PropertyMap customProperties = null) { + return new PrimitiveSchema(type, customProperties); } /// diff --git a/lang/csharp/src/apache/main/Schema/RecordSchema.cs b/lang/csharp/src/apache/main/Schema/RecordSchema.cs index 356f8baf4a2..910bc466fe9 100644 --- a/lang/csharp/src/apache/main/Schema/RecordSchema.cs +++ b/lang/csharp/src/apache/main/Schema/RecordSchema.cs @@ -17,6 +17,7 @@ */ using System; using System.Collections.Generic; +using System.Linq; using Newtonsoft.Json.Linq; namespace Avro @@ -28,10 +29,26 @@ namespace Avro /// public class RecordSchema : NamedSchema { + private List _fields; + /// /// List of fields in the record /// - public List Fields { get; private set; } + public List Fields + { + get + { + return _fields; + } + + set + { + _fields = SetFieldsPositions(value); + + fieldLookup = CreateFieldMap(_fields); + fieldAliasLookup = CreateFieldMap(_fields, true); + } + } /// /// Number of fields in the record @@ -41,10 +58,109 @@ public class RecordSchema : NamedSchema /// /// Map of field name and Field object for faster field lookups /// - private readonly IDictionary fieldLookup; + private IDictionary fieldLookup; - private readonly IDictionary fieldAliasLookup; - private bool request; + private IDictionary fieldAliasLookup; + private readonly bool request; + + /// + /// Creates a new instance of + /// + /// name of the record schema + /// list of fields for the record + /// type of record schema, either record or error + /// list of aliases for the record name + /// custom properties on this schema + /// documentation for this named schema + public static RecordSchema Create(string name, + List fields, + string space = null, + IEnumerable aliases = null, + PropertyMap customProperties = null, + string doc = null) + { + return new RecordSchema(Type.Record, + new SchemaName(name, space, null, doc), + Aliases.GetSchemaNames(aliases, name, space), + customProperties, + fields, + false, + CreateFieldMap(fields), + CreateFieldMap(fields, true), + new SchemaNames(), + doc); + } + + private static IEnumerable EnumerateSchemasRecursive(Schema schema) + { + yield return schema; + switch (schema.Tag) + { + case Type.Null: + break; + case Type.Boolean: + break; + case Type.Int: + break; + case Type.Long: + break; + case Type.Float: + break; + case Type.Double: + break; + case Type.Bytes: + break; + case Type.String: + break; + case Type.Record: + var recordSchema = (RecordSchema)schema; + recordSchema.Fields.SelectMany(f => EnumerateSchemasRecursive(f.Schema)); + break; + case Type.Enumeration: + break; + case Type.Array: + var arraySchema = (ArraySchema)schema; + EnumerateSchemasRecursive(arraySchema.ItemSchema); + break; + case Type.Map: + var mapSchema = (MapSchema)schema; + EnumerateSchemasRecursive(mapSchema.ValueSchema); + break; + case Type.Union: + var unionSchema = (UnionSchema)schema; + foreach (var innerSchema in unionSchema.Schemas) + { + EnumerateSchemasRecursive(innerSchema); + } + break; + case Type.Fixed: + break; + case Type.Error: + break; + case Type.Logical: + break; + } + } + + private static IDictionary CreateFieldMap(List fields, bool includeAliases = false) + { + var map = new Dictionary(); + if (fields != null) + { + foreach (Field field in fields) + { + addToFieldMap(map, field.Name, field); + + if (includeAliases && field.Aliases != null) + { + foreach (var alias in field.Aliases) + addToFieldMap(map, alias, field); + } + } + } + + return map; + } /// /// Static function to return new instance of the record schema @@ -99,8 +215,10 @@ internal static RecordSchema NewInstance(Type type, JToken jtok, PropertyMap pro if (null != field.Aliases) // add aliases to field lookup map so reader function will find it when writer field name appears only as an alias on the reader field foreach (string alias in field.Aliases) addToFieldMap(fieldAliasMap, alias, field); + + result._fields = fields; } - catch (SchemaParseException e) + catch (AvroException e) { throw new SchemaParseException($"{e.Message} at '{jfield.Path}'", e); } @@ -121,7 +239,7 @@ internal static RecordSchema NewInstance(Type type, JToken jtok, PropertyMap pro /// map of field aliases and field objects /// list of named schema already read /// documentation for this named schema - private RecordSchema(Type type, SchemaName name, IList aliases, PropertyMap props, + private RecordSchema(Type type, SchemaName name, IList aliases, PropertyMap props, List fields, bool request, IDictionary fieldMap, IDictionary fieldAliasMap, SchemaNames names, string doc) : base(type, name, aliases, props, names, doc) @@ -149,7 +267,7 @@ private static Field createField(JToken jfield, int pos, SchemaNames names, stri var jorder = JsonHelper.GetOptionalString(jfield, "order"); Field.SortOrder sortorder = Field.SortOrder.ignore; if (null != jorder) - sortorder = (Field.SortOrder) Enum.Parse(typeof(Field.SortOrder), jorder); + sortorder = (Field.SortOrder)Enum.Parse(typeof(Field.SortOrder), jorder); var aliases = Field.GetAliases(jfield); var props = Schema.GetProperties(jfield); @@ -165,10 +283,20 @@ private static Field createField(JToken jfield, int pos, SchemaNames names, stri private static void addToFieldMap(Dictionary map, string name, Field field) { if (map.ContainsKey(name)) - throw new SchemaParseException("field or alias " + name + " is a duplicate name"); + throw new AvroException("field or alias " + name + " is a duplicate name"); map.Add(name, field); } + /// + /// Clones the fields with updated positions. Updates the positions according to the order of the fields in the list. + /// + /// List of fields + /// New list of cloned fields with updated positions + private List SetFieldsPositions(List fields) + { + return fields.Select((field, i) => field.ChangePosition(i)).ToList(); + } + /// /// Returns the field with the given name. /// diff --git a/lang/csharp/src/apache/main/Schema/UnionSchema.cs b/lang/csharp/src/apache/main/Schema/UnionSchema.cs index e74fa954159..7e1575684dd 100644 --- a/lang/csharp/src/apache/main/Schema/UnionSchema.cs +++ b/lang/csharp/src/apache/main/Schema/UnionSchema.cs @@ -17,9 +17,8 @@ */ using System; using System.Collections.Generic; -using System.Text; +using System.Linq; using Newtonsoft.Json.Linq; -using Newtonsoft.Json; namespace Avro { @@ -68,14 +67,27 @@ internal static UnionSchema NewInstance(JArray jarr, PropertyMap props, SchemaNa } /// - /// Constructor for union schema + /// Creates a new + /// + /// The union schemas + /// Dictionary that provides access to custom properties + /// New + public static UnionSchema Create(List schemas, PropertyMap customProperties = null) + { + return new UnionSchema(schemas, customProperties); + } + + /// + /// Contructor for union schema /// /// - /// dictionary that provides access to custom properties - private UnionSchema(List schemas, PropertyMap props) : base(Type.Union, props) + /// dictionary that provides access to custom properties + private UnionSchema(List schemas, PropertyMap customProperties) + : base(Type.Union, customProperties) { if (schemas == null) throw new ArgumentNullException(nameof(schemas)); + VerifyChildSchemas(schemas); this.Schemas = schemas; } @@ -161,5 +173,20 @@ public override int GetHashCode() result += getHashCode(Props); return result; } + + private void VerifyChildSchemas(List schemas) + { + if (schemas.Any(schema => schema.Tag == Type.Union)) + { + throw new ArgumentException("Unions may not immediately contain other unions", nameof(schemas)); + } + + IGrouping duplicateType = schemas.GroupBy(schema => schema.Fullname).FirstOrDefault(x => x.Count() > 1); + + if (duplicateType != null) + { + throw new ArgumentException($"Duplicate type in union: {duplicateType.Key}"); + } + } } } diff --git a/lang/csharp/src/apache/test/Avro.test.csproj b/lang/csharp/src/apache/test/Avro.test.csproj index 24fa4b31fc6..3ba3a0ffa89 100644 --- a/lang/csharp/src/apache/test/Avro.test.csproj +++ b/lang/csharp/src/apache/test/Avro.test.csproj @@ -24,6 +24,8 @@ Avro.test false false + True + ..\..\..\Avro.snk diff --git a/lang/csharp/src/apache/test/Schema/AliasesTests.cs b/lang/csharp/src/apache/test/Schema/AliasesTests.cs new file mode 100644 index 00000000000..27ad4b23efd --- /dev/null +++ b/lang/csharp/src/apache/test/Schema/AliasesTests.cs @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +using NUnit.Framework; + +namespace Avro.Test +{ + [TestFixture] + public class AliasesTests + { + [TestCase] + public void TestNoNamespace() + { + CollectionAssert.AreEqual(new[] { new SchemaName("alias", null, null, null) }, Aliases.GetSchemaNames(new[] { "alias" }, "name", null)); + } + + [TestCase] + public void TestTypeWithNamespace() + { + CollectionAssert.AreEqual(new[] { new SchemaName("space.alias", null, null, null) }, Aliases.GetSchemaNames(new[] { "alias" }, "name", "space")); + } + + [TestCase] + public void TestTypeWithNamespaceInName() + { + CollectionAssert.AreEqual(new[] { new SchemaName("space.alias", null, null, null) }, Aliases.GetSchemaNames(new[] { "alias" }, "space.name", null)); + } + + [TestCase] + public void TestAliasWithNamespace() + { + CollectionAssert.AreEqual(new[] { new SchemaName("name.alias", null, null, null) }, Aliases.GetSchemaNames(new[] { "name.alias" }, "space.name", null)); + } + } +} diff --git a/lang/csharp/src/apache/test/Schema/SchemaTests.cs b/lang/csharp/src/apache/test/Schema/SchemaTests.cs index a061d53731e..54194b7b5a8 100644 --- a/lang/csharp/src/apache/test/Schema/SchemaTests.cs +++ b/lang/csharp/src/apache/test/Schema/SchemaTests.cs @@ -16,7 +16,9 @@ * limitations under the License. */ using System; +using System.Collections.Generic; using NUnit.Framework; +using System.Linq; namespace Avro.Test { @@ -138,8 +140,9 @@ public void TestBasic(string s, Type expectedExceptionType = null) public void TestPrimitive(string s, Schema.Type type) { Schema sc = Schema.Parse(s); - Assert.IsTrue(sc is PrimitiveSchema); - Assert.AreEqual(type, sc.Tag); + Schema schema = PrimitiveSchema.Create(type, null); + + Assert.AreEqual(sc, schema); testEquality(s, sc); testToString(sc); @@ -288,13 +291,133 @@ public void TestRecordDoc(string s, string expectedDoc) Assert.AreEqual(expectedDoc, roundTrip.Documentation); } - [TestCase("{\"type\": \"enum\", \"name\": \"Test\", \"symbols\": [\"A\", \"B\"]}", + [TestCase("{\"type\":\"record\",\"name\":\"Longs\",\"fields\":[{\"name\":\"value\",\"default\":\"100\",\"type\":\"long\",\"aliases\":[\"oldName\"]}]}", + "Longs", null, null, null, + new[] { "value" }, new[] { Schema.Type.Long }, new[] { "100" }, new[] { "oldName" }, new string[] { null })] + [TestCase("{\"type\":\"record\",\"name\":\"Longs\",\"fields\":[{\"name\":\"value\",\"doc\":\"Field With Documentation\",\"default\":\"100\",\"type\":\"long\",\"aliases\":[\"oldName\"]}]}", + "Longs", null, null, null, + new[] { "value" }, new[] { Schema.Type.Long }, new[] { "100" }, new[] { "oldName" }, new string[] { "Field With Documentation" })] + [TestCase("{\"type\":\"record\",\"name\":\"Longs\",\"namespace\":\"space\",\"fields\":[{\"name\":\"value\",\"default\":\"100\",\"type\":\"long\",\"aliases\":[\"oldName\"]}]}", + "Longs", "space", null, null, + new[] { "value" }, new[] { Schema.Type.Long }, new[] { "100" }, new[] { "oldName" }, new string[] { null })] + [TestCase("{\"type\":\"record\",\"name\":\"Longs\",\"doc\":\"Record with alias\",\"namespace\":\"space\",\"aliases\":[\"space.RecordAlias\"],\"fields\":[{\"name\":\"value\",\"default\":\"100\",\"type\":\"long\",\"aliases\":[\"oldName\"]}]}", + "Longs", "space", "RecordAlias", "Record with alias", + new[] { "value" }, new[] { Schema.Type.Long }, new[] { "100" }, new[] { "oldName" }, new string[] { null })] + [TestCase("{\"type\":\"record\",\"name\":\"Longs\",\"doc\":\"Record with two fields\",\"namespace\":\"space\",\"aliases\":[\"space.RecordAlias\"],\"fields\":[{\"name\":\"value\",\"doc\":\"first field\",\"default\":\"100\",\"type\":\"long\",\"aliases\":[\"oldName\"]},{\"name\":\"field2\",\"default\":\"true\",\"type\":\"boolean\"}]}", + "Longs", "space", "RecordAlias", "Record with two fields", + new[] { "value", "field2" }, new[] { Schema.Type.Long, Schema.Type.Boolean }, new[] { "100", "true" }, new[] { "oldName", null }, new string[] { "first field", null })] + public void TestRecordCreation(string expectedSchema, string name, string space, string alias, string documentation, string[] fieldsNames, Schema.Type[] fieldsTypes, object[] fieldsDefaultValues, string[] fieldsAliases, string[] fieldsDocs) + { + IEnumerable recordFields = fieldsNames.Select((fieldName, i) => new Field(PrimitiveSchema.Create(fieldsTypes[i]), + fieldName, + fieldsAliases[i] == null? null: new List { fieldsAliases[i] }, + i, + fieldsDocs[i], + fieldsDefaultValues[i].ToString(), + Field.SortOrder.ignore, + null)); + + string[] aliases = alias == null ? null : new[] { alias }; + + RecordSchema recordSchema = RecordSchema.Create(name, recordFields.ToList(), space, aliases, null, documentation); + + for(int i = 0; i < fieldsNames.Length; i++) + { + var fieldByName = recordSchema[fieldsNames[i]]; + if (fieldsAliases[i] != null) + { + recordSchema.TryGetFieldAlias(fieldsAliases[i], out Field fieldByAlias); + + Assert.AreSame(fieldByAlias, fieldByName); + } + Assert.AreEqual(expectedSchema, recordSchema.ToString()); + Assert.AreEqual(fieldsNames[i], fieldByName.Name); + Assert.AreEqual(i, fieldByName.Pos); + Assert.AreEqual(fieldsTypes[i], fieldByName.Schema.Tag); + Assert.AreEqual(fieldsDocs[i], fieldByName.Documentation); + Assert.AreEqual(fieldsDefaultValues[i], fieldByName.DefaultValue.ToString()); + CollectionAssert.AreEqual(fieldsAliases[i] == null? null: new[] {fieldsAliases[i]}, fieldByName.Aliases); + } + } + + [TestCase] + public void TestRecordCreationWithDuplicateFields() + { + var recordField = new Field(PrimitiveSchema.Create(Schema.Type.Long), + "value", + new List { "oldName" }, + 0, + null, + "100", + Field.SortOrder.ignore, + null); + + Assert.Throws(() => RecordSchema.Create("Longs", + new List + { + recordField, + recordField + })); + } + + [TestCase] + public void TestRecordCreationWithRecursiveRecord() + { + string schema = "{\"type\":\"record\",\"name\":\"LongList\",\"aliases\":[\"LinkedLongs\"],\"fields\":[{\"name\":\"value\",\"type\":\"long\"},{\"name\":\"next\",\"type\":[\"null\",\"LongList\"]}]}"; + + var recordSchema = RecordSchema.Create("LongList", new List(), null, new[] { "LinkedLongs" }); + + recordSchema.Fields = new List + { + new Field(PrimitiveSchema.Create(Schema.Type.Long), + "value", + null, + 0, + null, + null, + Field.SortOrder.ignore, + null), + new Field(UnionSchema.Create( + new List + { + PrimitiveSchema.Create(Schema.Type.Null), recordSchema + }), + "next", + 1) + }; + + Assert.AreEqual(schema, recordSchema.ToString()); + } + + [TestCase("{\"type\":\"enum\",\"name\":\"Test\",\"symbols\":[\"A\",\"B\"]}", + new string[] { "A", "B" })] + + [TestCase("{\"type\":\"enum\",\"name\":\"Test\",\"symbols\":[\"A\",\"B\"]}", new string[] { "A", "B" })] - [TestCase("{\"type\": \"enum\",\"name\":\"Market\",\"symbols\":[\"UNKNOWN\",\"A\",\"B\"],\"default\":\"UNKNOWN\"}", - new string[] { "UNKNOWN", "A", "B" })] - public void TestEnum(string s, string[] symbols) + [TestCase("{\"type\":\"enum\",\"name\":\"Test\",\"doc\":\"Some explanation\",\"namespace\":\"mynamespace\",\"aliases\":[\"mynamespace.Alias\"],\"symbols\":[\"UNKNOWN\",\"A\",\"B\"],\"default\":\"UNKNOWN\",\"propertyKey\":\"propertyValue\"}", + new string[] { "UNKNOWN", "A", "B" }, "mynamespace", new string[] { "Alias" }, "Some explanation", true, "UNKNOWN")] + [TestCase("{\"type\":\"enum\",\"name\":\"Test\",\"doc\":\"Some explanation\",\"namespace\":\"space\",\"aliases\":[\"internalNamespace.Alias\"],\"symbols\":[\"UNKNOWN\",\"A\",\"B\"]}", + new string[] { "UNKNOWN", "A", "B" }, "space", new string[] { "internalNamespace.Alias" }, "Some explanation")] + [TestCase("{\"type\":\"enum\",\"name\":\"Test\",\"doc\":\"Some explanation\",\"namespace\":\"space\",\"aliases\":[\"internalNamespace.Alias\"],\"symbols\":[]}", + new string[] { }, "space", new string[] { "internalNamespace.Alias" }, "Some explanation")] + + public void TestEnum(string s, string[] symbols, string space = null, IEnumerable aliases = null, string doc = null, bool? usePropertyMap = null, string defaultSymbol = null) { Schema sc = Schema.Parse(s); + + PropertyMap propertyMap = new PropertyMap(); + propertyMap.Add("propertyKey", "\"propertyValue\""); + Schema schema = EnumSchema.Create("Test", + symbols, + space, + aliases, + usePropertyMap == true ? propertyMap : null, + doc, + defaultSymbol); + + Assert.AreEqual(sc, schema); + Assert.AreEqual(s, schema.ToString()); + Assert.AreEqual(Schema.Type.Enumeration, sc.Tag); EnumSchema es = sc as EnumSchema; Assert.AreEqual(symbols.Length, es.Count); @@ -333,18 +456,66 @@ public void TestEnumDefaultSymbolDoesntExist(string s) Assert.Throws(() => Schema.Parse(s)); } + [TestCase("name", new string[] { "A", "B" }, "s", new[] { "L1", "L2" }, "regular enum", null, "name", "s")] + [TestCase("s.name", new string[] { "A", "B" }, null, new[] { "L1", "L2" }, "internal namespace", null, "name", "s")] + [TestCase("name", new string[] { "A", "B" }, null, new[] { "L1", "L2" }, "no namespace", null, "name", null)] + [TestCase("name", new string[] { "A", "B" }, null, new[] { "L1", "L2" }, "with default value", "A", "name", null)] + [TestCase("name", new string[] { "A1B2", "B4324" }, null, new[] { "L1", "L2" }, "with longer symbols", "B4324", "name", null)] + [TestCase("name", new string[] { "_A1B2_", "B4324" }, null, new[] { "L1", "L2" }, "underscore in symbols", "_A1B2_", "name", null)] + public void TestEnumCreation(string name, string[] symbols, string space, string[] aliases, string doc, string defaultSymbol, string expectedName, string expectedNamespace) + { + EnumSchema enumSchema = EnumSchema.Create(name, symbols, space, aliases, null, doc, defaultSymbol); + + Assert.AreEqual(expectedName, enumSchema.Name); + CollectionAssert.AreEqual(symbols, enumSchema.Symbols); + Assert.AreEqual(expectedNamespace, enumSchema.Namespace); + Assert.AreEqual(Schema.Type.Enumeration, enumSchema.Tag); + Assert.AreEqual(doc, enumSchema.Documentation); + Assert.AreEqual(defaultSymbol, enumSchema.Default); + } + + [TestCase(new[] {"A", "B"}, "C")] + [TestCase(new[] {null, "B"}, null)] + [TestCase(new[] {"", "B" }, null)] + [TestCase(new[] {"8", "B" }, null)] + [TestCase(new[] {"8", "B" }, null)] + [TestCase(new[] {"A", "A" }, null)] + [TestCase(new[] {" ", "A" }, null)] + [TestCase(new[] {"9A23", "A" }, null)] + public void TestEnumInvalidSymbols(string[] symbols, string defaultSymbol) + { + Assert.Throws(() => EnumSchema.Create("name", symbols, defaultSymbol: defaultSymbol)); + } + [TestCase("{\"type\": \"array\", \"items\": \"long\"}", "long")] public void TestArray(string s, string item) { Schema sc = Schema.Parse(s); Assert.AreEqual(Schema.Type.Array, sc.Tag); - ArraySchema ars = sc as ArraySchema; + ArraySchema ars = (ArraySchema)sc; Assert.AreEqual(item, ars.ItemSchema.Name); testEquality(s, sc); testToString(sc); } + [TestCase] + public void TestArrayCreation() + { + PrimitiveSchema itemsSchema = PrimitiveSchema.Create(Schema.Type.String); + ArraySchema arraySchema = ArraySchema.Create(itemsSchema); + + Assert.AreEqual("array", arraySchema.Name); + Assert.AreEqual(Schema.Type.Array, arraySchema.Tag); + Assert.AreEqual(itemsSchema, arraySchema.ItemSchema); + } + + [TestCase] + public void TestInvalidArrayCreation() + { + Assert.Throws(() => ArraySchema.Create(null)); + } + [TestCase("{\"type\": \"int\", \"logicalType\": \"date\"}", "int", "date")] public void TestLogicalPrimitive(string s, string baseType, string logicalType) { @@ -371,33 +542,82 @@ public void TestMap(string s, string value) { Schema sc = Schema.Parse(s); Assert.AreEqual(Schema.Type.Map, sc.Tag); - MapSchema ms = sc as MapSchema; + MapSchema ms = (MapSchema)sc; Assert.AreEqual(value, ms.ValueSchema.Name); testEquality(s, sc); testToString(sc); } - [TestCase("[\"string\", \"null\", \"long\"]", new string[] { "string", "null", "long" })] - public void TestUnion(string s, string[] types) + [TestCase] + public void TestMapCreation() + { + PrimitiveSchema mapType = PrimitiveSchema.Create(Schema.Type.Float); + MapSchema mapSchema = MapSchema.CreateMap(mapType); + + Assert.AreEqual("map", mapSchema.Fullname); + Assert.AreEqual("map", mapSchema.Name); + Assert.AreEqual(Schema.Type.Map, mapSchema.Tag); + Assert.AreEqual(mapType, mapSchema.ValueSchema); + } + + [TestCase] + public void TestInvalidMapCreation() + { + Assert.Throws(() => MapSchema.CreateMap(null)); + } + + [TestCase("[\"string\", \"null\", \"long\"]", + new Schema.Type[] { Schema.Type.String, Schema.Type.Null, Schema.Type.Long })] + public void TestUnion(string s, Schema.Type[] types) { Schema sc = Schema.Parse(s); + + UnionSchema schema = UnionSchema.Create(types.Select(t => (Schema)PrimitiveSchema.Create(t)).ToList()); + Assert.AreEqual(sc, schema); + Assert.AreEqual(Schema.Type.Union, sc.Tag); - UnionSchema us = sc as UnionSchema; + UnionSchema us = (UnionSchema)sc; Assert.AreEqual(types.Length, us.Count); for (int i = 0; i < us.Count; i++) { - Assert.AreEqual(types[i], us[i].Name); + Assert.AreEqual(types[i].ToString().ToLower(), us[i].Name); } testEquality(s, sc); testToString(sc); } - [TestCase("{ \"type\": \"fixed\", \"name\": \"Test\", \"size\": 1}", 1)] + [TestCase] + public void TestUnionCreation() + { + UnionSchema unionSchema = UnionSchema.Create(new List { PrimitiveSchema.Create(Schema.Type.Null), PrimitiveSchema.Create(Schema.Type.String) }); + + CollectionAssert.AreEqual(new List { PrimitiveSchema.Create(Schema.Type.Null), PrimitiveSchema.Create(Schema.Type.String) }, + unionSchema.Schemas); + } + + [TestCase] + public void TestUnionCreationWithDuplicateSchemas() + { + Assert.Throws(() => UnionSchema.Create(new List { PrimitiveSchema.Create(Schema.Type.String), PrimitiveSchema.Create(Schema.Type.String) })); + } + + [TestCase] + public void TestUnionNestedUnionCreation() + { + Assert.Throws(() => UnionSchema.Create(new List { UnionSchema.Create(new List()), PrimitiveSchema.Create(Schema.Type.String) })); + } + + [TestCase("{\"type\":\"fixed\",\"name\":\"Test\",\"size\":1}", 1)] public void TestFixed(string s, int size) { Schema sc = Schema.Parse(s); + FixedSchema schema = FixedSchema.Create("Test", 1); + + Assert.AreEqual(sc, schema); + Assert.AreEqual(s, schema.ToString()); + Assert.AreEqual(Schema.Type.Fixed, sc.Tag); FixedSchema fs = sc as FixedSchema; Assert.AreEqual(size, fs.Size); @@ -415,6 +635,19 @@ public void TestFixedDoc(string s, string expectedDoc) Assert.AreEqual(expectedDoc, fs.Documentation); } + [TestCase] + public void TestFixedCreation() + { + string s = @"{""type"":""fixed"",""name"":""fixedName"",""namespace"":""space"",""aliases"":[""space.fixedOldName""],""size"":10}"; + + FixedSchema fixedSchema = FixedSchema.Create("fixedName", 10, "space", new[] { "fixedOldName" }, null); + + Assert.AreEqual("fixedName", fixedSchema.Name); + Assert.AreEqual("space.fixedName", fixedSchema.Fullname); + Assert.AreEqual(10, fixedSchema.Size); + Assert.AreEqual(s, fixedSchema.ToString()); + } + [TestCase("a", "o.a.h", ExpectedResult = "o.a.h.a")] public string testFullname(string s1, string s2) { From bfc51656f38f3648026355a5fc9a7c92ba003f31 Mon Sep 17 00:00:00 2001 From: Zoltan Csizmadia Date: Tue, 19 Apr 2022 13:28:33 -0500 Subject: [PATCH 30/66] AVRO-3841: Try exact schema match first in union type (#1635) * Try exact schema match * Fix formatting * Add tests for exception Co-authored-by: Zoltan Csizmadia --- .../src/apache/main/Schema/UnionSchema.cs | 17 ++++- .../src/apache/test/Generic/GenericTests.cs | 12 +++ .../test/Specific/DoubleLongUnionRecord.cs | 73 +++++++++++++++++++ .../src/apache/test/Specific/SpecificTests.cs | 32 ++++++++ 4 files changed, 132 insertions(+), 2 deletions(-) create mode 100644 lang/csharp/src/apache/test/Specific/DoubleLongUnionRecord.cs diff --git a/lang/csharp/src/apache/main/Schema/UnionSchema.cs b/lang/csharp/src/apache/main/Schema/UnionSchema.cs index 7e1575684dd..af9ba758363 100644 --- a/lang/csharp/src/apache/main/Schema/UnionSchema.cs +++ b/lang/csharp/src/apache/main/Schema/UnionSchema.cs @@ -127,8 +127,21 @@ public int MatchingBranch(Schema s) { if (s is UnionSchema) throw new AvroException("Cannot find a match against union schema"); // Try exact match. - //for (int i = 0; i < Count; i++) if (Schemas[i].Equals(s)) return i; // removed this for performance's sake - for (int i = 0; i < Count; i++) if (Schemas[i].CanRead(s)) return i; + // CanRead might find a compatible schema which can read. e.g. double and long + for (int i = 0; i < Count; i++) + { + if (Schemas[i].Equals(s)) + { + return i; + } + } + for (int i = 0; i < Count; i++) + { + if (Schemas[i].CanRead(s)) + { + return i; + } + } return -1; } diff --git a/lang/csharp/src/apache/test/Generic/GenericTests.cs b/lang/csharp/src/apache/test/Generic/GenericTests.cs index 8e0a86cdf3c..b87ce69f890 100644 --- a/lang/csharp/src/apache/test/Generic/GenericTests.cs +++ b/lang/csharp/src/apache/test/Generic/GenericTests.cs @@ -128,6 +128,18 @@ private static void test(string s, T value) [TestCase("[\"int\", \"long\"]", 100L)] [TestCase("[\"float\", \"double\"]", 100.75)] [TestCase("[\"float\", \"double\"]", 23.67f)] + [TestCase("[\"float\", \"int\"]", 0)] + [TestCase("[\"float\", \"int\"]", 0.0f)] + [TestCase("[\"float\", \"int\"]", 100)] + [TestCase("[\"float\", \"int\"]", 100.0f)] + [TestCase("[\"float\", \"int\"]", -100)] + [TestCase("[\"float\", \"int\"]", -100.0f)] + [TestCase("[\"double\", \"long\"]", 0L)] + [TestCase("[\"double\", \"long\"]", 0.0)] + [TestCase("[\"double\", \"long\"]", 100L)] + [TestCase("[\"double\", \"long\"]", 100.0)] + [TestCase("[\"double\", \"long\"]", -100L)] + [TestCase("[\"double\", \"long\"]", -100.0)] [TestCase("[{\"type\": \"array\", \"items\": \"float\"}, \"double\"]", new float[] { 23.67f, 22.78f })] [TestCase("[{\"type\": \"array\", \"items\": \"float\"}, \"double\"]", 100.89)] [TestCase("[{\"type\": \"array\", \"items\": \"string\"}, \"string\"]", "a")] diff --git a/lang/csharp/src/apache/test/Specific/DoubleLongUnionRecord.cs b/lang/csharp/src/apache/test/Specific/DoubleLongUnionRecord.cs new file mode 100644 index 00000000000..97b94be7eed --- /dev/null +++ b/lang/csharp/src/apache/test/Specific/DoubleLongUnionRecord.cs @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// ------------------------------------------------------------------------------ +// +// Generated by avrogen, version 1.11.0.0 +// Changes to this file may cause incorrect behavior and will be lost if code +// is regenerated +// +// ------------------------------------------------------------------------------ +namespace Avro.Test.Specific +{ + using System; + using System.Collections.Generic; + using System.Text; + using Avro; + using Avro.Specific; + + public partial class DoubleLongUnionRecord : ISpecificRecord + { + public static Schema _SCHEMA = Avro.Schema.Parse("{\"type\":\"record\",\"name\":\"DoubleLongUnionRecord\",\"namespace\":\"Avro.Test.Specific\",\"fields\":[{\"name" + + "\":\"Property\",\"type\":[\"double\",\"long\"]}]}"); + private object _Property; + public virtual Schema Schema + { + get + { + return DoubleLongUnionRecord._SCHEMA; + } + } + public object Property + { + get + { + return this._Property; + } + set + { + this._Property = value; + } + } + public virtual object Get(int fieldPos) + { + switch (fieldPos) + { + case 0: return this.Property; + default: throw new AvroRuntimeException("Bad index " + fieldPos + " in Get()"); + }; + } + public virtual void Put(int fieldPos, object fieldValue) + { + switch (fieldPos) + { + case 0: this.Property = (System.Object)fieldValue; break; + default: throw new AvroRuntimeException("Bad index " + fieldPos + " in Put()"); + }; + } + } +} diff --git a/lang/csharp/src/apache/test/Specific/SpecificTests.cs b/lang/csharp/src/apache/test/Specific/SpecificTests.cs index 178f255449b..1aa3c3a03ae 100644 --- a/lang/csharp/src/apache/test/Specific/SpecificTests.cs +++ b/lang/csharp/src/apache/test/Specific/SpecificTests.cs @@ -272,6 +272,38 @@ public void TestEnumDefault() Assert.AreEqual(EnumType.DEFAULT, rec2.enumType); } + [TestCase(0L)] + [TestCase(100L)] + [TestCase(-100L)] + [TestCase(0.0)] + [TestCase(100.0)] + [TestCase(-100.0)] + public void TestDoubleLongUnion(object value) + { + var testRecord = new DoubleLongUnionRecord(); + testRecord.Property = value; + + // serialize + var stream = serialize(DoubleLongUnionRecord._SCHEMA, testRecord); + + // deserialize + var rec2 = deserialize(stream, DoubleLongUnionRecord._SCHEMA, DoubleLongUnionRecord._SCHEMA); + Assert.AreEqual(value, rec2.Property); + Assert.AreEqual(value.GetType(), rec2.Property.GetType()); + } + + [TestCase(0)] + [TestCase(100)] + [TestCase(-100)] + [TestCase(0.0f)] + [TestCase(100.0f)] + [TestCase(-100.0f)] + [TestCase("0")] + [TestCase("100")] + public void TestDoubleLongUnionNoMatchException(object value) + { + Assert.Throws(() => serialize(DoubleLongUnionRecord._SCHEMA, new DoubleLongUnionRecord() { Property = value })); + } [Test] public void TestArrayWithReservedWords() From bd3307f2650a5edec2767a5e11e2c17a76ea35d2 Mon Sep 17 00:00:00 2001 From: Martin Grigorov Date: Tue, 19 Apr 2022 21:37:45 +0300 Subject: [PATCH 31/66] AVRO-3495: Rust: Fields order should not matter (#1650) * AVRO-3495: The order of the struct's fields and schema's fields should not matter Signed-off-by: Martin Tzvetanov Grigorov * AVRO-3495: Use the lookup table when comparing values against fields by name Until now it was expected that both the schema fields and the input values are sorted the same way. Use BTreeMap instead of HashMap for the lookup table because otherwise the assertion on the validation error messages is impossible due to random printing of the map's entries Signed-off-by: Martin Tzvetanov Grigorov * AVRO-3495: Update the test case Signed-off-by: Martin Tzvetanov Grigorov --- lang/rust/avro/src/encode.rs | 27 ++++++++--- lang/rust/avro/src/error.rs | 3 ++ lang/rust/avro/src/schema.rs | 22 ++++----- lang/rust/avro/src/types.rs | 83 ++++++++++++++++++++------------ lang/rust/avro/tests/schema.rs | 32 ++++++++++++ lang/rust/avro_derive/src/lib.rs | 2 +- 6 files changed, 117 insertions(+), 52 deletions(-) diff --git a/lang/rust/avro/src/encode.rs b/lang/rust/avro/src/encode.rs index bd69ba50100..014a6f747fb 100644 --- a/lang/rust/avro/src/encode.rs +++ b/lang/rust/avro/src/encode.rs @@ -192,18 +192,29 @@ pub(crate) fn encode_internal>( if let Schema::Record { ref name, fields: ref schema_fields, + ref lookup, .. } = *schema { let record_namespace = name.fully_qualified_name(enclosing_namespace).namespace; - for (i, &(_, ref value)) in fields.iter().enumerate() { - encode_internal( - value, - &schema_fields[i].schema, - names, - &record_namespace, - buffer, - )?; + for &(ref name, ref value) in fields.iter() { + match lookup.get(name) { + Some(idx) => { + encode_internal( + value, + &schema_fields[*idx].schema, + names, + &record_namespace, + buffer, + )?; + } + None => { + return Err(Error::NoEntryInLookupTable( + name.clone(), + format!("{:?}", lookup), + )); + } + } } } else { error!("invalid schema type for Record: {:?}", schema); diff --git a/lang/rust/avro/src/error.rs b/lang/rust/avro/src/error.rs index 900e59b4662..c5fe46f6b9a 100644 --- a/lang/rust/avro/src/error.rs +++ b/lang/rust/avro/src/error.rs @@ -403,6 +403,9 @@ pub enum Error { #[error("Signed decimal bytes length {0} not equal to fixed schema size {1}.")] EncodeDecimalAsFixedError(usize, usize), + #[error("There is no entry for {0} in the lookup table: {1}.")] + NoEntryInLookupTable(String, String), + #[error("Can only encode value type {value_kind:?} as one of {supported_schema:?}")] EncodeValueAsSchemaError { value_kind: ValueKind, diff --git a/lang/rust/avro/src/schema.rs b/lang/rust/avro/src/schema.rs index 95a21b853d3..4e860a9dad4 100644 --- a/lang/rust/avro/src/schema.rs +++ b/lang/rust/avro/src/schema.rs @@ -106,7 +106,7 @@ pub enum Schema { aliases: Aliases, doc: Documentation, fields: Vec, - lookup: HashMap, + lookup: BTreeMap, }, /// An `enum` Avro schema. Enum { @@ -1143,7 +1143,7 @@ impl Parser { } } - let mut lookup = HashMap::new(); + let mut lookup = BTreeMap::new(); let fully_qualified_name = name.fully_qualified_name(enclosing_namespace); self.register_resolving_schema(&fully_qualified_name, &aliases); @@ -1936,7 +1936,7 @@ mod tests { order: RecordFieldOrder::Ignore, position: 0, }], - lookup: HashMap::from_iter(vec![("field_one".to_string(), 0)]), + lookup: BTreeMap::from_iter(vec![("field_one".to_string(), 0)]), }; assert_eq!(schema_c, schema_c_expected); @@ -1990,7 +1990,7 @@ mod tests { order: RecordFieldOrder::Ignore, position: 0, }], - lookup: HashMap::from_iter(vec![("field_one".to_string(), 0)]), + lookup: BTreeMap::from_iter(vec![("field_one".to_string(), 0)]), }; assert_eq!(schema_option_a, schema_option_a_expected); @@ -2012,7 +2012,7 @@ mod tests { ) .unwrap(); - let mut lookup = HashMap::new(); + let mut lookup = BTreeMap::new(); lookup.insert("a".to_owned(), 0); lookup.insert("b".to_owned(), 1); @@ -2068,10 +2068,10 @@ mod tests { ) .unwrap(); - let mut lookup = HashMap::new(); + let mut lookup = BTreeMap::new(); lookup.insert("recordField".to_owned(), 0); - let mut node_lookup = HashMap::new(); + let mut node_lookup = BTreeMap::new(); node_lookup.insert("children".to_owned(), 1); node_lookup.insert("label".to_owned(), 0); @@ -2239,7 +2239,7 @@ mod tests { ) .unwrap(); - let mut lookup = HashMap::new(); + let mut lookup = BTreeMap::new(); lookup.insert("value".to_owned(), 0); lookup.insert("next".to_owned(), 1); @@ -2305,7 +2305,7 @@ mod tests { ) .unwrap(); - let mut lookup = HashMap::new(); + let mut lookup = BTreeMap::new(); lookup.insert("value".to_owned(), 0); lookup.insert("next".to_owned(), 1); @@ -2369,7 +2369,7 @@ mod tests { ) .unwrap(); - let mut lookup = HashMap::new(); + let mut lookup = BTreeMap::new(); lookup.insert("enum".to_owned(), 0); lookup.insert("next".to_owned(), 1); @@ -2444,7 +2444,7 @@ mod tests { ) .unwrap(); - let mut lookup = HashMap::new(); + let mut lookup = BTreeMap::new(); lookup.insert("fixed".to_owned(), 0); lookup.insert("next".to_owned(), 1); diff --git a/lang/rust/avro/src/types.rs b/lang/rust/avro/src/types.rs index 4f7f13f8254..24bfeeed2a1 100644 --- a/lang/rust/avro/src/types.rs +++ b/lang/rust/avro/src/types.rs @@ -26,7 +26,12 @@ use crate::{ AvroResult, Error, }; use serde_json::{Number, Value as JsonValue}; -use std::{collections::HashMap, convert::TryFrom, hash::BuildHasher, str::FromStr, u8}; +use std::{ + collections::{BTreeMap, HashMap}, + convert::TryFrom, + hash::BuildHasher, + str::FromStr, +}; use uuid::Uuid; /// Compute the maximum decimal value precision of a byte array of length `len` could hold. @@ -207,7 +212,7 @@ pub struct Record<'a> { /// Ordered according to the fields in the schema given to create this /// `Record` object. Any unset field defaults to `Value::Null`. pub fields: Vec<(String, Value)>, - schema_lookup: &'a HashMap, + schema_lookup: &'a BTreeMap, } impl<'a> Record<'a> { @@ -462,7 +467,14 @@ impl Value { Value::accumulate(acc, value.validate_internal(inner, names)) }) } - (&Value::Record(ref record_fields), &Schema::Record { ref fields, .. }) => { + ( + &Value::Record(ref record_fields), + &Schema::Record { + ref fields, + ref lookup, + .. + }, + ) => { if fields.len() != record_fields.len() { return Some(format!( "The value's records length ({}) is different than the schema's ({})", @@ -471,19 +483,26 @@ impl Value { )); } - fields.iter().zip(record_fields.iter()).fold( - None, - |acc, (field, &(ref name, ref value))| { - if field.name != *name { - return Some(format!( - "Value's name '{}' does not match the expected field's name '{}'", - name, field.name - )); + record_fields + .iter() + .fold(None, |acc, (field_name, record_field)| { + match lookup.get(field_name) { + Some(idx) => { + let field = &fields[*idx]; + Value::accumulate( + acc, + record_field.validate_internal(&field.schema, names), + ) + } + None => Value::accumulate( + acc, + Some(format!( + "There is no schema field for field '{}'", + field_name + )), + ), } - let res = value.validate_internal(&field.schema, names); - Value::accumulate(acc, res) - }, - ) + }) } (&Value::Map(ref items), &Schema::Record { ref fields, .. }) => { fields.iter().fold(None, |acc, field| { @@ -1059,7 +1078,7 @@ mod tests { lookup: Default::default(), }, false, - "Invalid value: Record([(\"unknown_field_name\", Null)]) for schema: Record { name: Name { name: \"record_name\", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: \"field_name\", doc: None, default: None, schema: Int, order: Ignore, position: 0 }], lookup: {} }. Reason: Value's name 'unknown_field_name' does not match the expected field's name 'field_name'", + "Invalid value: Record([(\"unknown_field_name\", Null)]) for schema: Record { name: Name { name: \"record_name\", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: \"field_name\", doc: None, default: None, schema: Int, order: Ignore, position: 0 }], lookup: {} }. Reason: There is no schema field for field 'unknown_field_name'", ), ( Value::Record(vec![("field_name".to_string(), Value::Null)]), @@ -1077,10 +1096,10 @@ mod tests { order: RecordFieldOrder::Ignore, position: 0, }], - lookup: Default::default(), + lookup: [("field_name".to_string(), 0)].iter().cloned().collect(), }, false, - "Invalid value: Record([(\"field_name\", Null)]) for schema: Record { name: Name { name: \"record_name\", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: \"field_name\", doc: None, default: None, schema: Ref { name: Name { name: \"missing\", namespace: None } }, order: Ignore, position: 0 }], lookup: {} }. Reason: Unresolved schema reference: 'missing'. Parsed names: []", + "Invalid value: Record([(\"field_name\", Null)]) for schema: Record { name: Name { name: \"record_name\", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: \"field_name\", doc: None, default: None, schema: Ref { name: Name { name: \"missing\", namespace: None } }, order: Ignore, position: 0 }], lookup: {\"field_name\": 0} }. Reason: Unresolved schema reference: 'missing'. Parsed names: []", ), ]; @@ -1209,7 +1228,6 @@ mod tests { fn validate_record() { init(); - use std::collections::HashMap; // { // "type": "record", // "fields": [ @@ -1239,7 +1257,10 @@ mod tests { position: 1, }, ], - lookup: HashMap::new(), + lookup: [("a".to_string(), 0), ("b".to_string(), 1)] + .iter() + .cloned() + .collect(), }; assert!(Value::Record(vec![ @@ -1252,21 +1273,14 @@ mod tests { ("b".to_string(), Value::String("foo".to_string())), ("a".to_string(), Value::Long(42i64)), ]); - assert!(!value.validate(&schema)); - assert_log_message( - format!( - "Invalid value: {:?} for schema: {:?}. Reason: {}", - value, schema, "Value's name 'a' does not match the expected field's name 'b'" - ) - .as_str(), - ); + assert!(value.validate(&schema)); let value = Value::Record(vec![ ("a".to_string(), Value::Boolean(false)), ("b".to_string(), Value::String("foo".to_string())), ]); assert!(!value.validate(&schema)); - assert_log_message("Invalid value: Record([(\"a\", Boolean(false)), (\"b\", String(\"foo\"))]) for schema: Record { name: Name { name: \"some_record\", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: \"a\", doc: None, default: None, schema: Long, order: Ascending, position: 0 }, RecordField { name: \"b\", doc: None, default: None, schema: String, order: Ascending, position: 1 }], lookup: {} }. Reason: Unsupported value-schema combination"); + assert_log_message("Invalid value: Record([(\"a\", Boolean(false)), (\"b\", String(\"foo\"))]) for schema: Record { name: Name { name: \"some_record\", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: \"a\", doc: None, default: None, schema: Long, order: Ascending, position: 0 }, RecordField { name: \"b\", doc: None, default: None, schema: String, order: Ascending, position: 1 }], lookup: {\"a\": 0, \"b\": 1} }. Reason: Unsupported value-schema combination"); let value = Value::Record(vec![ ("a".to_string(), Value::Long(42i64)), @@ -1274,7 +1288,7 @@ mod tests { ]); assert!(!value.validate(&schema)); assert_log_message( - "Invalid value: Record([(\"a\", Long(42)), (\"c\", String(\"foo\"))]) for schema: Record { name: Name { name: \"some_record\", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: \"a\", doc: None, default: None, schema: Long, order: Ascending, position: 0 }, RecordField { name: \"b\", doc: None, default: None, schema: String, order: Ascending, position: 1 }], lookup: {} }. Reason: Value's name 'c' does not match the expected field's name 'b'" + "Invalid value: Record([(\"a\", Long(42)), (\"c\", String(\"foo\"))]) for schema: Record { name: Name { name: \"some_record\", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: \"a\", doc: None, default: None, schema: Long, order: Ascending, position: 0 }, RecordField { name: \"b\", doc: None, default: None, schema: String, order: Ascending, position: 1 }], lookup: {\"a\": 0, \"b\": 1} }. Reason: There is no schema field for field 'c'" ); let value = Value::Record(vec![ @@ -1283,7 +1297,9 @@ mod tests { ("c".to_string(), Value::Null), ]); assert!(!value.validate(&schema)); - assert_log_message("Invalid value: Record([(\"a\", Long(42)), (\"b\", String(\"foo\")), (\"c\", Null)]) for schema: Record { name: Name { name: \"some_record\", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: \"a\", doc: None, default: None, schema: Long, order: Ascending, position: 0 }, RecordField { name: \"b\", doc: None, default: None, schema: String, order: Ascending, position: 1 }], lookup: {} }. Reason: The value's records length (3) is different than the schema's (2)"); + assert_log_message( + r#"Invalid value: Record([("a", Long(42)), ("b", String("foo")), ("c", Null)]) for schema: Record { name: Name { name: "some_record", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: "a", doc: None, default: None, schema: Long, order: Ascending, position: 0 }, RecordField { name: "b", doc: None, default: None, schema: String, order: Ascending, position: 1 }], lookup: {"a": 0, "b": 1} }. Reason: The value's records length (3) is different than the schema's (2)"#, + ); assert!(Value::Map( vec![ @@ -1301,7 +1317,10 @@ mod tests { .collect() ) .validate(&schema)); - assert_log_message("Invalid value: Map({\"c\": Long(123)}) for schema: Record { name: Name { name: \"some_record\", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: \"a\", doc: None, default: None, schema: Long, order: Ascending, position: 0 }, RecordField { name: \"b\", doc: None, default: None, schema: String, order: Ascending, position: 1 }], lookup: {} }. Reason: Field with name '\"a\"' is not a member of the map items\nField with name '\"b\"' is not a member of the map items"); + assert_log_message( + r#"Invalid value: Map({"c": Long(123)}) for schema: Record { name: Name { name: "some_record", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: "a", doc: None, default: None, schema: Long, order: Ascending, position: 0 }, RecordField { name: "b", doc: None, default: None, schema: String, order: Ascending, position: 1 }], lookup: {"a": 0, "b": 1} }. Reason: Field with name '"a"' is not a member of the map items +Field with name '"b"' is not a member of the map items"#, + ); let union_schema = Schema::Union(UnionSchema::new(vec![Schema::Null, schema]).unwrap()); diff --git a/lang/rust/avro/tests/schema.rs b/lang/rust/avro/tests/schema.rs index f58b62f9e7e..62eeb2ddf3d 100644 --- a/lang/rust/avro/tests/schema.rs +++ b/lang/rust/avro/tests/schema.rs @@ -17,6 +17,7 @@ use apache_avro::{ schema::{Name, RecordField}, + to_avro_datum, to_value, types::{Record, Value}, Codec, Error, Reader, Schema, Writer, }; @@ -1311,3 +1312,34 @@ fn test_decimal_valid_type_attributes() { assert_eq!(0, bytes_decimal.get_attribute("scale")); } */ + +// https://github.com/flavray/avro-rs/issues/47 +#[test] +fn avro_old_issue_47() { + init(); + let schema_str = r#" + { + "type": "record", + "name": "my_record", + "fields": [ + {"name": "a", "type": "long"}, + {"name": "b", "type": "string"} + ] + }"#; + let schema = Schema::parse_str(schema_str).unwrap(); + + use serde::{Deserialize, Serialize}; + + #[derive(Deserialize, Serialize)] + pub struct MyRecord { + b: String, + a: i64, + } + + let record = MyRecord { + b: "hello".to_string(), + a: 1, + }; + + let _ = to_avro_datum(&schema, to_value(record).unwrap()).unwrap(); +} diff --git a/lang/rust/avro_derive/src/lib.rs b/lang/rust/avro_derive/src/lib.rs index 0055249cb3b..a32f2be3348 100644 --- a/lang/rust/avro_derive/src/lib.rs +++ b/lang/rust/avro_derive/src/lib.rs @@ -152,7 +152,7 @@ fn get_data_struct_schema_def( Ok(quote! { let schema_fields = vec![#(#record_field_exprs),*]; let name = apache_avro::schema::Name::new(#full_schema_name).expect(&format!("Unable to parse struct name for schema {}", #full_schema_name)[..]); - let lookup: HashMap = schema_fields + let lookup: std::collections::BTreeMap = schema_fields .iter() .map(|field| (field.name.to_owned(), field.position)) .collect(); From 2fff83def5663dc1034abef7149350ce1cd3935f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Apr 2022 21:39:00 +0300 Subject: [PATCH 32/66] Bump slf4j.version from 1.7.33 to 1.7.36 in /lang/java (#1646) Bumps `slf4j.version` from 1.7.33 to 1.7.36. Updates `slf4j-api` from 1.7.33 to 1.7.36 - [Release notes](https://github.com/qos-ch/slf4j/releases) - [Commits](https://github.com/qos-ch/slf4j/compare/v_1.7.33...v_1.7.36) Updates `slf4j-simple` from 1.7.33 to 1.7.36 - [Release notes](https://github.com/qos-ch/slf4j/releases) - [Commits](https://github.com/qos-ch/slf4j/compare/v_1.7.33...v_1.7.36) Updates `slf4j-log4j12` from 1.7.33 to 1.7.36 - [Release notes](https://github.com/qos-ch/slf4j/releases) - [Commits](https://github.com/qos-ch/slf4j/compare/v_1.7.33...v_1.7.36) --- updated-dependencies: - dependency-name: org.slf4j:slf4j-api dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.slf4j:slf4j-simple dependency-type: direct:development update-type: version-update:semver-patch - dependency-name: org.slf4j:slf4j-log4j12 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- lang/java/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lang/java/pom.xml b/lang/java/pom.xml index a40483ac3fc..36f94fc0794 100644 --- a/lang/java/pom.xml +++ b/lang/java/pom.xml @@ -46,7 +46,7 @@ 4.1.75.Final 3.19.4 0.15.0 - 1.7.33 + 1.7.36 1.2.19 1.1.8.4 2.3 From 6ede153d4b85305a28147b54c9dc228957c4f645 Mon Sep 17 00:00:00 2001 From: Kyle Schoonover Date: Tue, 19 Apr 2022 11:40:48 -0700 Subject: [PATCH 33/66] AVRO-3491 Avoid a cast after is check (#1645) * AVRO-3360 Updated XML documentation * Revert "AVRO-3360 Updated XML documentation" This reverts commit b8601c072a5083380d30b580804dd0908b8cf4cc. * AVRO-3491 Avoid a cast after is check Co-authored-by: Kyle T. Schoonover --- .../src/apache/main/Generic/GenericDatumReader.cs | 14 +++++++------- .../apache/main/Generic/PreresolvingDatumReader.cs | 11 +++++------ 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/lang/csharp/src/apache/main/Generic/GenericDatumReader.cs b/lang/csharp/src/apache/main/Generic/GenericDatumReader.cs index 76a95b94ead..1ec126b3aa4 100644 --- a/lang/csharp/src/apache/main/Generic/GenericDatumReader.cs +++ b/lang/csharp/src/apache/main/Generic/GenericDatumReader.cs @@ -98,16 +98,16 @@ public GenericEnumAccess(EnumSchema schema) public object CreateEnum(object reuse, int ordinal) { - if (reuse is GenericEnum) + if (reuse is GenericEnum ge) { - var ge = (GenericEnum) reuse; - if (ge.Schema.Equals(this.schema)) + if (ge.Schema.Equals(schema)) { - ge.Value = this.schema[ordinal]; + ge.Value = schema[ordinal]; return ge; } } - return new GenericEnum(this.schema, this.schema[ordinal]); + + return new GenericEnum(schema, schema[ordinal]); } } @@ -204,12 +204,12 @@ class GenericMapAccess : MapAccess { public object Create(object reuse) { - if (reuse is IDictionary) + if (reuse is IDictionary result) { - var result = (IDictionary)reuse; result.Clear(); return result; } + return new Dictionary(); } diff --git a/lang/csharp/src/apache/main/Generic/PreresolvingDatumReader.cs b/lang/csharp/src/apache/main/Generic/PreresolvingDatumReader.cs index 61273e85a49..22c80407dde 100644 --- a/lang/csharp/src/apache/main/Generic/PreresolvingDatumReader.cs +++ b/lang/csharp/src/apache/main/Generic/PreresolvingDatumReader.cs @@ -319,15 +319,14 @@ private ReadItem ResolveUnion(UnionSchema writerSchema, Schema readerSchema) for (int i = 0; i < writerSchema.Count; i++) { - var writerBranch = writerSchema[i]; + Schema writerBranch = writerSchema[i]; - if (readerSchema is UnionSchema) + if (readerSchema is UnionSchema unionReader) { - var unionReader = (UnionSchema) readerSchema; - var readerBranch = unionReader.MatchingBranch(writerBranch); + int readerBranch = unionReader.MatchingBranch(writerBranch); if (readerBranch == -1) { - lookup[i] = (r, d) => { throw new AvroException( "No matching schema for " + writerBranch + " in " + unionReader ); }; + lookup[i] = (r, d) => { throw new AvroException("No matching schema for " + writerBranch + " in " + unionReader); }; } else { @@ -338,7 +337,7 @@ private ReadItem ResolveUnion(UnionSchema writerSchema, Schema readerSchema) { if (!readerSchema.CanRead(writerBranch)) { - lookup[i] = (r, d) => { throw new AvroException( "Schema mismatch Reader: " + ReaderSchema + ", writer: " + WriterSchema ); }; + lookup[i] = (r, d) => { throw new AvroException("Schema mismatch Reader: " + ReaderSchema + ", writer: " + WriterSchema); }; } else { From 25c43c1cb97b71add15466df023245098864b4ac Mon Sep 17 00:00:00 2001 From: Martin Grigorov Date: Tue, 19 Apr 2022 22:20:05 +0300 Subject: [PATCH 34/66] AVRO-3496: Rust: Use visitor.visit_borrowed_str() when possible (#1652) Signed-off-by: Martin Tzvetanov Grigorov --- lang/rust/avro/src/de.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lang/rust/avro/src/de.rs b/lang/rust/avro/src/de.rs index 6324caea9a0..24e9958b29f 100644 --- a/lang/rust/avro/src/de.rs +++ b/lang/rust/avro/src/de.rs @@ -262,7 +262,7 @@ impl<'a, 'de> de::Deserializer<'de> for &'a Deserializer<'de> { Value::Double(d) => visitor.visit_f64(d), Value::Record(ref fields) => visitor.visit_map(StructDeserializer::new(fields)), Value::Array(ref fields) => visitor.visit_seq(SeqDeserializer::new(fields)), - Value::String(ref s) => visitor.visit_str(s), + Value::String(ref s) => visitor.visit_borrowed_str(s), Value::Map(ref items) => visitor.visit_map(MapDeserializer::new(items)), _ => Err(de::Error::custom(format!( "unsupported union: {:?}", @@ -271,7 +271,7 @@ impl<'a, 'de> de::Deserializer<'de> for &'a Deserializer<'de> { }, Value::Record(ref fields) => visitor.visit_map(StructDeserializer::new(fields)), Value::Array(ref fields) => visitor.visit_seq(SeqDeserializer::new(fields)), - Value::String(ref s) => visitor.visit_str(s), + Value::String(ref s) => visitor.visit_borrowed_str(s), Value::Map(ref items) => visitor.visit_map(MapDeserializer::new(items)), value => Err(de::Error::custom(format!( "incorrect value of type: {:?}", @@ -296,10 +296,10 @@ impl<'a, 'de> de::Deserializer<'de> for &'a Deserializer<'de> { V: Visitor<'de>, { match *self.input { - Value::String(ref s) => visitor.visit_str(s), + Value::String(ref s) => visitor.visit_borrowed_str(s), Value::Bytes(ref bytes) | Value::Fixed(_, ref bytes) => ::std::str::from_utf8(bytes) .map_err(|e| de::Error::custom(e.to_string())) - .and_then(|s| visitor.visit_str(s)), + .and_then(|s| visitor.visit_borrowed_str(s)), Value::Uuid(ref u) => visitor.visit_str(&u.to_string()), _ => Err(de::Error::custom("not a string|bytes|fixed")), } @@ -310,14 +310,14 @@ impl<'a, 'de> de::Deserializer<'de> for &'a Deserializer<'de> { V: Visitor<'de>, { match *self.input { - Value::String(ref s) => visitor.visit_string(s.to_owned()), + Value::String(ref s) => visitor.visit_borrowed_str(s), Value::Bytes(ref bytes) | Value::Fixed(_, ref bytes) => { String::from_utf8(bytes.to_owned()) .map_err(|e| de::Error::custom(e.to_string())) .and_then(|s| visitor.visit_string(s)) } Value::Union(_i, ref x) => match **x { - Value::String(ref s) => visitor.visit_string(s.to_owned()), + Value::String(ref s) => visitor.visit_borrowed_str(s), _ => Err(de::Error::custom("not a string|bytes|fixed")), }, _ => Err(de::Error::custom("not a string|bytes|fixed")), From 73d22fa0c0e8a6ac1bf159d917ee0fdd4973b088 Mon Sep 17 00:00:00 2001 From: Zoltan Csizmadia Date: Tue, 19 Apr 2022 15:32:18 -0500 Subject: [PATCH 35/66] AVRO-3477: Add unit tests for logical types with fixed base type (#1629) * Support fixed base type for logical types * Tweak * Revert * Fix fixed type definition * Add AvroGen tests Co-authored-by: Zoltan Csizmadia --- .../src/apache/test/AvroGen/AvroGenTests.cs | 58 ++++++++++------- .../src/apache/test/Util/LogicalTypeTests.cs | 64 +++++++++++++++---- 2 files changed, 86 insertions(+), 36 deletions(-) diff --git a/lang/csharp/src/apache/test/AvroGen/AvroGenTests.cs b/lang/csharp/src/apache/test/AvroGen/AvroGenTests.cs index 2426bc27519..1f61fc547dc 100644 --- a/lang/csharp/src/apache/test/AvroGen/AvroGenTests.cs +++ b/lang/csharp/src/apache/test/AvroGen/AvroGenTests.cs @@ -161,8 +161,11 @@ class AvroGenTests ""type"": [""null"", { ""namespace"": ""org.apache.avro.codegentest.testdata"", ""name"": ""FixedInUnion"", - ""type"": ""fixed"", - ""size"": 12, + ""type"": { + ""type"": ""fixed"", + ""size"": 12, + ""name"": ""FixedName"", + }, ""logicalType"": ""decimal"", ""precision"": 28, ""scale"": 15 @@ -442,6 +445,16 @@ private Assembly TestSchema( "org/apache/avro/codegentest/some/NestedSomeNamespaceRecord.cs", "org/apache/avro/codegentest/other/NestedOtherNamespaceRecord.cs" })] + [TestCase( + _nestedLogicalTypesUnionFixedDecimal, + new string[] + { + "org.apache.avro.codegentest.testdata.NestedLogicalTypesUnionFixedDecimal" + }, + new string[] + { + "org/apache/avro/codegentest/testdata/NestedLogicalTypesUnionFixedDecimal.cs" + })] [TestCase( _nullableLogicalTypes, new string[] @@ -595,7 +608,6 @@ public void GenerateSchemaWithNamespaceMapping( [TestCase(_logicalTypesWithCustomConversion, typeof(AvroTypeException))] [TestCase(_customConversionWithLogicalTypes, typeof(SchemaParseException))] - [TestCase(_nestedLogicalTypesUnionFixedDecimal, typeof(SchemaParseException))] public void NotSupportedSchema(string schema, Type expectedException) { // Create temp folder @@ -670,26 +682,28 @@ public void NotSupportedSchema(string schema, Type expectedException) new object[] { "schematest.SchemaObject", typeof(IList) })] [TestCase(@" { - ""type"" : ""record"", - ""name"" : ""LogicalTypes"", - ""namespace"" : ""schematest"", - ""fields"" : - [ - { ""name"" : ""nullibleguid"", ""type"" : [""null"", {""type"": ""string"", ""logicalType"": ""uuid"" } ]}, - { ""name"" : ""guid"", ""type"" : {""type"": ""string"", ""logicalType"": ""uuid"" } }, - { ""name"" : ""nullibletimestampmillis"", ""type"" : [""null"", {""type"": ""long"", ""logicalType"": ""timestamp-millis""}] }, - { ""name"" : ""timestampmillis"", ""type"" : {""type"": ""long"", ""logicalType"": ""timestamp-millis""} }, - { ""name"" : ""nullibiletimestampmicros"", ""type"" : [""null"", {""type"": ""long"", ""logicalType"": ""timestamp-micros""}] }, - { ""name"" : ""timestampmicros"", ""type"" : {""type"": ""long"", ""logicalType"": ""timestamp-micros""} }, - { ""name"" : ""nullibiletimemicros"", ""type"" : [""null"", {""type"": ""long"", ""logicalType"": ""time-micros""}] }, - { ""name"" : ""timemicros"", ""type"" : {""type"": ""long"", ""logicalType"": ""time-micros""} }, - { ""name"" : ""nullibiletimemillis"", ""type"" : [""null"", {""type"": ""int"", ""logicalType"": ""time-millis""}] }, - { ""name"" : ""timemillis"", ""type"" : {""type"": ""int"", ""logicalType"": ""time-millis""} }, - { ""name"" : ""nullibledecimal"", ""type"" : [""null"", {""type"": ""bytes"", ""logicalType"": ""decimal"", ""precision"": 4, ""scale"": 2}] }, - { ""name"" : ""decimal"", ""type"" : {""type"": ""bytes"", ""logicalType"": ""decimal"", ""precision"": 4, ""scale"": 2} } - ] + ""type"" : ""record"", + ""name"" : ""LogicalTypes"", + ""namespace"" : ""schematest"", + ""fields"" : + [ + { ""name"" : ""nullibleguid"", ""type"" : [""null"", {""type"": ""string"", ""logicalType"": ""uuid"" } ]}, + { ""name"" : ""guid"", ""type"" : {""type"": ""string"", ""logicalType"": ""uuid"" } }, + { ""name"" : ""nullibletimestampmillis"", ""type"" : [""null"", {""type"": ""long"", ""logicalType"": ""timestamp-millis""}] }, + { ""name"" : ""timestampmillis"", ""type"" : {""type"": ""long"", ""logicalType"": ""timestamp-millis""} }, + { ""name"" : ""nullibiletimestampmicros"", ""type"" : [""null"", {""type"": ""long"", ""logicalType"": ""timestamp-micros""}] }, + { ""name"" : ""timestampmicros"", ""type"" : {""type"": ""long"", ""logicalType"": ""timestamp-micros""} }, + { ""name"" : ""nullibiletimemicros"", ""type"" : [""null"", {""type"": ""long"", ""logicalType"": ""time-micros""}] }, + { ""name"" : ""timemicros"", ""type"" : {""type"": ""long"", ""logicalType"": ""time-micros""} }, + { ""name"" : ""nullibiletimemillis"", ""type"" : [""null"", {""type"": ""int"", ""logicalType"": ""time-millis""}] }, + { ""name"" : ""timemillis"", ""type"" : {""type"": ""int"", ""logicalType"": ""time-millis""} }, + { ""name"" : ""nullibledecimal"", ""type"" : [""null"", {""type"": ""bytes"", ""logicalType"": ""decimal"", ""precision"": 4, ""scale"": 2}] }, + { ""name"" : ""decimal"", ""type"" : {""type"": ""bytes"", ""logicalType"": ""decimal"", ""precision"": 4, ""scale"": 2} }, + { ""name"" : ""nullibledecimalfixed"", ""type"" : [""null"", {""type"": {""type"" : ""fixed"", ""size"": 16, ""name"": ""ndf""}, ""logicalType"": ""decimal"", ""precision"": 4, ""scale"": 2}] }, + { ""name"" : ""decimalfixed"", ""type"" : {""type"": {""type"" : ""fixed"", ""size"": 16, ""name"": ""df""}, ""logicalType"": ""decimal"", ""precision"": 4, ""scale"": 2} } + ] }", - new object[] { "schematest.LogicalTypes", typeof(Guid?), typeof(Guid), typeof(DateTime?), typeof(DateTime), typeof(DateTime?), typeof(DateTime), typeof(TimeSpan?), typeof(TimeSpan), typeof(TimeSpan?), typeof(TimeSpan), typeof(AvroDecimal?), typeof(AvroDecimal) })] + new object[] { "schematest.LogicalTypes", typeof(Guid?), typeof(Guid), typeof(DateTime?), typeof(DateTime), typeof(DateTime?), typeof(DateTime), typeof(TimeSpan?), typeof(TimeSpan), typeof(TimeSpan?), typeof(TimeSpan), typeof(AvroDecimal?), typeof(AvroDecimal), typeof(AvroDecimal?), typeof(AvroDecimal) })] [TestCase(@" { ""namespace"": ""enum.base"", diff --git a/lang/csharp/src/apache/test/Util/LogicalTypeTests.cs b/lang/csharp/src/apache/test/Util/LogicalTypeTests.cs index 9630b7c6716..3dcc2e35247 100644 --- a/lang/csharp/src/apache/test/Util/LogicalTypeTests.cs +++ b/lang/csharp/src/apache/test/Util/LogicalTypeTests.cs @@ -58,15 +58,22 @@ public void TestDecimalConvert(string s, int scale, byte[] converted) Assert.AreEqual(decimalVal, (AvroDecimal)avroDecimal.ConvertToLogicalValue(converted, schema)); } - [TestCase("1234.56")] - [TestCase("-1234.56")] - [TestCase("123456789123456789.56")] - [TestCase("-123456789123456789.56")] - [TestCase("000000000000000001.01")] - [TestCase("-000000000000000001.01")] - public void TestDecimal(string s) + [Test] + public void TestDecimal( + [Values( + "1234.56", + "-1234.56", + "123456789123456789.56", + "-123456789123456789.56", + "000000000000000001.01", + "-000000000000000001.01" + )] string s, + [Values( + "\"bytes\"", + "{\"type\": \"fixed\", \"size\": 16, \"name\": \"n\"}" + )] string baseType) { - var schema = (LogicalSchema)Schema.Parse("{\"type\": \"bytes\", \"logicalType\": \"decimal\", \"precision\": 4, \"scale\": 2 }"); + var schema = (LogicalSchema)Schema.Parse($"{{\"type\": {baseType}, \"logicalType\": \"decimal\", \"precision\": 4, \"scale\": 2 }}"); var avroDecimal = new Avro.Util.Decimal(); // CultureInfo.InvariantCulture ensures that "." is always accepted as the decimal point @@ -77,10 +84,38 @@ public void TestDecimal(string s) Assert.AreEqual(decimalVal, convertedDecimalVal); } - [TestCase] - public void TestDecimalMinMax() + [Test] + public void TestDecimalScale( + [Values( + "0", + "1", + "-1", + "1234567891234567890123456789", + "-1234567891234567890123456789", + "0000000000000000000000000001", + "-0000000000000000000000000001" + )] string s, + [Values(1, 2, 3, 4, 5, 6, 7, 8)] int scale, + [Values( + "\"bytes\"", + "{\"type\": \"fixed\", \"size\": 16, \"name\": \"n\"}" + )] string baseType) { - var schema = (LogicalSchema)Schema.Parse("{\"type\": \"bytes\", \"logicalType\": \"decimal\", \"precision\": 4, \"scale\": 0 }"); + var schema = (LogicalSchema)Schema.Parse($"{{\"type\": {baseType}, \"logicalType\": \"decimal\", \"precision\": 8, \"scale\": {scale} }}"); + + var avroDecimal = new Avro.Util.Decimal(); + var decimalVal = new AvroDecimal(BigInteger.Parse(s), scale); + + var convertedDecimalVal = (AvroDecimal)avroDecimal.ConvertToLogicalValue(avroDecimal.ConvertToBaseValue(decimalVal, schema), schema); + + Assert.AreEqual(decimalVal, convertedDecimalVal); + } + + [TestCase("\"bytes\"")] + [TestCase("{\"type\": \"fixed\", \"size\": 16, \"name\": \"n\"}")] + public void TestDecimalMinMax(string baseType) + { + var schema = (LogicalSchema)Schema.Parse($"{{\"type\": {baseType}, \"logicalType\": \"decimal\", \"precision\": 4, \"scale\": 0 }}"); var avroDecimal = new Avro.Util.Decimal(); @@ -92,10 +127,11 @@ public void TestDecimalMinMax() } } - [TestCase] - public void TestDecimalOutOfRangeException() + [TestCase("\"bytes\"")] + [TestCase("{\"type\": \"fixed\", \"size\": 16, \"name\": \"n\"}")] + public void TestDecimalOutOfRangeException(string baseType) { - var schema = (LogicalSchema)Schema.Parse("{\"type\": \"bytes\", \"logicalType\": \"decimal\", \"precision\": 4, \"scale\": 2 }"); + var schema = (LogicalSchema)Schema.Parse($"{{\"type\": {baseType}, \"logicalType\": \"decimal\", \"precision\": 4, \"scale\": 2 }}"); var avroDecimal = new Avro.Util.Decimal(); var decimalVal = (AvroDecimal)1234.567M; // scale of 3 should throw ArgumentOutOfRangeException From 8c9f3fa18b0661fab4a26a3f92d400a5f762a7c8 Mon Sep 17 00:00:00 2001 From: Zoltan Csizmadia Date: Wed, 20 Apr 2022 02:46:50 -0500 Subject: [PATCH 36/66] AVRO-3465: Add avrogen protocol tests (#1616) * Add avrogen protocol tests * Add protocol test case * Fix merge conflicts Co-authored-by: Zoltan Csizmadia --- .../src/apache/test/AvroGen/AvroGenHelper.cs | 151 ++++- .../test/AvroGen/AvroGenProtocolTests.cs | 517 ++++++++++++++++++ ...{AvroGenTests.cs => AvroGenSchemaTests.cs} | 8 +- 3 files changed, 661 insertions(+), 15 deletions(-) create mode 100644 lang/csharp/src/apache/test/AvroGen/AvroGenProtocolTests.cs rename lang/csharp/src/apache/test/AvroGen/{AvroGenTests.cs => AvroGenSchemaTests.cs} (98%) diff --git a/lang/csharp/src/apache/test/AvroGen/AvroGenHelper.cs b/lang/csharp/src/apache/test/AvroGen/AvroGenHelper.cs index 1d265af7ce2..49865c17052 100644 --- a/lang/csharp/src/apache/test/AvroGen/AvroGenHelper.cs +++ b/lang/csharp/src/apache/test/AvroGen/AvroGenHelper.cs @@ -25,6 +25,7 @@ using Microsoft.CodeAnalysis.CSharp; using Microsoft.CodeAnalysis.Emit; using NUnit.Framework; +using Avro.Specific; namespace Avro.Test.AvroGen { @@ -112,17 +113,18 @@ public static Assembly CompileCSharpFilesIntoLibrary(IEnumerable sourceF // Compile EmitResult compilationResult = compilation.Emit(compilerStream); - //Note: Comment the following out to analyze the compiler errors if needed - //if (!compilationResult.Success) - //{ - // foreach (Diagnostic diagnostic in compilationResult.Diagnostics) - // { - // if (diagnostic.IsWarningAsError || diagnostic.Severity == DiagnosticSeverity.Error) - // { - // TestContext.WriteLine($"{diagnostic.Id} - {diagnostic.GetMessage()} - {diagnostic.Location}"); - // } - // } - //} +#if DEBUG + if (!compilationResult.Success) + { + foreach (Diagnostic diagnostic in compilationResult.Diagnostics) + { + if (diagnostic.IsWarningAsError || diagnostic.Severity == DiagnosticSeverity.Error) + { + TestContext.WriteLine($"{diagnostic.Id} - {diagnostic.GetMessage()} - {diagnostic.Location}"); + } + } + } +#endif Assert.That(compilationResult.Success, Is.True); @@ -131,6 +133,7 @@ public static Assembly CompileCSharpFilesIntoLibrary(IEnumerable sourceF return null; } + // Load assembly from stream compilerStream.Seek(0, SeekOrigin.Begin); return Assembly.Load(compilerStream.ToArray()); } @@ -152,5 +155,131 @@ public static string CreateEmptyTemporyFolder(out string uniqueId, string path = return tempFolder; } + + public static Assembly CompileCSharpFilesAndCheckTypes( + string outputDir, + string assemblyName, + IEnumerable typeNamesToCheck = null, + IEnumerable generatedFilesToCheck = null) + { + // Check if all generated files exist + if (generatedFilesToCheck != null) + { + foreach (string generatedFile in generatedFilesToCheck) + { + Assert.That(new FileInfo(Path.Combine(outputDir, generatedFile)), Does.Exist); + } + } + + // Compile into netstandard library and load assembly + Assembly assembly = CompileCSharpFilesIntoLibrary( + new DirectoryInfo(outputDir) + .EnumerateFiles("*.cs", SearchOption.AllDirectories) + .Select(fi => fi.FullName), + assemblyName); + + if (typeNamesToCheck != null) + { + // Check if the compiled code has the same number of types defined as the check list + Assert.That(typeNamesToCheck.Count(), Is.EqualTo(assembly.DefinedTypes.Count())); + + // Check if types available in compiled assembly + foreach (string typeName in typeNamesToCheck) + { + Type type = assembly.GetType(typeName); + Assert.That(type, Is.Not.Null); + + // Protocols are abstract and cannot be instantiated + if (typeof(ISpecificProtocol).IsAssignableFrom(type)) + { + Assert.That(type.IsAbstract, Is.True); + + // If directly inherited from ISpecificProtocol, use reflection to read static private field + // holding the protocol. Callback objects are not directly inherited from ISpecificProtocol, + // so private fields in the base class cannot be accessed + if (type.BaseType.Equals(typeof(ISpecificProtocol))) + { + // Use reflection to read static field, holding the protocol + FieldInfo protocolField = type.GetField("protocol", BindingFlags.NonPublic | BindingFlags.Static); + Protocol protocol = protocolField.GetValue(null) as Protocol; + + Assert.That(protocol, Is.Not.Null); + } + } + else + { + Assert.That(type.IsClass || type.IsEnum, Is.True); + + // Instantiate object + object obj = Activator.CreateInstance(type); + Assert.That(obj, Is.Not.Null); + + // If ISpecificRecord, call its member for sanity check + if (obj is ISpecificRecord record) + { + // Read record's schema object + Assert.That(record.Schema, Is.Not.Null); + // Force exception by reading/writing invalid field + Assert.Throws(() => record.Get(-1)); + Assert.Throws(() => record.Put(-1, null)); + } + } + } + } + + return assembly; + } + + public static Assembly TestSchema( + string schema, + IEnumerable typeNamesToCheck = null, + IEnumerable> namespaceMapping = null, + IEnumerable generatedFilesToCheck = null) + { + // Create temp folder + string outputDir = CreateEmptyTemporyFolder(out string uniqueId); + + try + { + // Save schema + string schemaFileName = Path.Combine(outputDir, $"{uniqueId}.avsc"); + System.IO.File.WriteAllText(schemaFileName, schema); + + // Generate from schema file + Assert.That(AvroGenTool.GenSchema(schemaFileName, outputDir, namespaceMapping ?? new Dictionary()), Is.EqualTo(0)); + + return CompileCSharpFilesAndCheckTypes(outputDir, uniqueId, typeNamesToCheck, generatedFilesToCheck); + } + finally + { + Directory.Delete(outputDir, true); + } + } + + public static Assembly TestProtocol( + string protocol, + IEnumerable typeNamesToCheck = null, + IEnumerable> namespaceMapping = null, + IEnumerable generatedFilesToCheck = null) + { + // Create temp folder + string outputDir = CreateEmptyTemporyFolder(out string uniqueId); + + try + { + // Save protocol + string schemaFileName = Path.Combine(outputDir, $"{uniqueId}.avpr"); + System.IO.File.WriteAllText(schemaFileName, protocol); + + // Generate from protocol file + Assert.That(AvroGenTool.GenProtocol(schemaFileName, outputDir, namespaceMapping ?? new Dictionary()), Is.EqualTo(0)); + + return CompileCSharpFilesAndCheckTypes(outputDir, uniqueId, typeNamesToCheck, generatedFilesToCheck); + } + finally + { + Directory.Delete(outputDir, true); + } + } } } diff --git a/lang/csharp/src/apache/test/AvroGen/AvroGenProtocolTests.cs b/lang/csharp/src/apache/test/AvroGen/AvroGenProtocolTests.cs new file mode 100644 index 00000000000..b408650369f --- /dev/null +++ b/lang/csharp/src/apache/test/AvroGen/AvroGenProtocolTests.cs @@ -0,0 +1,517 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +using System.Collections.Generic; +using NUnit.Framework; + +namespace Avro.Test.AvroGen +{ + [TestFixture] + + class AvroGenProtocolTests + { + private const string _baseball = @" +{ + ""protocol"" : ""Baseball"", + ""namespace"" : ""avro.examples.baseball"", + ""doc"" : ""Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\""License\""); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n https://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \""AS IS\"" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License."", + ""types"" : [ { + ""type"" : ""enum"", + ""name"" : ""Position"", + ""symbols"" : [ ""P"", ""C"", ""B1"", ""B2"", ""B3"", ""SS"", ""LF"", ""CF"", ""RF"", ""DH"" ] + }, { + ""type"" : ""record"", + ""name"" : ""Player"", + ""fields"" : [ { + ""name"" : ""number"", + ""type"" : ""int"" + }, { + ""name"" : ""first_name"", + ""type"" : ""string"" + }, { + ""name"" : ""last_name"", + ""type"" : ""string"" + }, { + ""name"" : ""position"", + ""type"" : { + ""type"" : ""array"", + ""items"" : ""Position"" + } + } ] + } ], + ""messages"" : { + } +} +"; + private const string _comments = @" +{ + ""protocol"" : ""Comments"", + ""namespace"" : ""testing"", + ""types"" : [ { + ""type"" : ""enum"", + ""name"" : ""DocumentedEnum"", + ""doc"" : ""Documented Enum"", + ""symbols"" : [ ""A"", ""B"", ""C"" ], + ""default"" : ""A"" + }, { + ""type"" : ""enum"", + ""name"" : ""UndocumentedEnum"", + ""symbols"" : [ ""D"", ""E"" ] + }, { + ""type"" : ""fixed"", + ""name"" : ""DocumentedFixed"", + ""doc"" : ""Documented Fixed Type"", + ""size"" : 16 + }, { + ""type"" : ""fixed"", + ""name"" : ""UndocumentedFixed"", + ""size"" : 16 + }, { + ""type"" : ""error"", + ""name"" : ""DocumentedError"", + ""doc"" : ""Documented Error"", + ""fields"" : [ { + ""name"" : ""reason"", + ""type"" : ""string"", + ""doc"" : ""Documented Reason Field"" + }, { + ""name"" : ""explanation"", + ""type"" : ""string"", + ""doc"" : ""Default Doc Explanation Field"" + } ] + }, { + ""type"" : ""record"", + ""name"" : ""UndocumentedRecord"", + ""fields"" : [ { + ""name"" : ""description"", + ""type"" : ""string"" + } ] + } ], + ""messages"" : { + ""documentedMethod"" : { + ""doc"" : ""Documented Method"", + ""request"" : [ { + ""name"" : ""message"", + ""type"" : ""string"", + ""doc"" : ""Documented Parameter"" + }, { + ""name"" : ""defMsg"", + ""type"" : ""string"", + ""doc"" : ""Default Documented Parameter"" + } ], + ""response"" : ""null"", + ""errors"" : [ ""DocumentedError"" ] + }, + ""undocumentedMethod"" : { + ""request"" : [ { + ""name"" : ""message"", + ""type"" : ""string"" + } ], + ""response"" : ""null"" + } + } +} +"; + + private const string _interop = @" +{ + ""protocol"" : ""InteropProtocol"", + ""namespace"" : ""org.apache.avro.interop"", + ""doc"" : ""Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\""License\""); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n https://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \""AS IS\"" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License."", + ""types"" : [ { + ""type"" : ""record"", + ""name"" : ""Label"", + ""fields"" : [ { + ""name"" : ""label"", + ""type"" : ""string"" + } ] + }, { + ""type"" : ""enum"", + ""name"" : ""Kind"", + ""symbols"" : [ ""A"", ""B"", ""C"" ] + }, { + ""type"" : ""fixed"", + ""name"" : ""MD5"", + ""size"" : 16 + }, { + ""type"" : ""record"", + ""name"" : ""Node"", + ""fields"" : [ { + ""name"" : ""label"", + ""type"" : ""string"" + }, { + ""name"" : ""children"", + ""type"" : { + ""type"" : ""array"", + ""items"" : ""Node"" + }, + ""default"" : [ ] + } ] + }, { + ""type"" : ""record"", + ""name"" : ""Interop"", + ""fields"" : [ { + ""name"" : ""intField"", + ""type"" : ""int"", + ""default"" : 1 + }, { + ""name"" : ""longField"", + ""type"" : ""long"", + ""default"" : -1 + }, { + ""name"" : ""stringField"", + ""type"" : ""string"" + }, { + ""name"" : ""boolField"", + ""type"" : ""boolean"", + ""default"" : false + }, { + ""name"" : ""floatField"", + ""type"" : ""float"", + ""default"" : 0.0 + }, { + ""name"" : ""doubleField"", + ""type"" : ""double"", + ""default"" : -1.0E12 + }, { + ""name"" : ""nullField"", + ""type"" : ""null"" + }, { + ""name"" : ""arrayField"", + ""type"" : { + ""type"" : ""array"", + ""items"" : ""double"" + }, + ""default"" : [ ] + }, { + ""name"" : ""mapField"", + ""type"" : { + ""type"" : ""map"", + ""values"" : ""Label"" + } + }, { + ""name"" : ""unionField"", + ""type"" : [ ""boolean"", ""double"", { + ""type"" : ""array"", + ""items"" : ""bytes"" + } ] + }, { + ""name"" : ""enumField"", + ""type"" : ""Kind"" + }, { + ""name"" : ""fixedField"", + ""type"" : ""MD5"" + }, { + ""name"" : ""recordField"", + ""type"" : ""Node"" + } ] + } ], + ""messages"" : { } +} +"; + private const string _namespaces = @" +{ + ""protocol"" : ""TestNamespace"", + ""namespace"" : ""avro.test.protocol"", + ""doc"" : ""Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\""License\""); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n https://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \""AS IS\"" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License."", + ""types"" : [ { + ""type"" : ""fixed"", + ""name"" : ""FixedInOtherNamespace"", + ""namespace"" : ""avro.test.fixed"", + ""size"" : 16 + }, { + ""type"" : ""fixed"", + ""name"" : ""FixedInThisNamespace"", + ""size"" : 16 + }, { + ""type"" : ""record"", + ""name"" : ""RecordInOtherNamespace"", + ""namespace"" : ""avro.test.record"", + ""fields"" : [ ] + }, { + ""type"" : ""error"", + ""name"" : ""ErrorInOtherNamespace"", + ""namespace"" : ""avro.test.error"", + ""fields"" : [ ] + }, { + ""type"" : ""enum"", + ""name"" : ""EnumInOtherNamespace"", + ""namespace"" : ""avro.test.enum"", + ""symbols"" : [ ""FOO"" ] + }, { + ""type"" : ""record"", + ""name"" : ""RefersToOthers"", + ""fields"" : [ { + ""name"" : ""someFixed"", + ""type"" : ""avro.test.fixed.FixedInOtherNamespace"" + }, { + ""name"" : ""someRecord"", + ""type"" : ""avro.test.record.RecordInOtherNamespace"" + }, { + ""name"" : ""someError"", + ""type"" : ""avro.test.error.ErrorInOtherNamespace"" + }, { + ""name"" : ""someEnum"", + ""type"" : ""avro.test.enum.EnumInOtherNamespace"" + }, { + ""name"" : ""thisFixed"", + ""type"" : ""FixedInThisNamespace"" + } ] + } ], + ""messages"" : { + } +} +"; + private const string _forwardRef = @" +{ + ""protocol"": ""Import"", + ""namespace"": ""org.foo"", + ""types"": [ + { + ""type"": ""record"", + ""name"": ""ANameValue"", + ""fields"": [ + { ""name"":""name"", ""type"": ""string"", ""doc"":""the name"" }, + { ""name"": ""value"", ""type"": ""string"", ""doc"": ""the value"" }, + { ""name"": ""type"", ""type"": { ""type"": ""enum"", ""name"":""ValueType"", ""symbols"": [""JSON"",""BASE64BIN"",""PLAIN""] }, ""default"": ""PLAIN"" } + ] + } + ], + ""messages"": { } +} +"; + private const string _unicode = @" +{ + ""protocol"" : ""Протоколы"", + ""namespace"" : ""org.avro.test"", + ""doc"" : ""This is a test that UTF8 functions correctly.\nこのテストでは、UTF - 8で正しく機能している。\n这是一个测试，UTF - 8的正常运行。"", + ""types"" : [ { + ""type"" : ""record"", + ""name"" : ""Структура"", + ""fields"" : [ { + ""name"" : ""Строковый"", + ""type"" : ""string"" + }, { + ""name"" : ""文字列"", + ""type"" : ""string"" + } ] + } ], + ""messages"" : { + } +} +"; + + private const string _myProtocol = @" +{ + ""protocol"" : ""MyProtocol"", + ""namespace"" : ""com.foo"", + ""types"" : [ + { + ""type"" : ""record"", + ""name"" : ""A"", + ""fields"" : [ { ""name"" : ""f1"", ""type"" : ""long"" } ] + }, + { + ""type"" : ""enum"", + ""name"" : ""MyEnum"", + ""symbols"" : [ ""A"", ""B"", ""C"" ] + }, + { + ""type"": ""fixed"", + ""size"": 16, + ""name"": ""MyFixed"" + }, + { + ""type"" : ""record"", + ""name"" : ""Z"", + ""fields"" : + [ + { ""name"" : ""myUInt"", ""type"" : [ ""int"", ""null"" ] }, + { ""name"" : ""myULong"", ""type"" : [ ""long"", ""null"" ] }, + { ""name"" : ""myUBool"", ""type"" : [ ""boolean"", ""null"" ] }, + { ""name"" : ""myUDouble"", ""type"" : [ ""double"", ""null"" ] }, + { ""name"" : ""myUFloat"", ""type"" : [ ""float"", ""null"" ] }, + { ""name"" : ""myUBytes"", ""type"" : [ ""bytes"", ""null"" ] }, + { ""name"" : ""myUString"", ""type"" : [ ""string"", ""null"" ] }, + + { ""name"" : ""myInt"", ""type"" : ""int"" }, + { ""name"" : ""myLong"", ""type"" : ""long"" }, + { ""name"" : ""myBool"", ""type"" : ""boolean"" }, + { ""name"" : ""myDouble"", ""type"" : ""double"" }, + { ""name"" : ""myFloat"", ""type"" : ""float"" }, + { ""name"" : ""myBytes"", ""type"" : ""bytes"" }, + { ""name"" : ""myString"", ""type"" : ""string"" }, + { ""name"" : ""myNull"", ""type"" : ""null"" }, + + { ""name"" : ""myFixed"", ""type"" : ""MyFixed"" }, + { ""name"" : ""myA"", ""type"" : ""A"" }, + { ""name"" : ""myE"", ""type"" : ""MyEnum"" }, + { ""name"" : ""myArray"", ""type"" : { ""type"" : ""array"", ""items"" : ""bytes"" } }, + { ""name"" : ""myArray2"", ""type"" : { ""type"" : ""array"", ""items"" : { ""type"" : ""record"", ""name"" : ""newRec"", ""fields"" : [ { ""name"" : ""f1"", ""type"" : ""long""} ] } } }, + { ""name"" : ""myMap"", ""type"" : { ""type"" : ""map"", ""values"" : ""string"" } }, + { ""name"" : ""myMap2"", ""type"" : { ""type"" : ""map"", ""values"" : ""newRec"" } }, + { ""name"" : ""myObject"", ""type"" : [ ""MyEnum"", ""A"", ""null"" ] }, + { ""name"" : ""myArray3"", ""type"" : { ""type"" : ""array"", ""items"" : { ""type"" : ""array"", ""items"" : [ ""double"", ""string"", ""null"" ] } } } + ] + } + ] +}"; + + [TestCase( + _baseball, + new string[] + { + "avro.examples.baseball.Baseball", + "avro.examples.baseball.BaseballCallback", + "avro.examples.baseball.Player", + "avro.examples.baseball.Position" + }, + new string[] + { + "avro/examples/baseball/Baseball.cs", + "avro/examples/baseball/BaseballCallback.cs", + "avro/examples/baseball/Player.cs", + "avro/examples/baseball/Position.cs" + })] + [TestCase( + _comments, + new string[] + { + "testing.Comments", + "testing.CommentsCallback", + "testing.DocumentedEnum", + "testing.DocumentedError", + "testing.DocumentedFixed", + "testing.UndocumentedEnum", + "testing.UndocumentedFixed", + "testing.UndocumentedRecord" + }, + new string[] + { + "testing/Comments.cs", + "testing/CommentsCallback.cs", + "testing/DocumentedEnum.cs", + "testing/DocumentedError.cs", + "testing/DocumentedFixed.cs", + "testing/UndocumentedEnum.cs", + "testing/UndocumentedFixed.cs", + "testing/UndocumentedRecord.cs" + })] + [TestCase( + _interop, + new string[] + { + "org.apache.avro.interop.Label", + "org.apache.avro.interop.Interop", + "org.apache.avro.interop.InteropProtocol", + "org.apache.avro.interop.InteropProtocolCallback", + "org.apache.avro.interop.Kind", + "org.apache.avro.interop.MD5", + "org.apache.avro.interop.Node", + }, + new string[] + { + "org/apache/avro/interop/Label.cs", + "org/apache/avro/interop/Interop.cs", + "org/apache/avro/interop/InteropProtocol.cs", + "org/apache/avro/interop/InteropProtocolCallback.cs", + "org/apache/avro/interop/Kind.cs", + "org/apache/avro/interop/MD5.cs", + "org/apache/avro/interop/Node.cs", + })] + [TestCase( + _namespaces, + new string[] + { + "avro.test.enum.EnumInOtherNamespace", + "avro.test.error.ErrorInOtherNamespace", + "avro.test.fixed.FixedInOtherNamespace", + "avro.test.protocol.FixedInThisNamespace", + "avro.test.protocol.RefersToOthers", + "avro.test.protocol.TestNamespace", + "avro.test.protocol.TestNamespaceCallback", + "avro.test.record.RecordInOtherNamespace" + }, + new string[] + { + "avro/test/enum/EnumInOtherNamespace.cs", + "avro/test/error/ErrorInOtherNamespace.cs", + "avro/test/fixed/FixedInOtherNamespace.cs", + "avro/test/protocol/FixedInThisNamespace.cs", + "avro/test/protocol/RefersToOthers.cs", + "avro/test/protocol/TestNamespace.cs", + "avro/test/protocol/TestNamespaceCallback.cs", + "avro/test/record/RecordInOtherNamespace.cs" + })] + [TestCase( + _forwardRef, + new string[] + { + "org.foo.ANameValue", + "org.foo.Import", + "org.foo.ImportCallback", + "org.foo.ValueType" + }, + new string[] + { + "org/foo/ANameValue.cs", + "org/foo/Import.cs", + "org/foo/ImportCallback.cs", + "org/foo/ValueType.cs" + })] + [TestCase( + _unicode, + new string[] + { + "org.avro.test.Протоколы", + "org.avro.test.ПротоколыCallback", + "org.avro.test.Структура" + }, + new string[] + { + "org/avro/test/Протоколы.cs", + "org/avro/test/ПротоколыCallback.cs", + "org/avro/test/Структура.cs" + })] + [TestCase( + _myProtocol, + new string[] + { + "com.foo.A", + "com.foo.MyEnum", + "com.foo.MyFixed", + "com.foo.MyProtocol", + "com.foo.MyProtocolCallback", + "com.foo.newRec", + "com.foo.Z" + }, + new string[] + { + "com/foo/A.cs", + "com/foo/MyEnum.cs", + "com/foo/MyFixed.cs", + "com/foo/MyProtocol.cs", + "com/foo/MyProtocolCallback.cs", + "com/foo/newRec.cs", + "com/foo/Z.cs" + })] + public void GenerateProtocol(string protocol, IEnumerable typeNamesToCheck, IEnumerable generatedFilesToCheck) + { + AvroGenHelper.TestProtocol(protocol, typeNamesToCheck, generatedFilesToCheck: generatedFilesToCheck); + } + } +} diff --git a/lang/csharp/src/apache/test/AvroGen/AvroGenTests.cs b/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs similarity index 98% rename from lang/csharp/src/apache/test/AvroGen/AvroGenTests.cs rename to lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs index 1f61fc547dc..912284d6052 100644 --- a/lang/csharp/src/apache/test/AvroGen/AvroGenTests.cs +++ b/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs @@ -28,7 +28,7 @@ namespace Avro.Test.AvroGen { [TestFixture] - class AvroGenTests + class AvroGenSchemaTests { private const string _customConversionWithLogicalTypes = @" { @@ -489,7 +489,7 @@ private Assembly TestSchema( })] public void GenerateSchema(string schema, IEnumerable typeNamesToCheck, IEnumerable generatedFilesToCheck) { - TestSchema(schema, typeNamesToCheck, generatedFilesToCheck: generatedFilesToCheck); + AvroGenHelper.TestSchema(schema, typeNamesToCheck, generatedFilesToCheck: generatedFilesToCheck); } [TestCase( @@ -603,7 +603,7 @@ public void GenerateSchemaWithNamespaceMapping( IEnumerable typeNamesToCheck, IEnumerable generatedFilesToCheck) { - TestSchema(schema, typeNamesToCheck, new Dictionary { { namespaceMappingFrom, namespaceMappingTo } }, generatedFilesToCheck); + AvroGenHelper.TestSchema(schema, typeNamesToCheck, new Dictionary { { namespaceMappingFrom, namespaceMappingTo } }, generatedFilesToCheck); } [TestCase(_logicalTypesWithCustomConversion, typeof(AvroTypeException))] @@ -729,7 +729,7 @@ public void NotSupportedSchema(string schema, Type expectedException) new object[] { "enum.base.EnumInDifferentNamespace", "enum.base.other.AnEnum" })] public void GenerateSchemaCheckFields(string schema, object[] result) { - Assembly assembly = TestSchema(schema); + Assembly assembly = AvroGenHelper.TestSchema(schema); // Instantiate object Type type = assembly.GetType((string)result[0]); From 7e7e286331ffaa2fb51413e580d9033c423abb4a Mon Sep 17 00:00:00 2001 From: Martin Grigorov Date: Wed, 20 Apr 2022 21:07:35 +0300 Subject: [PATCH 37/66] AVRO-3484: Add support for deriving a default value for a record field (#1651) Signed-off-by: Martin Tzvetanov Grigorov --- lang/rust/avro_derive/Cargo.toml | 5 +- lang/rust/avro_derive/src/lib.rs | 18 +++- lang/rust/avro_derive/tests/derive.rs | 130 ++++++++++++++++++++++++++ 3 files changed, 147 insertions(+), 6 deletions(-) diff --git a/lang/rust/avro_derive/Cargo.toml b/lang/rust/avro_derive/Cargo.toml index e16e9ea957a..2ba8bc8c0ef 100644 --- a/lang/rust/avro_derive/Cargo.toml +++ b/lang/rust/avro_derive/Cargo.toml @@ -32,10 +32,11 @@ documentation = "https://docs.rs/apache-avro-derive" proc-macro = true [dependencies] -syn = {version= "1.0.91", features=["full", "fold"]} +darling = "0.14.0" quote = "1.0.18" +syn = {version= "1.0.91", features=["full", "fold"]} proc-macro2 = "1.0.37" -darling = "0.14.0" +serde_json = "1.0.79" [dev-dependencies] serde = { version = "1.0.136", features = ["derive"] } diff --git a/lang/rust/avro_derive/src/lib.rs b/lang/rust/avro_derive/src/lib.rs index a32f2be3348..42362537547 100644 --- a/lang/rust/avro_derive/src/lib.rs +++ b/lang/rust/avro_derive/src/lib.rs @@ -30,6 +30,8 @@ use syn::{ struct FieldOptions { #[darling(default)] doc: Option, + #[darling(default)] + default: Option, } #[derive(FromAttributes)] @@ -117,16 +119,24 @@ fn get_data_struct_schema_def( syn::Fields::Named(ref a) => { for (position, field) in a.named.iter().enumerate() { let name = field.ident.as_ref().unwrap().to_string(); // we know everything has a name - let field_documented = + let field_attrs = FieldOptions::from_attributes(&field.attrs[..]).map_err(darling_to_syn)?; - let doc = preserve_optional(field_documented.doc); + let doc = preserve_optional(field_attrs.doc); + let default_value = match field_attrs.default { + Some(default_value) => { + quote! { + Some(serde_json::from_str(#default_value).expect(format!("Invalid JSON: {:?}", #default_value).as_str())) + } + } + None => quote! { None }, + }; let schema_expr = type_to_schema_expr(&field.ty)?; let position = position; record_field_exprs.push(quote! { apache_avro::schema::RecordField { name: #name.to_string(), doc: #doc, - default: Option::None, + default: #default_value, schema: #schema_expr, order: apache_avro::schema::RecordFieldOrder::Ascending, position: #position, @@ -186,7 +196,7 @@ fn get_data_enum_schema_def( name: apache_avro::schema::Name::new(#full_schema_name).expect(&format!("Unable to parse enum name for schema {}", #full_schema_name)[..]), aliases: #enum_aliases, doc: #doc, - symbols: vec![#(#symbols.to_owned()),*] + symbols: vec![#(#symbols.to_owned()),*], } }) } else { diff --git a/lang/rust/avro_derive/tests/derive.rs b/lang/rust/avro_derive/tests/derive.rs index 284323a2af4..0be2bd4bfc8 100644 --- a/lang/rust/avro_derive/tests/derive.rs +++ b/lang/rust/avro_derive/tests/derive.rs @@ -1209,4 +1209,134 @@ mod test_derive { serde_assert(TestBasicEnumWithAliases2::B); } + + #[test] + fn test_basic_struct_with_defaults() { + #[derive(Debug, Deserialize, Serialize, AvroSchema, Clone, PartialEq)] + enum MyEnum { + Foo, + Bar, + Baz, + } + + #[derive(Debug, Serialize, Deserialize, AvroSchema, Clone, PartialEq)] + struct TestBasicStructWithDefaultValues { + #[avro(default = "123")] + a: i32, + #[avro(default = r#""The default value for 'b'""#)] + b: String, + #[avro(default = "true")] + condition: bool, + // no default value for 'c' + c: f64, + #[avro(default = r#"{"a": 1, "b": 2}"#)] + map: HashMap, + + #[avro(default = "[1, 2, 3]")] + array: Vec, + + #[avro(default = r#""Foo""#)] + myenum: MyEnum, + } + + let schema = r#" + { + "type":"record", + "name":"TestBasicStructWithDefaultValues", + "fields": [ + { + "name":"a", + "type":"int", + "default":123 + }, + { + "name":"b", + "type":"string", + "default": "The default value for 'b'" + }, + { + "name":"condition", + "type":"boolean", + "default":true + }, + { + "name":"c", + "type":"double" + }, + { + "name":"map", + "type":{ + "type":"map", + "values":"int" + }, + "default": { + "a": 1, + "b": 2 + } + }, + { + "name":"array", + "type":{ + "type":"array", + "items":"int" + }, + "default": [1, 2, 3] + }, + { + "name":"myenum", + "type":{ + "type":"enum", + "name":"MyEnum", + "symbols":["Foo", "Bar", "Baz"] + }, + "default":"Foo" + } + ] + } + "#; + + let schema = Schema::parse_str(schema).unwrap(); + if let Schema::Record { name, fields, .. } = TestBasicStructWithDefaultValues::get_schema() + { + assert_eq!("TestBasicStructWithDefaultValues", name.fullname(None)); + use serde_json::json; + for field in fields { + match field.name.as_str() { + "a" => assert_eq!(Some(json!(123_i32)), field.default), + "b" => assert_eq!( + Some(json!(r#"The default value for 'b'"#.to_owned())), + field.default + ), + "condition" => assert_eq!(Some(json!(true)), field.default), + "array" => assert_eq!(Some(json!([1, 2, 3])), field.default), + "map" => assert_eq!( + Some(json!({ + "a": 1, + "b": 2 + })), + field.default + ), + "c" => assert_eq!(None, field.default), + "myenum" => assert_eq!(Some(json!("Foo")), field.default), + _ => panic!("Unexpected field name"), + } + } + } else { + panic!("TestBasicStructWithDefaultValues schema must be a record schema") + } + assert_eq!(schema, TestBasicStructWithDefaultValues::get_schema()); + + serde_assert(TestBasicStructWithDefaultValues { + a: 321, + b: "A custom value for 'b'".to_owned(), + condition: false, + c: 987.654, + map: [("a".to_owned(), 1), ("b".to_owned(), 2)] + .iter() + .cloned() + .collect(), + array: vec![4, 5, 6], + myenum: MyEnum::Bar, + }); + } } From 80ecb272981e51a8b428d248d7ce38452401b798 Mon Sep 17 00:00:00 2001 From: Kyle Schoonover Date: Wed, 20 Apr 2022 12:57:04 -0700 Subject: [PATCH 38/66] AVRO-3497 Simplify conditional expression (#1658) * AVRO-3497 Simplify conditional expression * Added null check back * Updated tests --- .../src/apache/main/Generic/GenericEnum.cs | 12 ++-- .../apache/test/Generic/GenericEnumTests.cs | 72 +++++++++++++++++++ 2 files changed, 80 insertions(+), 4 deletions(-) create mode 100644 lang/csharp/src/apache/test/Generic/GenericEnumTests.cs diff --git a/lang/csharp/src/apache/main/Generic/GenericEnum.cs b/lang/csharp/src/apache/main/Generic/GenericEnum.cs index db58bb1ec5e..27bba94ce15 100644 --- a/lang/csharp/src/apache/main/Generic/GenericEnum.cs +++ b/lang/csharp/src/apache/main/Generic/GenericEnum.cs @@ -70,10 +70,14 @@ public GenericEnum(EnumSchema schema, string value) /// public override bool Equals(object obj) { - if (obj == this) return true; - return (obj != null && obj is GenericEnum) - ? Value.Equals((obj as GenericEnum).Value, System.StringComparison.Ordinal) - : false; + if (obj == this) + { + return true; + } + + return obj != null + && obj.GetType() == typeof(GenericEnum) + && Value.Equals(((GenericEnum)obj).Value, System.StringComparison.Ordinal); } /// diff --git a/lang/csharp/src/apache/test/Generic/GenericEnumTests.cs b/lang/csharp/src/apache/test/Generic/GenericEnumTests.cs new file mode 100644 index 00000000000..aba0038ea1a --- /dev/null +++ b/lang/csharp/src/apache/test/Generic/GenericEnumTests.cs @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Avro.Generic; +using NUnit.Framework; + +namespace Avro.test.Generic +{ + [TestFixture] + public class GenericEnumTests + { + private const string baseSchema = "{\"type\": \"enum\", \"name\": \"Test\", \"symbols\": " + + "[\"Unknown\", \"A\", \"B\"], \"default\": \"Unknown\" }"; + + [Test] + public void TestEquals() + { + GenericEnum genericEnum = GetBaseGenericEnum(); + GenericEnum genericEnum2 = GetBaseGenericEnum(); + + Assert.IsTrue(genericEnum.Equals(genericEnum2)); + } + + [Test] + public void TestEqualsNotEqual() + { + GenericEnum genericEnum = GetBaseGenericEnum(); + GenericEnum genericEnum2 = new GenericEnum(Schema.Parse(baseSchema) as EnumSchema, "B"); + + Assert.IsFalse(genericEnum.Equals(genericEnum2)); + } + + [Test] + public void TestEqualsObject() + { + GenericEnum genericEnum = GetBaseGenericEnum(); + object genericEnum2 = genericEnum; + + Assert.IsTrue(genericEnum.Equals(genericEnum2)); + } + + [Test] + public void TestEqualsObjectNullObject() + { + GenericEnum genericEnum = GetBaseGenericEnum(); + + Assert.IsFalse(genericEnum.Equals(null)); + } + + private GenericEnum GetBaseGenericEnum() + { + GenericEnum genericEnum = new GenericEnum(Schema.Parse(baseSchema) as EnumSchema, "A"); + + return genericEnum; + } + } +} From 9cb9275be4668952a61b8963d4be618781987c81 Mon Sep 17 00:00:00 2001 From: Martin Grigorov Date: Wed, 20 Apr 2022 22:58:41 +0300 Subject: [PATCH 39/66] AVRO-3500: Use property-based testing for the IT tests in avro_derive module (#1659) Signed-off-by: Martin Tzvetanov Grigorov --- lang/rust/avro/tests/schema.rs | 1 - lang/rust/avro_derive/Cargo.toml | 3 +- lang/rust/avro_derive/tests/derive.rs | 162 ++++++++++++++------------ 3 files changed, 87 insertions(+), 79 deletions(-) diff --git a/lang/rust/avro/tests/schema.rs b/lang/rust/avro/tests/schema.rs index 62eeb2ddf3d..376af6c72b1 100644 --- a/lang/rust/avro/tests/schema.rs +++ b/lang/rust/avro/tests/schema.rs @@ -159,7 +159,6 @@ const RECORD_EXAMPLES: &[(&str, bool)] = &[ }"#, false, ), - // TODO: (#92) properly support recursive types and uncomment ( r#"{ "type": "record", diff --git a/lang/rust/avro_derive/Cargo.toml b/lang/rust/avro_derive/Cargo.toml index 2ba8bc8c0ef..2dfe2e73323 100644 --- a/lang/rust/avro_derive/Cargo.toml +++ b/lang/rust/avro_derive/Cargo.toml @@ -39,5 +39,6 @@ proc-macro2 = "1.0.37" serde_json = "1.0.79" [dev-dependencies] -serde = { version = "1.0.136", features = ["derive"] } apache-avro = { version = "0.14.0", path = "../avro", features = ["derive"] } +proptest = "1.0.0" +serde = { version = "1.0.136", features = ["derive"] } diff --git a/lang/rust/avro_derive/tests/derive.rs b/lang/rust/avro_derive/tests/derive.rs index 0be2bd4bfc8..0886156ed46 100644 --- a/lang/rust/avro_derive/tests/derive.rs +++ b/lang/rust/avro_derive/tests/derive.rs @@ -21,6 +21,7 @@ use apache_avro::{ Reader, Schema, Writer, }; use apache_avro_derive::*; +use proptest::prelude::*; use serde::{de::DeserializeOwned, ser::Serialize}; use std::collections::HashMap; @@ -87,8 +88,9 @@ mod test_derive { b: String, } + proptest! { #[test] - fn test_smoke_test() { + fn test_smoke_test(a: i32, b: String) { let schema = r#" { "type":"record", @@ -108,11 +110,11 @@ mod test_derive { let schema = Schema::parse_str(schema).unwrap(); assert_eq!(schema, TestBasic::get_schema()); let test = TestBasic { - a: 27, - b: "foo".to_owned(), + a, + b, }; serde_assert(test); - } + }} #[derive(Debug, Serialize, Deserialize, AvroSchema, Clone, PartialEq)] #[avro(namespace = "com.testing.namespace")] @@ -226,8 +228,9 @@ mod test_derive { j: String, } + proptest! { #[test] - fn test_basic_types() { + fn test_basic_types(a: bool, b: i8, c: i16, d: i32, e: u8, f: u16, g: i64, h: f32, i: f64, j: String) { let schema = r#" { "type":"record", @@ -278,21 +281,20 @@ mod test_derive { "#; let schema = Schema::parse_str(schema).unwrap(); assert_eq!(schema, TestAllSupportedBaseTypes::get_schema()); - // TODO mgrigorov Use property based testing in the future let all_basic = TestAllSupportedBaseTypes { - a: true, - b: 8_i8, - c: 16_i16, - d: 32_i32, - e: 8_u8, - f: 16_u16, - g: 64_i64, - h: 32.3333_f32, - i: 64.4444_f64, - j: "testing string".to_owned(), + a, + b, + c, + d, + e, + f, + g, + h, + i, + j, }; serde_assert(all_basic); - } + }} #[derive(Debug, Serialize, Deserialize, AvroSchema, Clone, PartialEq)] struct TestNested { @@ -300,8 +302,9 @@ mod test_derive { b: TestAllSupportedBaseTypes, } + proptest! { #[test] - fn test_inner_struct() { + fn test_inner_struct(a: bool, b: i8, c: i16, d: i32, e: u8, f: u16, g: i64, h: f32, i: f64, j: String, aa: i32) { let schema = r#" { "type":"record", @@ -365,33 +368,33 @@ mod test_derive { "#; let schema = Schema::parse_str(schema).unwrap(); assert_eq!(schema, TestNested::get_schema()); - // TODO mgrigorov Use property based testing in the future let all_basic = TestAllSupportedBaseTypes { - a: true, - b: 8_i8, - c: 16_i16, - d: 32_i32, - e: 8_u8, - f: 16_u16, - g: 64_i64, - h: 32.3333_f32, - i: 64.4444_f64, - j: "testing string".to_owned(), + a, + b, + c, + d, + e, + f, + g, + h, + i, + j, }; let inner_struct = TestNested { - a: -1600, + a: aa, b: all_basic, }; serde_assert(inner_struct); - } + }} #[derive(Debug, Serialize, Deserialize, AvroSchema, Clone, PartialEq)] struct TestOptional { a: Option, } + proptest! { #[test] - fn test_optional_field_some() { + fn test_optional_field_some(a: i32) { let schema = r#" { "type":"record", @@ -406,9 +409,9 @@ mod test_derive { "#; let schema = Schema::parse_str(schema).unwrap(); assert_eq!(schema, TestOptional::get_schema()); - let optional_field = TestOptional { a: Some(4) }; + let optional_field = TestOptional { a: Some(a) }; serde_assert(optional_field); - } + }} #[test] fn test_optional_field_none() { @@ -424,8 +427,9 @@ mod test_derive { c: HashMap, } + proptest! { #[test] - fn test_generic_container_1() { + fn test_generic_container_1(a: String, b: Vec, c: HashMap) { let schema = r#" { "type":"record", @@ -455,15 +459,16 @@ mod test_derive { let schema = Schema::parse_str(schema).unwrap(); assert_eq!(schema, TestGeneric::::get_schema()); let test_generic = TestGeneric:: { - a: "testing".to_owned(), - b: vec![0, 1, 2, 3], - c: vec![("key".to_owned(), 3)].into_iter().collect(), + a, + b, + c, }; serde_assert(test_generic); - } + }} + proptest! { #[test] - fn test_generic_container_2() { + fn test_generic_container_2(a: bool, b: i8, c: i16, d: i32, e: u8, f: u16, g: i64, h: f32, i: f64, j: String) { let schema = r#" { "type":"record", @@ -543,37 +548,37 @@ mod test_derive { let test_generic = TestGeneric:: { a: "testing".to_owned(), b: vec![TestAllSupportedBaseTypes { - a: true, - b: 8_i8, - c: 16_i16, - d: 32_i32, - e: 8_u8, - f: 16_u16, - g: 64_i64, - h: 32.3333_f32, - i: 64.4444_f64, - j: "testing string".to_owned(), + a, + b, + c, + d, + e, + f, + g, + h, + i, + j: j.clone(), }], c: vec![( "key".to_owned(), TestAllSupportedBaseTypes { - a: true, - b: 8_i8, - c: 16_i16, - d: 32_i32, - e: 8_u8, - f: 16_u16, - g: 64_i64, - h: 32.3333_f32, - i: 64.4444_f64, - j: "testing string".to_owned(), + a, + b, + c, + d, + e, + f, + g, + h, + i, + j, }, )] .into_iter() .collect(), }; serde_assert(test_generic); - } + }} #[derive(Debug, Serialize, Deserialize, AvroSchema, Clone, PartialEq)] enum TestAllowedEnum { @@ -723,8 +728,9 @@ mod test_derive { a: [i32; 4], } + proptest! { #[test] - fn test_simple_array() { + fn test_simple_array(a: [i32; 4]) { let schema = r#" { "type":"record", @@ -742,9 +748,9 @@ mod test_derive { "#; let schema = Schema::parse_str(schema).unwrap(); assert_eq!(schema, TestSimpleArray::get_schema()); - let test = TestSimpleArray { a: [2, 3, 4, 5] }; + let test = TestSimpleArray { a }; serde_assert(test) - } + }} #[derive(Debug, Serialize, Deserialize, AvroSchema, Clone, PartialEq)] struct TestComplexArray { @@ -803,15 +809,17 @@ mod test_derive { a: Vec, b: [u8; 2], } + + proptest! { #[test] - fn test_bytes_handled() { + fn test_bytes_handled(a: Vec, b: [u8; 2]) { let test = Testu8 { - a: vec![1, 2], - b: [3, 4], + a, + b, }; serde_assert(test) // don't check for schema equality to allow for transitioning to bytes or fixed types in the future - } + }} #[derive(Debug, Serialize, Deserialize, AvroSchema)] #[allow(unknown_lints)] // Rust 1.51.0 (MSRV) does not support #[allow(clippy::box_collection)] @@ -868,8 +876,9 @@ mod test_derive { c: &'a f64, } + proptest! { #[test] - fn test_reference_struct() { + fn test_reference_struct(a: Vec, c: f64) { let schema = r#" { "type":"record", @@ -895,15 +904,15 @@ mod test_derive { "#; let schema = Schema::parse_str(schema).unwrap(); assert_eq!(schema, TestReference::get_schema()); - let a = vec![34]; - let c = 4.55555555_f64; + // let a = vec![34]; + // let c = 4.55555555_f64; let test = TestReference { a: &a, b: "testing_static", c: &c, }; ser(test); - } + }} #[derive(Debug, Serialize, Deserialize, AvroSchema, Clone, PartialEq)] #[avro(namespace = "com.testing.namespace", doc = "A Documented Record")] @@ -1037,8 +1046,9 @@ mod test_derive { a: u32, } + proptest! { #[test] - fn test_basic_with_u32() { + fn test_basic_with_u32(a in any::()) { let schema = r#" { "type":"record", @@ -1059,10 +1069,8 @@ mod test_derive { } assert_eq!(schema, TestBasicWithU32::get_schema()); - serde_assert(TestBasicWithU32 { a: u32::MAX }); - serde_assert(TestBasicWithU32 { a: u32::MIN }); - serde_assert(TestBasicWithU32 { a: 1_u32 }); - } + serde_assert(TestBasicWithU32 { a }); + }} #[derive(Debug, Serialize, Deserialize, AvroSchema, Clone, PartialEq)] #[avro(alias = "a", alias = "b", alias = "c")] From faa355cd75c528e59a770477b1602036318fba22 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Thu, 21 Apr 2022 14:42:13 +0300 Subject: [PATCH 40/66] Configure Dependabot to check for Rust updates daily Signed-off-by: Martin Tzvetanov Grigorov --- .github/dependabot.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 20598218472..554b8987f33 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -82,6 +82,7 @@ updates: - package-ecosystem: "cargo" directory: "/lang/rust/" schedule: - interval: "weekly" - day: "sunday" + interval: "daily" open-pull-requests-limit: 20 + target-branch: master + labels: [auto-dependencies] From 1d591390b11651dfb9577d537b39587eb0dcda7d Mon Sep 17 00:00:00 2001 From: Martin Grigorov Date: Thu, 21 Apr 2022 16:10:14 +0300 Subject: [PATCH 41/66] AVRO-3501: Rust: Cache ~/.cargo and target folder for faster builds (#1661) Signed-off-by: Martin Tzvetanov Grigorov --- .github/workflows/test-lang-rust-ci.yml | 30 +++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.github/workflows/test-lang-rust-ci.yml b/.github/workflows/test-lang-rust-ci.yml index e89f4ca65a6..a1837096311 100644 --- a/.github/workflows/test-lang-rust-ci.yml +++ b/.github/workflows/test-lang-rust-ci.yml @@ -49,6 +49,21 @@ jobs: - name: Checkout uses: actions/checkout@v2 + - name: Cache Cargo + uses: actions/cache@v2 + with: + # these represent dependencies downloaded by cargo + # and thus do not depend on the OS, arch nor rust version. + path: ~/.cargo + key: cargo-cache1- + - name: Cache Rust dependencies + uses: actions/cache@v2 + with: + # these represent compiled steps of both dependencies and avro + # and thus are specific for a particular OS, arch and rust version. + path: ~/target + key: ${{ runner.os }}-target-cache1-${{ matrix.rust }}- + - name: Rust Toolchain uses: actions-rs/toolchain@v1 with: @@ -96,6 +111,21 @@ jobs: toolchain: stable override: true + - name: Cache Cargo + uses: actions/cache@v2 + with: + # these represent dependencies downloaded by cargo + # and thus do not depend on the OS, arch nor rust version. + path: ~/.cargo + key: cargo-cache1- + - name: Cache Rust dependencies + uses: actions/cache@v2 + with: + # these represent compiled steps of both dependencies and avro + # and thus are specific for a particular OS, arch and rust version. + path: ~/target + key: ${{ runner.os }}-target-cache1-stable- + - name: Cache Local Maven Repository uses: actions/cache@v2 with: From d0b679ce6bac8889da192daee10c4717cfbaef3a Mon Sep 17 00:00:00 2001 From: Martin Grigorov Date: Thu, 21 Apr 2022 21:12:17 +0300 Subject: [PATCH 42/66] Avro 3502 logical type wrong order (#1664) * AVRO-3501: Rust: Cache ~/.cargo and target folder for faster builds Signed-off-by: Martin Tzvetanov Grigorov * AVRO-3502: Rust: Wrong [ORDER] for Parsing Canonical Form Signed-off-by: Martin Tzvetanov Grigorov --- lang/rust/avro/src/schema.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lang/rust/avro/src/schema.rs b/lang/rust/avro/src/schema.rs index 4e860a9dad4..1b618bcbc85 100644 --- a/lang/rust/avro/src/schema.rs +++ b/lang/rust/avro/src/schema.rs @@ -1591,8 +1591,8 @@ const RESERVED_FIELDS: &[&str] = &[ "symbols", "items", "values", - "logicalType", "size", + "logicalType", "order", "doc", "aliases", From 7d060ab4062a56c94d9af32cc43f2c8616228378 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Apr 2022 21:13:27 +0300 Subject: [PATCH 43/66] Update uuid requirement from 0.8.2 to 1.0.0 in /lang/rust (#1660) * Update uuid requirement from 0.8.2 to 1.0.0 in /lang/rust Updates the requirements on [uuid](https://github.com/uuid-rs/uuid) to permit the latest version. - [Release notes](https://github.com/uuid-rs/uuid/releases) - [Commits](https://github.com/uuid-rs/uuid/compare/0.8.2...1.0.0) --- updated-dependencies: - dependency-name: uuid dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Issue #1660 - Fix compilation errors after updating uuid crate from 0.8 to 1.0 Signed-off-by: Martin Tzvetanov Grigorov Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Martin Tzvetanov Grigorov --- lang/rust/avro/Cargo.toml | 2 +- lang/rust/avro/src/encode.rs | 8 +++++++- lang/rust/avro/src/lib.rs | 2 +- lang/rust/avro/src/types.rs | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/lang/rust/avro/Cargo.toml b/lang/rust/avro/Cargo.toml index be547acca91..15a9eb980af 100644 --- a/lang/rust/avro/Cargo.toml +++ b/lang/rust/avro/Cargo.toml @@ -69,7 +69,7 @@ strum = "0.24.0" strum_macros = "0.24.0" thiserror = "1.0.30" typed-builder = "0.10.0" -uuid = { version = "0.8.2", features = ["serde", "v4"] } +uuid = { version = "1.0.0", features = ["serde", "v4"] } zerocopy = "0.6.1" lazy_static = "1.4.0" log = "0.4.16" diff --git a/lang/rust/avro/src/encode.rs b/lang/rust/avro/src/encode.rs index 014a6f747fb..cf4e0f5da02 100644 --- a/lang/rust/avro/src/encode.rs +++ b/lang/rust/avro/src/encode.rs @@ -105,7 +105,13 @@ pub(crate) fn encode_internal>( let slice: [u8; 12] = duration.into(); buffer.extend_from_slice(&slice); } - Value::Uuid(uuid) => encode_bytes(&uuid.to_string(), buffer), + Value::Uuid(uuid) => encode_bytes( + #[allow(unknown_lints)] // for Rust 1.51.0 + #[allow(clippy::unnecessary_to_owned)] + // we need the call .to_string() to properly convert ASCII to UTF-8 + &uuid.to_string(), + buffer, + ), Value::Bytes(bytes) => match *schema { Schema::Bytes => encode_bytes(bytes, buffer), Schema::Fixed { .. } => buffer.extend(bytes), diff --git a/lang/rust/avro/src/lib.rs b/lang/rust/avro/src/lib.rs index ba9ec3a3194..fc391f6ca3b 100644 --- a/lang/rust/avro/src/lib.rs +++ b/lang/rust/avro/src/lib.rs @@ -512,7 +512,7 @@ //! `apache-avro` also supports the logical types listed in the [Avro specification](https://avro.apache.org/docs/current/spec.html#Logical+Types): //! //! 1. `Decimal` using the [`num_bigint`](https://docs.rs/num-bigint/0.2.6/num_bigint) crate -//! 1. UUID using the [`uuid`](https://docs.rs/uuid/0.8.1/uuid) crate +//! 1. UUID using the [`uuid`](https://docs.rs/uuid/1.0.0/uuid) crate //! 1. Date, Time (milli) as `i32` and Time (micro) as `i64` //! 1. Timestamp (milli and micro) as `i64` //! 1. Duration as a custom type with `months`, `days` and `millis` accessor methods each of which returns an `i32` diff --git a/lang/rust/avro/src/types.rs b/lang/rust/avro/src/types.rs index 24bfeeed2a1..78b000552e9 100644 --- a/lang/rust/avro/src/types.rs +++ b/lang/rust/avro/src/types.rs @@ -328,7 +328,7 @@ impl std::convert::TryFrom for JsonValue { Value::Duration(d) => Ok(Self::Array( <[u8; 12]>::from(d).iter().map(|&v| v.into()).collect(), )), - Value::Uuid(uuid) => Ok(Self::String(uuid.to_hyphenated().to_string())), + Value::Uuid(uuid) => Ok(Self::String(uuid.as_hyphenated().to_string())), } } } From b44b520ef86c574ca02bcc8db1ab956fce5c4205 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Apr 2022 21:14:51 +0300 Subject: [PATCH 44/66] Bump jmh.version from 1.34 to 1.35 in /lang/java (#1662) Bumps `jmh.version` from 1.34 to 1.35. Updates `jmh-core` from 1.34 to 1.35 Updates `jmh-generator-annprocess` from 1.34 to 1.35 --- updated-dependencies: - dependency-name: org.openjdk.jmh:jmh-core dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: org.openjdk.jmh:jmh-generator-annprocess dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- lang/java/perf/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lang/java/perf/pom.xml b/lang/java/perf/pom.xml index 4fb6b1184b9..fd456755bee 100644 --- a/lang/java/perf/pom.xml +++ b/lang/java/perf/pom.xml @@ -34,7 +34,7 @@ jar - 1.34 + 1.35 ${project.parent.parent.basedir} From 19bd6625b8884d91b12c98756124dd5e358590d2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Apr 2022 21:15:22 +0300 Subject: [PATCH 45/66] Bump zstd-jni from 1.5.1-1 to 1.5.2-2 in /lang/java (#1663) Bumps [zstd-jni](https://github.com/luben/zstd-jni) from 1.5.1-1 to 1.5.2-2. - [Release notes](https://github.com/luben/zstd-jni/releases) - [Commits](https://github.com/luben/zstd-jni/compare/v1.5.1-1...v1.5.2-2) --- updated-dependencies: - dependency-name: com.github.luben:zstd-jni dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- lang/java/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lang/java/pom.xml b/lang/java/pom.xml index 36f94fc0794..6c96f355b67 100644 --- a/lang/java/pom.xml +++ b/lang/java/pom.xml @@ -59,7 +59,7 @@ 4.3.1 2.2 1.45.0 - 1.5.1-1 + 1.5.2-2 3.2.1 5.1.4 From 95643aed0388638431b048ad90c9598d671c2a55 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Apr 2022 21:15:51 +0300 Subject: [PATCH 46/66] Bump libthrift from 0.15.0 to 0.16.0 in /lang/java (#1665) Bumps [libthrift](https://github.com/apache/thrift) from 0.15.0 to 0.16.0. - [Release notes](https://github.com/apache/thrift/releases) - [Changelog](https://github.com/apache/thrift/blob/master/CHANGES.md) - [Commits](https://github.com/apache/thrift/compare/v0.15.0...v0.16.0) --- updated-dependencies: - dependency-name: org.apache.thrift:libthrift dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- lang/java/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lang/java/pom.xml b/lang/java/pom.xml index 6c96f355b67..19c59ff7384 100644 --- a/lang/java/pom.xml +++ b/lang/java/pom.xml @@ -45,7 +45,7 @@ 4.13.2 4.1.75.Final 3.19.4 - 0.15.0 + 0.16.0 1.7.36 1.2.19 1.1.8.4 From 31c4bc3c55f98ccd3663e7daf9338ac915716814 Mon Sep 17 00:00:00 2001 From: Kyle Schoonover Date: Thu, 21 Apr 2022 11:17:27 -0700 Subject: [PATCH 47/66] AVRO-3498 Deprecate NameCtorKey (#1657) --- .../src/apache/main/Specific/ObjectCreator.cs | 58 ------------------- 1 file changed, 58 deletions(-) diff --git a/lang/csharp/src/apache/main/Specific/ObjectCreator.cs b/lang/csharp/src/apache/main/Specific/ObjectCreator.cs index e69a490283d..073b107958a 100644 --- a/lang/csharp/src/apache/main/Specific/ObjectCreator.cs +++ b/lang/csharp/src/apache/main/Specific/ObjectCreator.cs @@ -58,13 +58,6 @@ public sealed class ObjectCreator private readonly Assembly entryAssembly; private readonly bool diffAssembly; - /// - /// Obsolete: This will be removed from the public API in a future version. - /// - /// Obsolete - [Obsolete("This will be removed from the public API in a future version.")] - public delegate object CtorDelegate(); - /// /// Initializes a new instance of the class. /// @@ -78,57 +71,6 @@ public ObjectCreator() diffAssembly = entryAssembly != null && execAssembly != entryAssembly; } -#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member -#pragma warning disable CA1034 // Nested types should not be visible -#pragma warning disable SA1600 // Elements should be documented - /// - /// Obsolete: This will be removed from the public API in a future version. - /// - [Obsolete("This will be removed from the public API in a future version.")] - public struct NameCtorKey : IEquatable - { - public string name { get; private set; } - public Schema.Type type { get; private set; } - public NameCtorKey(string value1, Schema.Type value2) - : this() - { - name = value1; - type = value2; - } - public bool Equals(NameCtorKey other) - { - return Equals(other.name, name) && other.type == type; - } - public override bool Equals(object obj) - { - if (ReferenceEquals(null, obj)) - return false; - if (obj.GetType() != typeof(NameCtorKey)) - return false; - return Equals((NameCtorKey)obj); - } - public override int GetHashCode() - { - unchecked - { -#pragma warning disable CA1307 // Specify StringComparison - return ((name != null ? name.GetHashCode() : 0) * 397) ^ type.GetHashCode(); -#pragma warning restore CA1307 // Specify StringComparison - } - } - public static bool operator ==(NameCtorKey left, NameCtorKey right) - { - return left.Equals(right); - } - public static bool operator !=(NameCtorKey left, NameCtorKey right) - { - return !left.Equals(right); - } - } -#pragma warning restore SA1600 // Elements should be documented -#pragma warning restore CA1034 // Nested types should not be visible -#pragma warning restore CS1591 // Missing XML comment for publicly visible type or member - /// /// Find the type with the given name /// From cddea879975934741af1a36d896222fdb9ce501e Mon Sep 17 00:00:00 2001 From: Kyle Schoonover Date: Thu, 21 Apr 2022 11:19:00 -0700 Subject: [PATCH 48/66] AVRO-3490 Updated to use throw expressions (#1644) * AVRO-3360 Updated XML documentation * Revert "AVRO-3360 Updated XML documentation" This reverts commit b8601c072a5083380d30b580804dd0908b8cf4cc. * AVRO-3490 Updated to use throw expressions * Additional expressions Co-authored-by: Kyle T. Schoonover --- .../src/apache/main/Schema/ArraySchema.cs | 3 +- lang/csharp/src/apache/main/Schema/Field.cs | 33 ++++++++++--------- .../src/apache/main/Schema/LogicalSchema.cs | 3 +- .../src/apache/main/Schema/MapSchema.cs | 3 +- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/lang/csharp/src/apache/main/Schema/ArraySchema.cs b/lang/csharp/src/apache/main/Schema/ArraySchema.cs index 03a331289aa..7c4d8e1a9c8 100644 --- a/lang/csharp/src/apache/main/Schema/ArraySchema.cs +++ b/lang/csharp/src/apache/main/Schema/ArraySchema.cs @@ -66,8 +66,7 @@ public static ArraySchema Create(Schema items, PropertyMap customAttributes = nu private ArraySchema(Schema items, PropertyMap customAttributes) : base(Type.Array, customAttributes) { - if (null == items) throw new ArgumentNullException(nameof(items)); - this.ItemSchema = items; + ItemSchema = items ?? throw new ArgumentNullException(nameof(items)); } /// diff --git a/lang/csharp/src/apache/main/Schema/Field.cs b/lang/csharp/src/apache/main/Schema/Field.cs index 08ea03305d7..0c3f7afcaee 100644 --- a/lang/csharp/src/apache/main/Schema/Field.cs +++ b/lang/csharp/src/apache/main/Schema/Field.cs @@ -138,12 +138,7 @@ internal Field ChangePosition(int newPosition) } /// - /// A flag to indicate if reader schema has a field that is missing from writer schema and has a default value - /// This is set in CanRead() which is always be called before deserializing data - /// - - /// - /// Constructor for the field class + /// Initializes a new instance of the class. /// /// schema for the field type /// name of the field @@ -153,21 +148,29 @@ internal Field ChangePosition(int newPosition) /// field's default value if it exists /// sort order of the field /// dictionary that provides access to custom properties + /// + /// name - name cannot be null. + /// or + /// type - type cannot be null. + /// internal Field(Schema schema, string name, IList aliases, int pos, string doc, JToken defaultValue, SortOrder sortorder, PropertyMap props) { - if (string.IsNullOrEmpty(name)) throw new ArgumentNullException(nameof(name), "name cannot be null."); - if (null == schema) throw new ArgumentNullException("type", "type cannot be null."); - this.Schema = schema; - this.Name = name; + if (string.IsNullOrEmpty(name)) + { + throw new ArgumentNullException(nameof(name), "name cannot be null."); + } + + Schema = schema ?? throw new ArgumentNullException("type", "type cannot be null."); + Name = name; #pragma warning disable CS0618 // Type or member is obsolete this.aliases = aliases; #pragma warning restore CS0618 // Type or member is obsolete - this.Pos = pos; - this.Documentation = doc; - this.DefaultValue = defaultValue; - this.Ordering = sortorder; - this.Props = props; + Pos = pos; + Documentation = doc; + DefaultValue = defaultValue; + Ordering = sortorder; + Props = props; } /// diff --git a/lang/csharp/src/apache/main/Schema/LogicalSchema.cs b/lang/csharp/src/apache/main/Schema/LogicalSchema.cs index 3c1928ee47f..0f23bdf4dfe 100644 --- a/lang/csharp/src/apache/main/Schema/LogicalSchema.cs +++ b/lang/csharp/src/apache/main/Schema/LogicalSchema.cs @@ -52,8 +52,7 @@ internal static LogicalSchema NewInstance(JToken jtok, PropertyMap props, Schema private LogicalSchema(Schema baseSchema, string logicalTypeName, PropertyMap props) : base(Type.Logical, props) { - if (null == baseSchema) throw new ArgumentNullException(nameof(baseSchema)); - BaseSchema = baseSchema; + BaseSchema = baseSchema ?? throw new ArgumentNullException(nameof(baseSchema)); LogicalTypeName = logicalTypeName; LogicalType = LogicalTypeFactory.Instance.GetFromLogicalSchema(this); } diff --git a/lang/csharp/src/apache/main/Schema/MapSchema.cs b/lang/csharp/src/apache/main/Schema/MapSchema.cs index d9f4995abf6..a1a6a4222b9 100644 --- a/lang/csharp/src/apache/main/Schema/MapSchema.cs +++ b/lang/csharp/src/apache/main/Schema/MapSchema.cs @@ -73,8 +73,7 @@ internal static MapSchema NewInstance(JToken jtok, PropertyMap props, SchemaName private MapSchema(Schema valueSchema, PropertyMap cutsomProperties) : base(Type.Map, cutsomProperties) { - if (null == valueSchema) throw new ArgumentNullException(nameof(valueSchema), "valueSchema cannot be null."); - this.ValueSchema = valueSchema; + ValueSchema = valueSchema ?? throw new ArgumentNullException(nameof(valueSchema), "valueSchema cannot be null."); } /// From ddbb8cc16b83e5f375add642e82b92ad6d22c1ca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Apr 2022 20:59:59 +0300 Subject: [PATCH 49/66] Bump grpc.version from 1.45.0 to 1.45.1 in /lang/java (#1671) Bumps `grpc.version` from 1.45.0 to 1.45.1. Updates `grpc-core` from 1.45.0 to 1.45.1 - [Release notes](https://github.com/grpc/grpc-java/releases) - [Commits](https://github.com/grpc/grpc-java/compare/v1.45.0...v1.45.1) Updates `grpc-stub` from 1.45.0 to 1.45.1 - [Release notes](https://github.com/grpc/grpc-java/releases) - [Commits](https://github.com/grpc/grpc-java/compare/v1.45.0...v1.45.1) Updates `grpc-netty` from 1.45.0 to 1.45.1 - [Release notes](https://github.com/grpc/grpc-java/releases) - [Commits](https://github.com/grpc/grpc-java/compare/v1.45.0...v1.45.1) --- updated-dependencies: - dependency-name: io.grpc:grpc-core dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: io.grpc:grpc-stub dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: io.grpc:grpc-netty dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- lang/java/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lang/java/pom.xml b/lang/java/pom.xml index 19c59ff7384..1125d549a79 100644 --- a/lang/java/pom.xml +++ b/lang/java/pom.xml @@ -58,7 +58,7 @@ 1.9 4.3.1 2.2 - 1.45.0 + 1.45.1 1.5.2-2 3.2.1 From 7ab95ecc0489b1f32e236a434fa0efc80aa02aee Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Apr 2022 21:00:48 +0300 Subject: [PATCH 50/66] Bump underscore from 1.13.2 to 1.13.3 in /lang/js (#1669) Bumps [underscore](https://github.com/jashkenas/underscore) from 1.13.2 to 1.13.3. - [Release notes](https://github.com/jashkenas/underscore/releases) - [Commits](https://github.com/jashkenas/underscore/compare/1.13.2...1.13.3) --- updated-dependencies: - dependency-name: underscore dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- lang/js/package-lock.json | 547 ++++++++++++++++---------------------- 1 file changed, 233 insertions(+), 314 deletions(-) diff --git a/lang/js/package-lock.json b/lang/js/package-lock.json index 818c705164b..be07c87b4b3 100644 --- a/lang/js/package-lock.json +++ b/lang/js/package-lock.json @@ -8,6 +8,7 @@ "version": "2.1.2", "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.1.2.tgz", "integrity": "sha512-hoyByceqwKirw7w3Z7gnIIZC3Wx3J484Y3L/cMpXFbr7d9ZQj2mODrirNzcJa+SM3UlpWXYvKV4RlRpFXlWgXg==", + "dev": true, "requires": { "@jridgewell/trace-mapping": "^0.3.0" } @@ -16,6 +17,7 @@ "version": "7.16.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.16.7.tgz", "integrity": "sha512-iAXqUn8IIeBTNd72xsFlgaXHkMBMt6y4HJp1tIaK465CWLT/fG1aqB7ykr95gHHmlBdGbFeWWfyB4NJJ0nmeIg==", + "dev": true, "requires": { "@babel/highlight": "^7.16.7" } @@ -23,12 +25,14 @@ "@babel/compat-data": { "version": "7.17.7", "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.17.7.tgz", - "integrity": "sha512-p8pdE6j0a29TNGebNm7NzYZWB3xVZJBZ7XGs42uAKzQo8VQ3F0By/cQCtUEABwIqw5zo6WA4NbmxsfzADzMKnQ==" + "integrity": "sha512-p8pdE6j0a29TNGebNm7NzYZWB3xVZJBZ7XGs42uAKzQo8VQ3F0By/cQCtUEABwIqw5zo6WA4NbmxsfzADzMKnQ==", + "dev": true }, "@babel/core": { "version": "7.17.9", "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.17.9.tgz", "integrity": "sha512-5ug+SfZCpDAkVp9SFIZAzlW18rlzsOcJGaetCjkySnrXXDUw9AR8cDUm1iByTmdWM6yxX6/zycaV76w3YTF2gw==", + "dev": true, "requires": { "@ampproject/remapping": "^2.1.0", "@babel/code-frame": "^7.16.7", @@ -51,6 +55,7 @@ "version": "7.17.9", "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.17.9.tgz", "integrity": "sha512-rAdDousTwxbIxbz5I7GEQ3lUip+xVCXooZNbsydCWs3xA7ZsYOv+CFRdzGxRX78BmQHu9B1Eso59AOZQOJDEdQ==", + "dev": true, "requires": { "@babel/types": "^7.17.0", "jsesc": "^2.5.1", @@ -60,7 +65,8 @@ "source-map": { "version": "0.5.7", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", - "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=" + "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=", + "dev": true } } }, @@ -68,6 +74,7 @@ "version": "7.17.7", "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.17.7.tgz", "integrity": "sha512-UFzlz2jjd8kroj0hmCFV5zr+tQPi1dpC2cRsDV/3IEW8bJfCPrPpmcSN6ZS8RqIq4LXcmpipCQFPddyFA5Yc7w==", + "dev": true, "requires": { "@babel/compat-data": "^7.17.7", "@babel/helper-validator-option": "^7.16.7", @@ -79,6 +86,7 @@ "version": "7.16.7", "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.16.7.tgz", "integrity": "sha512-SLLb0AAn6PkUeAfKJCCOl9e1R53pQlGAfc4y4XuMRZfqeMYLE0dM1LMhqbGAlGQY0lfw5/ohoYWAe9V1yibRag==", + "dev": true, "requires": { "@babel/types": "^7.16.7" } @@ -87,6 +95,7 @@ "version": "7.17.9", "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.17.9.tgz", "integrity": "sha512-7cRisGlVtiVqZ0MW0/yFB4atgpGLWEHUVYnb448hZK4x+vih0YO5UoS11XIYtZYqHd0dIPMdUSv8q5K4LdMnIg==", + "dev": true, "requires": { "@babel/template": "^7.16.7", "@babel/types": "^7.17.0" @@ -96,6 +105,7 @@ "version": "7.16.7", "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.16.7.tgz", "integrity": "sha512-m04d/0Op34H5v7pbZw6pSKP7weA6lsMvfiIAMeIvkY/R4xQtBSMFEigu9QTZ2qB/9l22vsxtM8a+Q8CzD255fg==", + "dev": true, "requires": { "@babel/types": "^7.16.7" } @@ -104,6 +114,7 @@ "version": "7.16.7", "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.16.7.tgz", "integrity": "sha512-LVtS6TqjJHFc+nYeITRo6VLXve70xmq7wPhWTqDJusJEgGmkAACWwMiTNrvfoQo6hEhFwAIixNkvB0jPXDL8Wg==", + "dev": true, "requires": { "@babel/types": "^7.16.7" } @@ -112,6 +123,7 @@ "version": "7.17.7", "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.17.7.tgz", "integrity": "sha512-VmZD99F3gNTYB7fJRDTi+u6l/zxY0BE6OIxPSU7a50s6ZUQkHwSDmV92FfM+oCG0pZRVojGYhkR8I0OGeCVREw==", + "dev": true, "requires": { "@babel/helper-environment-visitor": "^7.16.7", "@babel/helper-module-imports": "^7.16.7", @@ -127,6 +139,7 @@ "version": "7.17.7", "resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.17.7.tgz", "integrity": "sha512-txyMCGroZ96i+Pxr3Je3lzEJjqwaRC9buMUgtomcrLe5Nd0+fk1h0LLA+ixUF5OW7AhHuQ7Es1WcQJZmZsz2XA==", + "dev": true, "requires": { "@babel/types": "^7.17.0" } @@ -135,6 +148,7 @@ "version": "7.16.7", "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.16.7.tgz", "integrity": "sha512-xbWoy/PFoxSWazIToT9Sif+jJTlrMcndIsaOKvTA6u7QEo7ilkRZpjew18/W3c7nm8fXdUDXh02VXTbZ0pGDNw==", + "dev": true, "requires": { "@babel/types": "^7.16.7" } @@ -142,17 +156,20 @@ "@babel/helper-validator-identifier": { "version": "7.16.7", "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.16.7.tgz", - "integrity": "sha512-hsEnFemeiW4D08A5gUAZxLBTXpZ39P+a+DGDsHw1yxqyQ/jzFEnxf5uTEGp+3bzAbNOxU1paTgYS4ECU/IgfDw==" + "integrity": "sha512-hsEnFemeiW4D08A5gUAZxLBTXpZ39P+a+DGDsHw1yxqyQ/jzFEnxf5uTEGp+3bzAbNOxU1paTgYS4ECU/IgfDw==", + "dev": true }, "@babel/helper-validator-option": { "version": "7.16.7", "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.16.7.tgz", - "integrity": "sha512-TRtenOuRUVo9oIQGPC5G9DgK4743cdxvtOw0weQNpZXaS16SCBi5MNjZF8vba3ETURjZpTbVn7Vvcf2eAwFozQ==" + "integrity": "sha512-TRtenOuRUVo9oIQGPC5G9DgK4743cdxvtOw0weQNpZXaS16SCBi5MNjZF8vba3ETURjZpTbVn7Vvcf2eAwFozQ==", + "dev": true }, "@babel/helpers": { "version": "7.17.9", "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.17.9.tgz", "integrity": "sha512-cPCt915ShDWUEzEp3+UNRktO2n6v49l5RSnG9M5pS24hA+2FAc5si+Pn1i4VVbQQ+jh+bIZhPFQOJOzbrOYY1Q==", + "dev": true, "requires": { "@babel/template": "^7.16.7", "@babel/traverse": "^7.17.9", @@ -163,6 +180,7 @@ "version": "7.17.9", "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.17.9.tgz", "integrity": "sha512-J9PfEKCbFIv2X5bjTMiZu6Vf341N05QIY+d6FvVKynkG1S7G0j3I0QoRtWIrXhZ+/Nlb5Q0MzqL7TokEJ5BNHg==", + "dev": true, "requires": { "@babel/helper-validator-identifier": "^7.16.7", "chalk": "^2.0.0", @@ -173,6 +191,7 @@ "version": "3.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, "requires": { "color-convert": "^1.9.0" } @@ -181,6 +200,7 @@ "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, "requires": { "ansi-styles": "^3.2.1", "escape-string-regexp": "^1.0.5", @@ -191,6 +211,7 @@ "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, "requires": { "color-name": "1.1.3" } @@ -198,22 +219,26 @@ "color-name": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "dev": true }, "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "dev": true }, "has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=" + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "dev": true }, "supports-color": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, "requires": { "has-flag": "^3.0.0" } @@ -223,12 +248,14 @@ "@babel/parser": { "version": "7.17.9", "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.17.9.tgz", - "integrity": "sha512-vqUSBLP8dQHFPdPi9bc5GK9vRkYHJ49fsZdtoJ8EQ8ibpwk5rPKfvNIwChB0KVXcIjcepEBBd2VHC5r9Gy8ueg==" + "integrity": "sha512-vqUSBLP8dQHFPdPi9bc5GK9vRkYHJ49fsZdtoJ8EQ8ibpwk5rPKfvNIwChB0KVXcIjcepEBBd2VHC5r9Gy8ueg==", + "dev": true }, "@babel/template": { "version": "7.16.7", "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.16.7.tgz", "integrity": "sha512-I8j/x8kHUrbYRTUxXrrMbfCa7jxkE7tZre39x3kjr9hvI82cK1FfqLygotcWN5kdPGWcLdWMHpSBavse5tWw3w==", + "dev": true, "requires": { "@babel/code-frame": "^7.16.7", "@babel/parser": "^7.16.7", @@ -239,6 +266,7 @@ "version": "7.17.9", "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.17.9.tgz", "integrity": "sha512-PQO8sDIJ8SIwipTPiR71kJQCKQYB5NGImbOviK8K+kg5xkNSYXLBupuX9QhatFowrsvo9Hj8WgArg3W7ijNAQw==", + "dev": true, "requires": { "@babel/code-frame": "^7.16.7", "@babel/generator": "^7.17.9", @@ -256,6 +284,7 @@ "version": "7.17.0", "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.17.0.tgz", "integrity": "sha512-TmKSNO4D5rzhL5bjWFcVHHLETzfQ/AmbKpKPOSjlP0WoHZ6L911fgoOKY4Alp/emzG4cHJdyN49zpgkbXFEHHw==", + "dev": true, "requires": { "@babel/helper-validator-identifier": "^7.16.7", "to-fast-properties": "^2.0.0" @@ -265,6 +294,7 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", "integrity": "sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==", + "dev": true, "requires": { "camelcase": "^5.3.1", "find-up": "^4.1.0", @@ -276,12 +306,14 @@ "camelcase": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", - "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==" + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "dev": true }, "find-up": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, "requires": { "locate-path": "^5.0.0", "path-exists": "^4.0.0" @@ -291,6 +323,7 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, "requires": { "p-locate": "^4.1.0" } @@ -299,6 +332,7 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, "requires": { "p-try": "^2.0.0" } @@ -307,6 +341,7 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, "requires": { "p-limit": "^2.2.0" } @@ -316,22 +351,26 @@ "@istanbuljs/schema": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", - "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==" + "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==", + "dev": true }, "@jridgewell/resolve-uri": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.0.5.tgz", - "integrity": "sha512-VPeQ7+wH0itvQxnG+lIzWgkysKIr3L9sslimFW55rHMdGu/qCQ5z5h9zq4gI8uBtqkpHhsF4Z/OwExufUCThew==" + "integrity": "sha512-VPeQ7+wH0itvQxnG+lIzWgkysKIr3L9sslimFW55rHMdGu/qCQ5z5h9zq4gI8uBtqkpHhsF4Z/OwExufUCThew==", + "dev": true }, "@jridgewell/sourcemap-codec": { "version": "1.4.11", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.11.tgz", - "integrity": "sha512-Fg32GrJo61m+VqYSdRSjRXMjQ06j8YIYfcTqndLYVAaHmroZHLJZCydsWBOTDqXS2v+mjxohBWEMfg97GXmYQg==" + "integrity": "sha512-Fg32GrJo61m+VqYSdRSjRXMjQ06j8YIYfcTqndLYVAaHmroZHLJZCydsWBOTDqXS2v+mjxohBWEMfg97GXmYQg==", + "dev": true }, "@jridgewell/trace-mapping": { "version": "0.3.4", "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.4.tgz", "integrity": "sha512-vFv9ttIedivx0ux3QSjhgtCVjPZd5l46ZOMDSCwnH1yUO2e964gO8LZGyv2QkqcgR6TnBU1v+1IFqmeoG+0UJQ==", + "dev": true, "requires": { "@jridgewell/resolve-uri": "^3.0.3", "@jridgewell/sourcemap-codec": "^1.4.10" @@ -343,16 +382,11 @@ "integrity": "sha512-sL/cEvJWAnClXw0wHk85/2L0G6Sj8UB0Ctc1TEMbKSsmpRosqhwj9gWgFRZSrBr2f9tiXISwNhCPmlfqUqyb9Q==", "dev": true }, - "abbrev": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.0.9.tgz", - "integrity": "sha1-kbR5JYinc4wl813W9jdSovh3YTU=", - "dev": true - }, "aggregate-error": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==", + "dev": true, "requires": { "clean-stack": "^2.0.0", "indent-string": "^4.0.0" @@ -370,13 +404,6 @@ "uri-js": "^4.2.2" } }, - "amdefine": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/amdefine/-/amdefine-1.0.1.tgz", - "integrity": "sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU=", - "dev": true, - "optional": true - }, "ansi-colors": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz", @@ -386,12 +413,14 @@ "ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true }, "ansi-styles": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, "requires": { "color-convert": "^2.0.1" } @@ -410,6 +439,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/append-transform/-/append-transform-2.0.0.tgz", "integrity": "sha512-7yeyCEurROLQJFv5Xj4lEGTy0borxepjFv1g22oAdqFu//SrAlDl1O1Nxx15SH1RoliUml6p8dwJW9jvZughhg==", + "dev": true, "requires": { "default-require-extensions": "^3.0.0" } @@ -417,12 +447,14 @@ "archy": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/archy/-/archy-1.0.0.tgz", - "integrity": "sha1-+cjBN1fMHde8N5rHeyxipcKGjEA=" + "integrity": "sha1-+cjBN1fMHde8N5rHeyxipcKGjEA=", + "dev": true }, "argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, "requires": { "sprintf-js": "~1.0.2" } @@ -442,12 +474,6 @@ "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=", "dev": true }, - "async": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", - "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", - "dev": true - }, "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -469,7 +495,8 @@ "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", + "dev": true }, "bcrypt-pbkdf": { "version": "1.0.2", @@ -490,6 +517,7 @@ "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -514,6 +542,7 @@ "version": "4.20.2", "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.20.2.tgz", "integrity": "sha512-CQOBCqp/9pDvDbx3xfMi+86pr4KXIf2FDkTTdeuYw8OxS9t898LA1Khq57gtufFILXpfgsSx5woNgsBgvGjpsA==", + "dev": true, "requires": { "caniuse-lite": "^1.0.30001317", "electron-to-chromium": "^1.4.84", @@ -526,6 +555,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/caching-transform/-/caching-transform-4.0.0.tgz", "integrity": "sha512-kpqOvwXnjjN44D89K5ccQC+RUrsy7jB/XLlRrx0D7/2HNcTPqzsb6XgYoErwko6QsV184CA2YgS1fxDiiDZMWA==", + "dev": true, "requires": { "hasha": "^5.0.0", "make-dir": "^3.0.0", @@ -542,7 +572,8 @@ "caniuse-lite": { "version": "1.0.30001331", "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001331.tgz", - "integrity": "sha512-Y1xk6paHpUXKP/P6YjQv1xqyTbgAP05ycHBcRdQjTcyXlWol868sJJPlmk5ylOekw2BrucWes5jk+LvVd7WZ5Q==" + "integrity": "sha512-Y1xk6paHpUXKP/P6YjQv1xqyTbgAP05ycHBcRdQjTcyXlWol868sJJPlmk5ylOekw2BrucWes5jk+LvVd7WZ5Q==", + "dev": true }, "caseless": { "version": "0.12.0", @@ -596,7 +627,8 @@ "clean-stack": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", - "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==" + "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==", + "dev": true }, "cli": { "version": "1.0.1", @@ -639,6 +671,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, "requires": { "color-name": "~1.1.4" } @@ -646,7 +679,8 @@ "color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true }, "combined-stream": { "version": "1.0.8", @@ -660,12 +694,14 @@ "commondir": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", - "integrity": "sha1-3dgA2gxmEnOTzKWVDqloo6rxJTs=" + "integrity": "sha1-3dgA2gxmEnOTzKWVDqloo6rxJTs=", + "dev": true }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", + "dev": true }, "console-browserify": { "version": "1.1.0", @@ -680,6 +716,7 @@ "version": "1.8.0", "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.8.0.tgz", "integrity": "sha512-+OQdjP49zViI/6i7nIJpA8rAl4sV/JdPfU9nZs3VqOwGIgizICvuN2ru6fMd+4llL0tar18UYJXfZ/TWtmhUjA==", + "dev": true, "requires": { "safe-buffer": "~5.1.1" }, @@ -687,7 +724,8 @@ "safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true } } }, @@ -714,6 +752,7 @@ "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dev": true, "requires": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", @@ -724,6 +763,7 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, "requires": { "isexe": "^2.0.0" } @@ -749,6 +789,7 @@ "version": "4.3.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz", "integrity": "sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==", + "dev": true, "requires": { "ms": "2.1.2" }, @@ -756,7 +797,8 @@ "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true } } }, @@ -766,16 +808,11 @@ "integrity": "sha512-9iE1PgSik9HeIIw2JO94IidnE3eBoQrFJ3w7sFuzSX4DpmZ3v5sZpUiV5Swcf6mQEF+Y0ru8Neo+p+nyh2J+hQ==", "dev": true }, - "deep-is": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", - "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=", - "dev": true - }, "default-require-extensions": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/default-require-extensions/-/default-require-extensions-3.0.0.tgz", "integrity": "sha512-ek6DpXq/SCpvjhpFsLFRVtIxJCRw6fUR42lYMVZuUMK7n8eMz4Uh5clckdBjEpLhn/gEBZo7hDJnJcwdKLKQjg==", + "dev": true, "requires": { "strip-bom": "^4.0.0" } @@ -854,12 +891,14 @@ "electron-to-chromium": { "version": "1.4.107", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.107.tgz", - "integrity": "sha512-Huen6taaVrUrSy8o7mGStByba8PfOWWluHNxSHGBrCgEdFVLtvdQDBr9LBCF9Uci8SYxh28QNNMO0oC17wbGAg==" + "integrity": "sha512-Huen6taaVrUrSy8o7mGStByba8PfOWWluHNxSHGBrCgEdFVLtvdQDBr9LBCF9Uci8SYxh28QNNMO0oC17wbGAg==", + "dev": true }, "emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true }, "entities": { "version": "1.0.0", @@ -870,12 +909,14 @@ "es6-error": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", - "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==" + "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==", + "dev": true }, "escalade": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", - "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "dev": true }, "escape-string-regexp": { "version": "4.0.0", @@ -883,37 +924,6 @@ "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", "dev": true }, - "escodegen": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.8.1.tgz", - "integrity": "sha1-WltTr0aTEQvrsIZ6o0MN07cKEBg=", - "dev": true, - "requires": { - "esprima": "^2.7.1", - "estraverse": "^1.9.1", - "esutils": "^2.0.2", - "optionator": "^0.8.1", - "source-map": "~0.2.0" - } - }, - "esprima": { - "version": "2.7.3", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-2.7.3.tgz", - "integrity": "sha1-luO3DVd59q1JzQMmc9HDEnZ7pYE=", - "dev": true - }, - "estraverse": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-1.9.3.tgz", - "integrity": "sha1-r2fy3JIlgkFZUJJgkaQAXSnJu0Q=", - "dev": true - }, - "esutils": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", - "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", - "dev": true - }, "exit": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz", @@ -944,12 +954,6 @@ "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", "dev": true }, - "fast-levenshtein": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", - "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", - "dev": true - }, "fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", @@ -963,6 +967,7 @@ "version": "3.3.2", "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-3.3.2.tgz", "integrity": "sha512-wXZV5emFEjrridIgED11OoUKLxiYjAcqot/NJdAkOhlJ+vGzwhOAfcG5OX1jP+S0PcjEn8bdMJv+g2jwQ3Onig==", + "dev": true, "requires": { "commondir": "^1.0.1", "make-dir": "^3.0.2", @@ -989,6 +994,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-2.0.0.tgz", "integrity": "sha512-dCIq9FpEcyQyXKCkyzmlPTFNgrCzPudOe+mhvJU5zAtlBnGVy2yKxtfsxK2tQBThwq225jcvBjpw1Gr40uzZCA==", + "dev": true, "requires": { "cross-spawn": "^7.0.0", "signal-exit": "^3.0.2" @@ -1014,12 +1020,14 @@ "fromentries": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/fromentries/-/fromentries-1.3.2.tgz", - "integrity": "sha512-cHEpEQHUg0f8XdtZCc2ZAhrHzKzT0MrFUTcvx+hfxYu7rGMDc5SKoXFh+n4YigxsHXRzc6OrCshdR1bWH6HHyg==" + "integrity": "sha512-cHEpEQHUg0f8XdtZCc2ZAhrHzKzT0MrFUTcvx+hfxYu7rGMDc5SKoXFh+n4YigxsHXRzc6OrCshdR1bWH6HHyg==", + "dev": true }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", + "dev": true }, "fsevents": { "version": "2.3.2", @@ -1031,17 +1039,20 @@ "gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", - "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==" + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true }, "get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true }, "get-package-type": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", - "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==" + "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", + "dev": true }, "getpass": { "version": "0.1.7", @@ -1052,19 +1063,6 @@ "assert-plus": "^1.0.0" } }, - "glob": { - "version": "5.0.15", - "resolved": "https://registry.npmjs.org/glob/-/glob-5.0.15.tgz", - "integrity": "sha1-G8k2ueAvSmA/zCIuz3Yz0wuLk7E=", - "dev": true, - "requires": { - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "2 || 3", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - } - }, "glob-parent": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", @@ -1077,12 +1075,14 @@ "globals": { "version": "11.12.0", "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", - "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==" + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", + "dev": true }, "graceful-fs": { "version": "4.2.10", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", - "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==" + "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==", + "dev": true }, "growl": { "version": "1.10.5", @@ -1090,27 +1090,6 @@ "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==", "dev": true }, - "handlebars": { - "version": "4.7.7", - "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.7.tgz", - "integrity": "sha512-aAcXm5OAfE/8IXkcZvCepKU3VzW1/39Fb5ZuqMtgI/hT8X2YgoMvBY5dLhq/cpOvw7Lk1nK/UF71aLG/ZnVYRA==", - "dev": true, - "requires": { - "minimist": "^1.2.5", - "neo-async": "^2.6.0", - "source-map": "^0.6.1", - "uglify-js": "^3.1.4", - "wordwrap": "^1.0.0" - }, - "dependencies": { - "source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true - } - } - }, "har-schema": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", @@ -1127,16 +1106,11 @@ "har-schema": "^2.0.0" } }, - "has-flag": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-1.0.0.tgz", - "integrity": "sha1-nZ55MWXOAXoA8AQYxD+UKnsdEfo=", - "dev": true - }, "hasha": { "version": "5.2.2", "resolved": "https://registry.npmjs.org/hasha/-/hasha-5.2.2.tgz", "integrity": "sha512-Hrp5vIK/xr5SkeN2onO32H0MgNZ0f17HRNH39WfL0SYUNOTZ5Lz1TJ8Pajo/87dYGEFlLMm7mIc/k/s6Bvz9HQ==", + "dev": true, "requires": { "is-stream": "^2.0.0", "type-fest": "^0.8.0" @@ -1151,7 +1125,8 @@ "html-escaper": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", - "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==" + "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", + "dev": true }, "htmlparser2": { "version": "3.8.3", @@ -1180,17 +1155,20 @@ "imurmurhash": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", - "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=" + "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=", + "dev": true }, "indent-string": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", - "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==" + "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", + "dev": true }, "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "dev": true, "requires": { "once": "^1.3.0", "wrappy": "1" @@ -1199,7 +1177,8 @@ "inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true }, "is-binary-path": { "version": "2.1.0", @@ -1219,7 +1198,8 @@ "is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true }, "is-glob": { "version": "4.0.3", @@ -1245,12 +1225,14 @@ "is-stream": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", - "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==" + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true }, "is-typedarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", - "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=", + "dev": true }, "is-unicode-supported": { "version": "0.1.0", @@ -1261,7 +1243,8 @@ "is-windows": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz", - "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==" + "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==", + "dev": true }, "isarray": { "version": "0.0.1", @@ -1272,7 +1255,8 @@ "isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=" + "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", + "dev": true }, "isstream": { "version": "0.1.2", @@ -1280,37 +1264,17 @@ "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=", "dev": true }, - "istanbul": { - "version": "0.4.5", - "resolved": "https://registry.npmjs.org/istanbul/-/istanbul-0.4.5.tgz", - "integrity": "sha1-ZcfXPUxNqE1POsMQuRj7C4Azczs=", - "dev": true, - "requires": { - "abbrev": "1.0.x", - "async": "1.x", - "escodegen": "1.8.x", - "esprima": "2.7.x", - "glob": "^5.0.15", - "handlebars": "^4.0.1", - "js-yaml": "3.x", - "mkdirp": "0.5.x", - "nopt": "3.x", - "once": "1.x", - "resolve": "1.1.x", - "supports-color": "^3.1.0", - "which": "^1.1.1", - "wordwrap": "^1.0.0" - } - }, "istanbul-lib-coverage": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.0.tgz", - "integrity": "sha512-eOeJ5BHCmHYvQK7xt9GkdHuzuCGS1Y6g9Gvnx3Ym33fz/HpLRYxiS0wHNr+m/MBC8B647Xt608vCDEvhl9c6Mw==" + "integrity": "sha512-eOeJ5BHCmHYvQK7xt9GkdHuzuCGS1Y6g9Gvnx3Ym33fz/HpLRYxiS0wHNr+m/MBC8B647Xt608vCDEvhl9c6Mw==", + "dev": true }, "istanbul-lib-hook": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/istanbul-lib-hook/-/istanbul-lib-hook-3.0.0.tgz", "integrity": "sha512-Pt/uge1Q9s+5VAZ+pCo16TYMWPBIl+oaNIjgLQxcX0itS6ueeaA+pEfThZpH8WxhFgCiEb8sAJY6MdUKgiIWaQ==", + "dev": true, "requires": { "append-transform": "^2.0.0" } @@ -1319,6 +1283,7 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-4.0.3.tgz", "integrity": "sha512-BXgQl9kf4WTCPCCpmFGoJkz/+uhvm7h7PFKUYxh7qarQd3ER33vHG//qaE8eN25l07YqZPpHXU9I09l/RD5aGQ==", + "dev": true, "requires": { "@babel/core": "^7.7.5", "@istanbuljs/schema": "^0.1.2", @@ -1330,6 +1295,7 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/istanbul-lib-processinfo/-/istanbul-lib-processinfo-2.0.2.tgz", "integrity": "sha512-kOwpa7z9hme+IBPZMzQ5vdQj8srYgAtaRqeI48NGmAQ+/5yKiHLV0QbYqQpxsdEF0+w14SoB8YbnHKcXE2KnYw==", + "dev": true, "requires": { "archy": "^1.0.0", "cross-spawn": "^7.0.0", @@ -1344,6 +1310,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.0.tgz", "integrity": "sha512-wcdi+uAKzfiGT2abPpKZ0hSU1rGQjUQnLvtY5MpQ7QCTahD3VODhcu4wcfY1YtkGaDD5yuydOLINXsfbus9ROw==", + "dev": true, "requires": { "istanbul-lib-coverage": "^3.0.0", "make-dir": "^3.0.0", @@ -1353,12 +1320,14 @@ "has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==" + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true }, "supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, "requires": { "has-flag": "^4.0.0" } @@ -1369,6 +1338,7 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-4.0.1.tgz", "integrity": "sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw==", + "dev": true, "requires": { "debug": "^4.1.1", "istanbul-lib-coverage": "^3.0.0", @@ -1378,7 +1348,8 @@ "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==" + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true } } }, @@ -1386,6 +1357,7 @@ "version": "3.1.4", "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.1.4.tgz", "integrity": "sha512-r1/DshN4KSE7xWEknZLLLLDn5CJybV3nw01VTkp6D5jzLuELlcbudfj/eSQFvrKsJuTVCGnePO7ho82Nw9zzfw==", + "dev": true, "requires": { "html-escaper": "^2.0.0", "istanbul-lib-report": "^3.0.0" @@ -1394,12 +1366,14 @@ "js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", - "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true }, "js-yaml": { "version": "3.14.1", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, "requires": { "argparse": "^1.0.7", "esprima": "^4.0.0" @@ -1408,7 +1382,8 @@ "esprima": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==" + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true } } }, @@ -1421,7 +1396,8 @@ "jsesc": { "version": "2.5.2", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", - "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==" + "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==", + "dev": true }, "jshint": { "version": "2.13.4", @@ -1459,7 +1435,8 @@ "json5": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.1.tgz", - "integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==" + "integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==", + "dev": true }, "jsprim": { "version": "1.4.2", @@ -1479,16 +1456,6 @@ "integrity": "sha1-6w1GtUER68VhrLTECO+TY73I9+A=", "dev": true }, - "levn": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", - "integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=", - "dev": true, - "requires": { - "prelude-ls": "~1.1.2", - "type-check": "~0.3.2" - } - }, "locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", @@ -1507,7 +1474,8 @@ "lodash.flattendeep": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/lodash.flattendeep/-/lodash.flattendeep-4.4.0.tgz", - "integrity": "sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI=" + "integrity": "sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI=", + "dev": true }, "log-driver": { "version": "1.2.7", @@ -1529,6 +1497,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", + "dev": true, "requires": { "semver": "^6.0.0" } @@ -1552,6 +1521,7 @@ "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "dev": true, "requires": { "brace-expansion": "^1.1.7" } @@ -1562,15 +1532,6 @@ "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==", "dev": true }, - "mkdirp": { - "version": "0.5.5", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", - "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", - "dev": true, - "requires": { - "minimist": "^1.2.5" - } - }, "mocha": { "version": "9.2.2", "resolved": "https://registry.npmjs.org/mocha/-/mocha-9.2.2.tgz", @@ -1696,16 +1657,11 @@ "integrity": "sha512-n6Vs/3KGyxPQd6uO0eH4Bv0ojGSUvuLlIHtC3Y0kEO23YRge8H9x1GCzLn28YX0H66pMkxuaeESFq4tKISKwdw==", "dev": true }, - "neo-async": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", - "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", - "dev": true - }, "node-preload": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/node-preload/-/node-preload-0.2.1.tgz", "integrity": "sha512-RM5oyBy45cLEoHqCeh+MNuFAxO0vTFBLskvQbOKnEE7YTTSN4tbN8QWDIPQ6L+WvKsB/qLEGpYe2ZZ9d4W9OIQ==", + "dev": true, "requires": { "process-on-spawn": "^1.0.0" } @@ -1713,16 +1669,8 @@ "node-releases": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.3.tgz", - "integrity": "sha512-maHFz6OLqYxz+VQyCAtA3PTX4UP/53pa05fyDNc9CwjvJ0yEh6+xBwKsgCxMNhS8taUKBFYxfuiaD9U/55iFaw==" - }, - "nopt": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz", - "integrity": "sha1-xkZdvwirzU2zWTF/eaxopkayj/k=", - "dev": true, - "requires": { - "abbrev": "1" - } + "integrity": "sha512-maHFz6OLqYxz+VQyCAtA3PTX4UP/53pa05fyDNc9CwjvJ0yEh6+xBwKsgCxMNhS8taUKBFYxfuiaD9U/55iFaw==", + "dev": true }, "normalize-path": { "version": "3.0.0", @@ -1734,6 +1682,7 @@ "version": "15.1.0", "resolved": "https://registry.npmjs.org/nyc/-/nyc-15.1.0.tgz", "integrity": "sha512-jMW04n9SxKdKi1ZMGhvUTHBN0EICCRkHemEoE5jm6mTYcqcdas0ATzgUgejlQUHMvpnOZqGB5Xxsv9KxJW1j8A==", + "dev": true, "requires": { "@istanbuljs/load-nyc-config": "^1.0.0", "@istanbuljs/schema": "^0.1.2", @@ -1767,12 +1716,14 @@ "camelcase": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", - "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==" + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "dev": true }, "cliui": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz", "integrity": "sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==", + "dev": true, "requires": { "string-width": "^4.2.0", "strip-ansi": "^6.0.0", @@ -1782,12 +1733,14 @@ "decamelize": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", - "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=" + "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", + "dev": true }, "find-up": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, "requires": { "locate-path": "^5.0.0", "path-exists": "^4.0.0" @@ -1797,6 +1750,7 @@ "version": "7.2.0", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", + "dev": true, "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -1810,6 +1764,7 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, "requires": { "p-locate": "^4.1.0" } @@ -1818,6 +1773,7 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, "requires": { "p-try": "^2.0.0" } @@ -1826,6 +1782,7 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, "requires": { "p-limit": "^2.2.0" } @@ -1834,6 +1791,7 @@ "version": "6.2.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", + "dev": true, "requires": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", @@ -1843,12 +1801,14 @@ "y18n": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", - "integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==" + "integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==", + "dev": true }, "yargs": { "version": "15.4.1", "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.4.1.tgz", "integrity": "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==", + "dev": true, "requires": { "cliui": "^6.0.0", "decamelize": "^1.2.0", @@ -1867,6 +1827,7 @@ "version": "18.1.3", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz", "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==", + "dev": true, "requires": { "camelcase": "^5.0.0", "decamelize": "^1.2.0" @@ -1884,22 +1845,9 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", - "requires": { - "wrappy": "1" - } - }, - "optionator": { - "version": "0.8.3", - "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz", - "integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==", "dev": true, "requires": { - "deep-is": "~0.1.3", - "fast-levenshtein": "~2.0.6", - "levn": "~0.3.0", - "prelude-ls": "~1.1.2", - "type-check": "~0.3.2", - "word-wrap": "~1.2.3" + "wrappy": "1" } }, "p-limit": { @@ -1924,6 +1872,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/p-map/-/p-map-3.0.0.tgz", "integrity": "sha512-d3qXVTF/s+W+CdJ5A29wywV2n8CQQYahlgz2bFiA+4eVNJbHJodPZ+/gXwPGh0bOqA+j8S+6+ckmvLGPk1QpxQ==", + "dev": true, "requires": { "aggregate-error": "^3.0.0" } @@ -1931,12 +1880,14 @@ "p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true }, "package-hash": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/package-hash/-/package-hash-4.0.0.tgz", "integrity": "sha512-whdkPIooSu/bASggZ96BWVvZTRMOFxnyUG5PnTSGKoJE2gd5mbVNmR2Nj20QFzxYYgAXpoqC+AiXzl+UMRh7zQ==", + "dev": true, "requires": { "graceful-fs": "^4.1.15", "hasha": "^5.0.0", @@ -1947,17 +1898,20 @@ "path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", - "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==" + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true }, "path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", + "dev": true }, "path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==" + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true }, "performance-now": { "version": "2.1.0", @@ -1968,7 +1922,8 @@ "picocolors": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", - "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==" + "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==", + "dev": true }, "picomatch": { "version": "2.3.1", @@ -1980,6 +1935,7 @@ "version": "4.2.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "dev": true, "requires": { "find-up": "^4.0.0" }, @@ -1988,6 +1944,7 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, "requires": { "locate-path": "^5.0.0", "path-exists": "^4.0.0" @@ -1997,6 +1954,7 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, "requires": { "p-locate": "^4.1.0" } @@ -2005,6 +1963,7 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, "requires": { "p-try": "^2.0.0" } @@ -2013,22 +1972,18 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, "requires": { "p-limit": "^2.2.0" } } } }, - "prelude-ls": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", - "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=", - "dev": true - }, "process-on-spawn": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/process-on-spawn/-/process-on-spawn-1.0.0.tgz", "integrity": "sha512-1WsPDsUSMmZH5LeMLegqkPDrsGgsWwk1Exipy2hvB0o/F0ASzbpIctSCcZIK1ykJvtTJULEH+20WOFjMvGnCTg==", + "dev": true, "requires": { "fromentries": "^1.2.0" } @@ -2085,6 +2040,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/release-zalgo/-/release-zalgo-1.0.0.tgz", "integrity": "sha1-CXALflB0Mpc5Mw5TXFqQ+2eFFzA=", + "dev": true, "requires": { "es6-error": "^4.0.1" } @@ -2120,28 +2076,26 @@ "require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=" + "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=", + "dev": true }, "require-main-filename": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", - "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==" - }, - "resolve": { - "version": "1.1.7", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.1.7.tgz", - "integrity": "sha1-IDEU2CrSxe2ejgQRs5ModeiJ6Xs=", + "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==", "dev": true }, "resolve-from": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", - "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==" + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", + "dev": true }, "rimraf": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, "requires": { "glob": "^7.1.3" }, @@ -2150,6 +2104,7 @@ "version": "7.2.0", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", + "dev": true, "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -2176,7 +2131,8 @@ "semver": { "version": "6.3.0", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true }, "serialize-javascript": { "version": "6.0.0", @@ -2190,12 +2146,14 @@ "set-blocking": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", - "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" + "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=", + "dev": true }, "shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, "requires": { "shebang-regex": "^3.0.0" } @@ -2203,27 +2161,20 @@ "shebang-regex": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==" + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true }, "signal-exit": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", - "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==" - }, - "source-map": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.2.0.tgz", - "integrity": "sha1-2rc/vPwrqBm03gO9b26qSBZLP50=", - "dev": true, - "optional": true, - "requires": { - "amdefine": ">=0.0.4" - } + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true }, "spawn-wrap": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/spawn-wrap/-/spawn-wrap-2.0.0.tgz", "integrity": "sha512-EeajNjfN9zMnULLwhZZQU3GWBoFNkbngTUPfaawT4RkMiviTxcX0qfhVbGey39mfctfDHkWtuecgQ8NJcyQWHg==", + "dev": true, "requires": { "foreground-child": "^2.0.0", "is-windows": "^1.0.2", @@ -2237,6 +2188,7 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, "requires": { "isexe": "^2.0.0" } @@ -2246,7 +2198,8 @@ "sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" + "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", + "dev": true }, "sshpk": { "version": "1.16.1", @@ -2269,6 +2222,7 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, "requires": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -2285,6 +2239,7 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, "requires": { "ansi-regex": "^5.0.1" } @@ -2292,7 +2247,8 @@ "strip-bom": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz", - "integrity": "sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==" + "integrity": "sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==", + "dev": true }, "strip-json-comments": { "version": "1.0.4", @@ -2300,19 +2256,11 @@ "integrity": "sha1-HhX7ysl9Pumb8tc7TGVrCCu6+5E=", "dev": true }, - "supports-color": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz", - "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=", - "dev": true, - "requires": { - "has-flag": "^1.0.0" - } - }, "test-exclude": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", "integrity": "sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w==", + "dev": true, "requires": { "@istanbuljs/schema": "^0.1.2", "glob": "^7.1.4", @@ -2323,6 +2271,7 @@ "version": "7.2.0", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", + "dev": true, "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -2346,7 +2295,8 @@ "to-fast-properties": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", - "integrity": "sha1-3F5pjL0HkmW8c+A3doGk5Og/YW4=" + "integrity": "sha1-3F5pjL0HkmW8c+A3doGk5Og/YW4=", + "dev": true }, "to-regex-range": { "version": "5.0.1", @@ -2382,39 +2332,25 @@ "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=", "dev": true }, - "type-check": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", - "integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=", - "dev": true, - "requires": { - "prelude-ls": "~1.1.2" - } - }, "type-fest": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz", - "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==" + "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==", + "dev": true }, "typedarray-to-buffer": { "version": "3.1.5", "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz", "integrity": "sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==", + "dev": true, "requires": { "is-typedarray": "^1.0.0" } }, - "uglify-js": { - "version": "3.13.0", - "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.13.0.tgz", - "integrity": "sha512-TWYSWa9T2pPN4DIJYbU9oAjQx+5qdV5RUDxwARg8fmJZrD/V27Zj0JngW5xg1DFz42G0uDYl2XhzF6alSzD62w==", - "dev": true, - "optional": true - }, "underscore": { - "version": "1.13.2", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.2.tgz", - "integrity": "sha512-ekY1NhRzq0B08g4bGuX4wd2jZx5GnKz6mKSqFL4nqBlfyMGiG10gDFhDTMEfYmDL6Jy0FUIZp7wiRB+0BP7J2g==" + "version": "1.13.3", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.3.tgz", + "integrity": "sha512-QvjkYpiD+dJJraRA8+dGAU4i7aBbb2s0S3jA45TFOvg2VgqvdCDd/3N6CqA8gluk1W91GLoXg5enMUx560QzuA==" }, "uri-js": { "version": "4.4.1", @@ -2428,7 +2364,8 @@ "uuid": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", + "dev": true }, "verror": { "version": "1.10.0", @@ -2441,30 +2378,10 @@ "extsprintf": "^1.2.0" } }, - "which": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", - "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", - "dev": true, - "requires": { - "isexe": "^2.0.0" - } - }, "which-module": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", - "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=" - }, - "word-wrap": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", - "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", - "dev": true - }, - "wordwrap": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", - "integrity": "sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus=", + "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=", "dev": true }, "workerpool": { @@ -2487,12 +2404,14 @@ "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", + "dev": true }, "write-file-atomic": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-3.0.3.tgz", "integrity": "sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q==", + "dev": true, "requires": { "imurmurhash": "^0.1.4", "is-typedarray": "^1.0.0", From 0aa6da262008c5f9c29d2991ce5a34258ca0baba Mon Sep 17 00:00:00 2001 From: Jack Klamer Date: Mon, 25 Apr 2022 13:05:37 -0500 Subject: [PATCH 51/66] AVRO-3484: Followup Check default json parsing at compile time for derive macro (#1668) * check json parsing at compile time * fmt --- lang/rust/avro_derive/src/lib.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lang/rust/avro_derive/src/lib.rs b/lang/rust/avro_derive/src/lib.rs index 42362537547..149f85eaad3 100644 --- a/lang/rust/avro_derive/src/lib.rs +++ b/lang/rust/avro_derive/src/lib.rs @@ -124,6 +124,13 @@ fn get_data_struct_schema_def( let doc = preserve_optional(field_attrs.doc); let default_value = match field_attrs.default { Some(default_value) => { + let _: serde_json::Value = serde_json::from_str(&default_value[..]) + .map_err(|e| { + vec![Error::new( + field.ident.span(), + format!("Invalid avro default json: \n{}", e), + )] + })?; quote! { Some(serde_json::from_str(#default_value).expect(format!("Invalid JSON: {:?}", #default_value).as_str())) } From 4731dee0c70eb080cb31faf8357df899e4dd9a5d Mon Sep 17 00:00:00 2001 From: kordos Date: Mon, 25 Apr 2022 20:36:51 +0200 Subject: [PATCH 52/66] AVRO-3427: skip creation of namespace directories for csharp schema (#1578) * Add new argument parameter --skip-directories. It will skip creation of directories for namespace. Just generate classes in output directory * Add missing doc param description * Fix Unit tests after merge with master * Fix Unit tests after merge with master * C# Add unit tests for --skip-directories option Co-authored-by: Pawel Kordowski --- lang/csharp/src/apache/codegen/AvroGen.cs | 14 ++++-- .../csharp/src/apache/main/CodeGen/CodeGen.cs | 11 +++-- .../src/apache/test/AvroGen/AvroGenHelper.cs | 11 +++-- .../apache/test/AvroGen/AvroGenSchemaTests.cs | 46 +++++++++++++++++-- .../apache/test/AvroGen/AvroGenToolTests.cs | 10 ++++ 5 files changed, 75 insertions(+), 17 deletions(-) diff --git a/lang/csharp/src/apache/codegen/AvroGen.cs b/lang/csharp/src/apache/codegen/AvroGen.cs index d1169027269..3b07ca59fdc 100644 --- a/lang/csharp/src/apache/codegen/AvroGen.cs +++ b/lang/csharp/src/apache/codegen/AvroGen.cs @@ -53,6 +53,7 @@ public static int Main(string[] args) bool? isProtocol = null; string inputFile = null; string outputDir = null; + bool skipDirectoriesCreation = false; var namespaceMapping = new Dictionary(); for (int i = 0; i < args.Length; ++i) { @@ -99,6 +100,10 @@ public static int Main(string[] args) namespaceMapping[parts[0]] = parts[1]; } + else if (args[i] == "--skip-directories") + { + skipDirectoriesCreation = true; + } else if (outputDir == null) { outputDir = args[i]; @@ -133,7 +138,7 @@ public static int Main(string[] args) else if (isProtocol.Value) rc = GenProtocol(inputFile, outputDir, namespaceMapping); else - rc = GenSchema(inputFile, outputDir, namespaceMapping); + rc = GenSchema(inputFile, outputDir, namespaceMapping, skipDirectoriesCreation); return rc; } @@ -149,7 +154,8 @@ static void Usage() " -V --version Show version.\n" + " --namespace Map an Avro schema/protocol namespace to a C# namespace.\n" + " The format is \"my.avro.namespace:my.csharp.namespace\".\n" + - " May be specified multiple times to map multiple namespaces.\n", + " May be specified multiple times to map multiple namespaces.\n" + + " --skip-directories Skip creation of namespace directories. It will generate classes right inside output directory\n", AppDomain.CurrentDomain.FriendlyName); } @@ -176,7 +182,7 @@ public static int GenProtocol(string infile, string outdir, } public static int GenSchema(string infile, string outdir, - IEnumerable> namespaceMapping) + IEnumerable> namespaceMapping, bool skipDirectories) { try { @@ -185,7 +191,7 @@ public static int GenSchema(string infile, string outdir, codegen.AddSchema(text, namespaceMapping); codegen.GenerateCode(); - codegen.WriteTypes(outdir); + codegen.WriteTypes(outdir, skipDirectories); } catch (Exception ex) { diff --git a/lang/csharp/src/apache/main/CodeGen/CodeGen.cs b/lang/csharp/src/apache/main/CodeGen/CodeGen.cs index 72776036e2d..e579d8bb07c 100644 --- a/lang/csharp/src/apache/main/CodeGen/CodeGen.cs +++ b/lang/csharp/src/apache/main/CodeGen/CodeGen.cs @@ -1140,7 +1140,8 @@ public virtual void WriteCompileUnit(string outputFile) /// Writes each types in each namespaces into individual files. /// /// name of directory to write to. - public virtual void WriteTypes(string outputdir) + /// skip creation of directories based on schema namespace + public virtual void WriteTypes(string outputdir, bool skipDirectories = false) { var cscp = new CSharpCodeProvider(); @@ -1155,11 +1156,13 @@ public virtual void WriteTypes(string outputdir) var ns = nsc[i]; string dir = outputdir; - foreach (string name in CodeGenUtil.Instance.UnMangle(ns.Name).Split('.')) + if (skipDirectories != true) { - dir = Path.Combine(dir, name); + foreach (string name in CodeGenUtil.Instance.UnMangle(ns.Name).Split('.')) + { + dir = Path.Combine(dir, name); + } } - Directory.CreateDirectory(dir); var new_ns = new CodeNamespace(ns.Name); diff --git a/lang/csharp/src/apache/test/AvroGen/AvroGenHelper.cs b/lang/csharp/src/apache/test/AvroGen/AvroGenHelper.cs index 49865c17052..04f88617a65 100644 --- a/lang/csharp/src/apache/test/AvroGen/AvroGenHelper.cs +++ b/lang/csharp/src/apache/test/AvroGen/AvroGenHelper.cs @@ -139,7 +139,7 @@ public static Assembly CompileCSharpFilesIntoLibrary(IEnumerable sourceF } } - public static string CreateEmptyTemporyFolder(out string uniqueId, string path = null) + public static string CreateEmptyTemporaryFolder(out string uniqueId, string path = null) { // Create unique id uniqueId = Guid.NewGuid().ToString(); @@ -234,10 +234,11 @@ public static Assembly TestSchema( string schema, IEnumerable typeNamesToCheck = null, IEnumerable> namespaceMapping = null, - IEnumerable generatedFilesToCheck = null) + IEnumerable generatedFilesToCheck = null, + bool skipDirectories = false) { // Create temp folder - string outputDir = CreateEmptyTemporyFolder(out string uniqueId); + string outputDir = CreateEmptyTemporaryFolder(out string uniqueId); try { @@ -246,7 +247,7 @@ public static Assembly TestSchema( System.IO.File.WriteAllText(schemaFileName, schema); // Generate from schema file - Assert.That(AvroGenTool.GenSchema(schemaFileName, outputDir, namespaceMapping ?? new Dictionary()), Is.EqualTo(0)); + Assert.That(AvroGenTool.GenSchema(schemaFileName, outputDir, namespaceMapping ?? new Dictionary(), skipDirectories), Is.EqualTo(0)); return CompileCSharpFilesAndCheckTypes(outputDir, uniqueId, typeNamesToCheck, generatedFilesToCheck); } @@ -263,7 +264,7 @@ public static Assembly TestProtocol( IEnumerable generatedFilesToCheck = null) { // Create temp folder - string outputDir = CreateEmptyTemporyFolder(out string uniqueId); + string outputDir = CreateEmptyTemporaryFolder(out string uniqueId); try { diff --git a/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs b/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs index 912284d6052..e31dc383bc4 100644 --- a/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs +++ b/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs @@ -316,7 +316,7 @@ private Assembly TestSchema( IEnumerable generatedFilesToCheck = null) { // Create temp folder - string outputDir = AvroGenHelper.CreateEmptyTemporyFolder(out string uniqueId); + string outputDir = AvroGenHelper.CreateEmptyTemporaryFolder(out string uniqueId); try { @@ -325,7 +325,7 @@ private Assembly TestSchema( System.IO.File.WriteAllText(schemaFileName, schema); // Generate from schema file - Assert.That(AvroGenTool.GenSchema(schemaFileName, outputDir, namespaceMapping ?? new Dictionary()), Is.EqualTo(0)); + Assert.That(AvroGenTool.GenSchema(schemaFileName, outputDir, namespaceMapping ?? new Dictionary(), false), Is.EqualTo(0)); // Check if all generated files exist if (generatedFilesToCheck != null) @@ -611,7 +611,7 @@ public void GenerateSchemaWithNamespaceMapping( public void NotSupportedSchema(string schema, Type expectedException) { // Create temp folder - string outputDir = AvroGenHelper.CreateEmptyTemporyFolder(out string uniqueId); + string outputDir = AvroGenHelper.CreateEmptyTemporaryFolder(out string uniqueId); try { @@ -619,7 +619,7 @@ public void NotSupportedSchema(string schema, Type expectedException) string schemaFileName = Path.Combine(outputDir, $"{uniqueId}.avsc"); System.IO.File.WriteAllText(schemaFileName, schema); - Assert.That(AvroGenTool.GenSchema(schemaFileName, outputDir, new Dictionary()), Is.EqualTo(1)); + Assert.That(AvroGenTool.GenSchema(schemaFileName, outputDir, new Dictionary(), false), Is.EqualTo(1)); } finally { @@ -771,5 +771,43 @@ public void GenerateSchemaCheckFields(string schema, object[] result) } } } + + [TestCase( + _nullableLogicalTypesArray, + new string[] + { + "org.apache.avro.codegentest.testdata.NullableLogicalTypesArray" + }, + new string[] + { + "NullableLogicalTypesArray.cs" + })] + [TestCase( + _nestedSomeNamespaceRecord, + new string[] + { + "org.apache.avro.codegentest.some.NestedSomeNamespaceRecord", + "org.apache.avro.codegentest.other.NestedOtherNamespaceRecord" + }, + new string[] + { + "NestedSomeNamespaceRecord.cs", + "NestedOtherNamespaceRecord.cs" + })] + [TestCase(_schema_avro_2883, + new string[] + { + "my.avro.ns.TestModel", + "my.avro.ns.EventType", + }, + new string[] + { + "TestModel.cs", + "EventType.cs" + })] + public void GenerateSchemaWithSkipDirectoriesOption(string schema, IEnumerable typeNamesToCheck, IEnumerable generatedFilesToCheck) + { + AvroGenHelper.TestSchema(schema, typeNamesToCheck, generatedFilesToCheck: generatedFilesToCheck, skipDirectories: true); + } } } diff --git a/lang/csharp/src/apache/test/AvroGen/AvroGenToolTests.cs b/lang/csharp/src/apache/test/AvroGen/AvroGenToolTests.cs index c1433bcc057..698ff468c2d 100644 --- a/lang/csharp/src/apache/test/AvroGen/AvroGenToolTests.cs +++ b/lang/csharp/src/apache/test/AvroGen/AvroGenToolTests.cs @@ -16,6 +16,7 @@ * limitations under the License. */ using System; +using System.Linq; using System.Reflection; using NUnit.Framework; @@ -89,5 +90,14 @@ public void CommandLineInvalidArgs(params string[] args) Assert.That(result.StdOut, Is.Not.Empty); Assert.That(result.StdErr, Is.Not.Empty); } + + [Theory] + public void CommandLineHelpContainsSkipDirectoriesParameter() + { + AvroGenToolResult result = AvroGenHelper.RunAvroGenTool("-h"); + + Assert.That(result.ExitCode, Is.EqualTo(0)); + Assert.IsTrue(result.StdOut.Any(s => s.Contains("--skip-directories"))); + } } } From b63eacb02bc5783963ec58cc085ad74bd9a7c5b0 Mon Sep 17 00:00:00 2001 From: rbalamohan Date: Wed, 27 Apr 2022 20:02:19 +0530 Subject: [PATCH 53/66] AVRO-3482: Reuse MAGIC in DataFileReader (#1639) DataFileReader reads magic information twice. seek(0) is invoked twice due to this. In cloud object stores, seeking back to 0 will cause it to fall back to "random IO policy". Example of this is S3A connector for s3. This causes suboptimal reads in object stores. Refactoring in the patch addresses this case by reusing MAGIC. --- .../org/apache/avro/file/DataFileReader.java | 20 +++++++++------- .../apache/avro/file/DataFileReader12.java | 1 + .../org/apache/avro/file/DataFileStream.java | 24 ++++++++++++++----- 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/file/DataFileReader.java b/lang/java/avro/src/main/java/org/apache/avro/file/DataFileReader.java index 7a235352e50..10067cd6c5c 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/file/DataFileReader.java +++ b/lang/java/avro/src/main/java/org/apache/avro/file/DataFileReader.java @@ -69,10 +69,9 @@ public static FileReader openReader(SeekableInput in, DatumReader read length -= bytesRead; offset += bytesRead; } - in.seek(0); if (Arrays.equals(MAGIC, magic)) // current format - return new DataFileReader<>(in, reader); + return new DataFileReader<>(in, reader, magic); if (Arrays.equals(DataFileReader12.MAGIC, magic)) // 1.2 format return new DataFileReader12<>(in, reader); @@ -111,7 +110,7 @@ public static DataFileReader openReader(SeekableInput in, DatumReader * 
    */
   public DataFileReader(File file, DatumReader reader) throws IOException {
-    this(new SeekableFileInput(file), reader, true);
+    this(new SeekableFileInput(file), reader, true, null);
   }
 
   /**
@@ -128,15 +127,20 @@ public DataFileReader(File file, DatumReader reader) throws IOException {
    * 
    */
   public DataFileReader(SeekableInput sin, DatumReader reader) throws IOException {
-    this(sin, reader, false);
+    this(sin, reader, false, null);
+  }
+
+  private DataFileReader(SeekableInput sin, DatumReader reader, byte[] magic) throws IOException {
+    this(sin, reader, false, magic);
   }
 
   /** Construct a reader for a file. Please close resource files yourself. */
-  protected DataFileReader(SeekableInput sin, DatumReader reader, boolean closeOnError) throws IOException {
+  protected DataFileReader(SeekableInput sin, DatumReader reader, boolean closeOnError, byte[] magic)
+      throws IOException {
     super(reader);
     try {
       this.sin = new SeekableInputStream(sin);
-      initialize(this.sin);
+      initialize(this.sin, magic);
       blockFinished();
     } catch (final Throwable e) {
       if (closeOnError) {
@@ -153,7 +157,7 @@ protected DataFileReader(SeekableInput sin, DatumReader reader, boolean close
   protected DataFileReader(SeekableInput sin, DatumReader reader, Header header) throws IOException {
     super(reader);
     this.sin = new SeekableInputStream(sin);
-    initialize(this.sin, header);
+    initialize(header);
   }
 
   /**
@@ -180,7 +184,7 @@ public void sync(final long position) throws IOException {
     seek(position);
     // work around an issue where 1.5.4 C stored sync in metadata
     if ((position == 0L) && (getMeta("avro.sync") != null)) {
-      initialize(sin); // re-init to skip header
+      initialize(sin, null); // re-init to skip header
       return;
     }
 
diff --git a/lang/java/avro/src/main/java/org/apache/avro/file/DataFileReader12.java b/lang/java/avro/src/main/java/org/apache/avro/file/DataFileReader12.java
index f24a978d6f8..c057a86db73 100644
--- a/lang/java/avro/src/main/java/org/apache/avro/file/DataFileReader12.java
+++ b/lang/java/avro/src/main/java/org/apache/avro/file/DataFileReader12.java
@@ -61,6 +61,7 @@ public DataFileReader12(SeekableInput sin, DatumReader reader) throws IOExcep
     this.in = new DataFileReader.SeekableInputStream(sin);
 
     byte[] magic = new byte[4];
+    in.seek(0); // seek to 0 to read magic header
     in.read(magic);
     if (!Arrays.equals(MAGIC, magic))
       throw new InvalidAvroMagicException("Not a data file.");
diff --git a/lang/java/avro/src/main/java/org/apache/avro/file/DataFileStream.java b/lang/java/avro/src/main/java/org/apache/avro/file/DataFileStream.java
index 8d2697104e2..a2b5172251d 100644
--- a/lang/java/avro/src/main/java/org/apache/avro/file/DataFileStream.java
+++ b/lang/java/avro/src/main/java/org/apache/avro/file/DataFileStream.java
@@ -87,7 +87,7 @@ private Header() {
    */
   public DataFileStream(InputStream in, DatumReader reader) throws IOException {
     this.reader = reader;
-    initialize(in);
+    initialize(in, null);
   }
 
   /**
@@ -97,18 +97,30 @@ protected DataFileStream(DatumReader reader) throws IOException {
     this.reader = reader;
   }
 
-  /** Initialize the stream by reading from its head. */
-  void initialize(InputStream in) throws IOException {
-    this.header = new Header();
-    this.vin = DecoderFactory.get().binaryDecoder(in, vin);
+  byte[] readMagic() throws IOException {
+    if (this.vin == null) {
+      throw new IOException("InputStream is not initialized");
+    }
     byte[] magic = new byte[DataFileConstants.MAGIC.length];
     try {
       vin.readFixed(magic); // read magic
     } catch (IOException e) {
       throw new IOException("Not an Avro data file.", e);
     }
+    return magic;
+  }
+
+  void validateMagic(byte[] magic) throws InvalidAvroMagicException {
     if (!Arrays.equals(DataFileConstants.MAGIC, magic))
       throw new InvalidAvroMagicException("Not an Avro data file.");
+  }
+
+  /** Initialize the stream by reading from its head. */
+  void initialize(InputStream in, byte[] magic) throws IOException {
+    this.header = new Header();
+    this.vin = DecoderFactory.get().binaryDecoder(in, vin);
+    magic = (magic == null) ? readMagic() : magic;
+    validateMagic(magic);
 
     long l = vin.readMapStart(); // read meta data
     if (l > 0) {
@@ -134,7 +146,7 @@ void initialize(InputStream in) throws IOException {
   }
 
   /** Initialize the stream without reading from it. */
-  void initialize(InputStream in, Header header) throws IOException {
+  void initialize(Header header) throws IOException {
     this.header = header;
     this.codec = resolveCodec();
     reader.setSchema(header.schema);

From 8f9dd6ffa75ff7308326c312cdb0c7fccfca780b Mon Sep 17 00:00:00 2001
From: Andrew Onyshchuk 
Date: Fri, 29 Apr 2022 17:54:36 -0700
Subject: [PATCH 54/66] AVRO-2870: Avoid throwing from destructor in
 DataFileWriterBase (#921)

Co-authored-by: Thiruvalluvan M G 
---
 lang/c++/impl/DataFile.cc | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/lang/c++/impl/DataFile.cc b/lang/c++/impl/DataFile.cc
index b0c574630fa..32f27ff2230 100644
--- a/lang/c++/impl/DataFile.cc
+++ b/lang/c++/impl/DataFile.cc
@@ -118,9 +118,12 @@ void DataFileWriterBase::init(const ValidSchema &schema, size_t syncInterval, co
     lastSync_ = stream_->byteCount();
 }
 
-DataFileWriterBase::~DataFileWriterBase() {
+DataFileWriterBase::~DataFileWriterBase()
+{
     if (stream_) {
-        close();
+        try {
+            close();
+        } catch(...) {}
     }
 }
 

From 773500139e4b73ae5a9c1cd3c53e192dffcfd722 Mon Sep 17 00:00:00 2001
From: Thiruvalluvan M G 
Date: Sun, 1 May 2022 21:33:54 -0700
Subject: [PATCH 55/66] Updated the checksum for PHP composer download (#1677)

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 8668fe638b0..22d94812df2 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -204,7 +204,7 @@ matrix:
             - libpq5
       install:
         - php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
-        - php -r "if (hash_file('sha384', 'composer-setup.php') === '906a84df04cea2aa72f40b5f787e49f22d4c2f19492ac310e8cba5b96ac8b64115ac402c8cd292b8a03482574915d1a8') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
+        - php -r "if (hash_file('sha384', 'composer-setup.php') === '55ce33d7678c5a611085589f1f3ddf8b3c52d662cd01d4ba75c0ee0459970c2200a51f492d557530c71c15d8dba01eae') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
         - php composer-setup.php --version=2.2.5
         - php -r "unlink('composer-setup.php');"
         - sudo mv composer.phar /usr/local/bin/composer

From 5f109d6d6f0ae9d730cc98798e450501a89c56bd Mon Sep 17 00:00:00 2001
From: Martin Tzvetanov Grigorov 
Date: Tue, 3 May 2022 12:18:56 +0300
Subject: [PATCH 56/66] Remove trailing ^M to make Git happy

Related to: https://github.com/apache/avro/commit/72e1135f7c1171b7f028f7f4f5fa0a652eb4edc2

Signed-off-by: Martin Tzvetanov Grigorov 
---
 lang/c/cmake_avrolib.bat | 96 ++++++++++++++++++++--------------------
 1 file changed, 48 insertions(+), 48 deletions(-)

diff --git a/lang/c/cmake_avrolib.bat b/lang/c/cmake_avrolib.bat
index 76934bca205..40e8b39e3d7 100644
--- a/lang/c/cmake_avrolib.bat
+++ b/lang/c/cmake_avrolib.bat
@@ -1,48 +1,48 @@
-REM  Licensed to the Apache Software Foundation (ASF) under one
-REM  or more contributor license agreements.  See the NOTICE file
-REM  distributed with this work for additional information
-REM  regarding copyright ownership.  The ASF licenses this file
-REM  to you under the Apache License, Version 2.0 (the
-REM  "License"); you may not use this file except in compliance
-REM  with the License.  You may obtain a copy of the License at
-REM 
-REM    https://www.apache.org/licenses/LICENSE-2.0
-REM 
-REM  Unless required by applicable law or agreed to in writing,
-REM  software distributed under the License is distributed on an
-REM  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-REM  KIND, either express or implied.  See the License for the
-REM  specific language governing permissions and limitations
-REM  under the License.
-
-echo off
-
-REM Set up the solution file in Windows. 
-
-set my_cmake_path="put_your_cmake_path_here"
-set cmake_path_win7="C:\Program Files (x86)\CMake 2.8\bin\cmake.exe"
-set cmake_path_xp="C:\Program Files\CMake 2.8\bin\cmake.exe"
-
-if exist %my_cmake_path% (
-   set cmake_path=%my_cmake_path%
-   goto RUN_CMAKE
-)
-
-if exist %cmake_path_win7% (
-   set cmake_path=%cmake_path_win7%
-   goto RUN_CMAKE
-)
-
-if exist %cmake_path_xp% (
-   set cmake_path=%cmake_path_xp%
-   goto RUN_CMAKE
-)
-
-echo "Set the proper cmake path in the variable 'my_cmake_path' in cmake_windows.bat, and re-run"
-goto EXIT_ERROR
-
-:RUN_CMAKE
-%cmake_path% -G"Visual Studio 9 2008" -H. -Bbuild_win32
-
-
-:EXIT_ERROR
+REM  Licensed to the Apache Software Foundation (ASF) under one
+REM  or more contributor license agreements.  See the NOTICE file
+REM  distributed with this work for additional information
+REM  regarding copyright ownership.  The ASF licenses this file
+REM  to you under the Apache License, Version 2.0 (the
+REM  "License"); you may not use this file except in compliance
+REM  with the License.  You may obtain a copy of the License at
+REM 
+REM    https://www.apache.org/licenses/LICENSE-2.0
+REM 
+REM  Unless required by applicable law or agreed to in writing,
+REM  software distributed under the License is distributed on an
+REM  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+REM  KIND, either express or implied.  See the License for the
+REM  specific language governing permissions and limitations
+REM  under the License.
+
+echo off
+
+REM Set up the solution file in Windows. 
+
+set my_cmake_path="put_your_cmake_path_here"
+set cmake_path_win7="C:\Program Files (x86)\CMake 2.8\bin\cmake.exe"
+set cmake_path_xp="C:\Program Files\CMake 2.8\bin\cmake.exe"
+
+if exist %my_cmake_path% (
+   set cmake_path=%my_cmake_path%
+   goto RUN_CMAKE
+)
+
+if exist %cmake_path_win7% (
+   set cmake_path=%cmake_path_win7%
+   goto RUN_CMAKE
+)
+
+if exist %cmake_path_xp% (
+   set cmake_path=%cmake_path_xp%
+   goto RUN_CMAKE
+)
+
+echo "Set the proper cmake path in the variable 'my_cmake_path' in cmake_windows.bat, and re-run"
+goto EXIT_ERROR
+
+:RUN_CMAKE
+%cmake_path% -G"Visual Studio 9 2008" -H. -Bbuild_win32
+
+
+:EXIT_ERROR

From 18ddaf54fc2f7c541349819623634e4ce6f25bf3 Mon Sep 17 00:00:00 2001
From: Jack Klamer 
Date: Thu, 21 Apr 2022 21:16:32 -0500
Subject: [PATCH 57/66] Encoer v1 with interop data

---
 .../apache/avro/message/TestInteropData.java  | 39 +++++++++++++++++++
 lang/rust/avro/src/ser.rs                     |  4 +-
 lang/rust/avro/src/writer.rs                  |  3 +-
 share/test/data/messageV1/README.md           |  2 +-
 share/test/data/messageV1/test_schema.json    | 22 +++++++++++
 5 files changed, 65 insertions(+), 5 deletions(-)
 create mode 100644 lang/java/avro/src/test/java/org/apache/avro/message/TestInteropData.java
 create mode 100644 share/test/data/messageV1/test_schema.json

diff --git a/lang/java/avro/src/test/java/org/apache/avro/message/TestInteropData.java b/lang/java/avro/src/test/java/org/apache/avro/message/TestInteropData.java
new file mode 100644
index 00000000000..c7489c1a4ec
--- /dev/null
+++ b/lang/java/avro/src/test/java/org/apache/avro/message/TestInteropData.java
@@ -0,0 +1,39 @@
+package org.apache.avro.message;
+
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericRecordBuilder;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+public class TestInteropData {
+  private static String inDir = System.getProperty("share.dir", "../../../share") + "/test/data/messageV1";
+  private static File SCHEMA_FILE = new File(inDir + "/test_schema.json");
+  private static File MESSAGE_FILE = new File(inDir + "/test_message.bin");
+  private static final Schema SCHEMA;
+  private static final GenericRecordBuilder BUILDER;
+
+  static {
+    try {
+      SCHEMA = new Schema.Parser().parse(new FileInputStream(SCHEMA_FILE));
+      BUILDER = new GenericRecordBuilder(SCHEMA);
+    } catch (IOException e) {
+      throw new RuntimeException("Interop Message Data Schema not found");
+    }
+  }
+
+  @Test
+  public void generateData() throws IOException {
+    MessageEncoder encoder = new BinaryMessageEncoder<>(GenericData.get(), SCHEMA);
+    BUILDER.set("id", 5L).set("name", "Bill").set("tags", Arrays.asList("dog_lover", "cat_hater")).build();
+    ByteBuffer buffer = encoder
+        .encode(BUILDER.set("id", 5L).set("name", "Bill").set("tags", Arrays.asList("dog_lover", "cat_hater")).build());
+    new FileOutputStream(MESSAGE_FILE).write(buffer.array());
+  }
+}
diff --git a/lang/rust/avro/src/ser.rs b/lang/rust/avro/src/ser.rs
index 5cff13e5666..9520ef69cbd 100644
--- a/lang/rust/avro/src/ser.rs
+++ b/lang/rust/avro/src/ser.rs
@@ -178,7 +178,7 @@ impl<'b> ser::Serializer for &'b mut Serializer {
     }
 
     fn serialize_none(self) -> Result {
-        Ok(Value::from(None::))
+        Ok(Value::Null)
     }
 
     fn serialize_some(self, value: &T) -> Result
@@ -186,7 +186,7 @@ impl<'b> ser::Serializer for &'b mut Serializer {
         T: Serialize,
     {
         let v = value.serialize(&mut Serializer::default())?;
-        Ok(Value::from(Some(v)))
+        Ok(v)
     }
 
     fn serialize_unit(self) -> Result {
diff --git a/lang/rust/avro/src/writer.rs b/lang/rust/avro/src/writer.rs
index c313e1667d5..1de63cab800 100644
--- a/lang/rust/avro/src/writer.rs
+++ b/lang/rust/avro/src/writer.rs
@@ -18,11 +18,10 @@
 //! Logic handling writing in Avro format at user level.
 use crate::{
     encode::{encode, encode_internal, encode_to_vec},
-    rabin::Rabin,
     schema::{AvroSchema, ResolvedOwnedSchema, ResolvedSchema, Schema},
     ser::Serializer,
     types::Value,
-    AvroResult, Codec, Error,
+    AvroResult, Codec, Error, rabin::Rabin,
 };
 use rand::random;
 use serde::Serialize;
diff --git a/share/test/data/messageV1/README.md b/share/test/data/messageV1/README.md
index e79f334c7e6..f0350288ce3 100644
--- a/share/test/data/messageV1/README.md
+++ b/share/test/data/messageV1/README.md
@@ -42,4 +42,4 @@ The sample binary message will have the values equal to the json serialized vers
 	"name": "Bill",
 	"tags": ["dog_lover", "cat_hater"]
 }
-```
\ No newline at end of file
+```
diff --git a/share/test/data/messageV1/test_schema.json b/share/test/data/messageV1/test_schema.json
new file mode 100644
index 00000000000..cd49b72a456
--- /dev/null
+++ b/share/test/data/messageV1/test_schema.json
@@ -0,0 +1,22 @@
+{
+	"type":"record",
+	"namespace":"org.apache.avro",
+	"name":"TestMessage",
+	"fields":[
+		{
+			"name":"id",
+			"type":"long"
+		},
+		{
+			"name":"name",
+			"type":"string"
+		},
+		{
+			"name":"tags",
+			"type":{
+				"type":"array",
+				"items":"string"
+			}
+		}
+	]
+}
\ No newline at end of file

From 6545a4be2ce4d4ab1ff26fe9383d0caa04d923b9 Mon Sep 17 00:00:00 2001
From: Jack Klamer 
Date: Sat, 23 Apr 2022 16:11:37 -0500
Subject: [PATCH 58/66] unit tested

---
 lang/rust/avro/src/writer.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lang/rust/avro/src/writer.rs b/lang/rust/avro/src/writer.rs
index 1de63cab800..5d2c3b086e2 100644
--- a/lang/rust/avro/src/writer.rs
+++ b/lang/rust/avro/src/writer.rs
@@ -450,7 +450,7 @@ where
         self.inner.write_value_ref(&v, writer)
     }
 
-    /// Wrtite the Serialize object to the provided Write object. Returns a result with the number of bytes writtern including the header
+    /// Write the Serialize object to the provided Write object. Returns a result with the number of bytes writtern including the header
     pub fn write(&mut self, data: T, writer: &mut W) -> AvroResult {
         self.write_ref(&data, writer)
     }

From 91f75be06def00a18603d4b09adca44d821a4361 Mon Sep 17 00:00:00 2001
From: Jack Klamer 
Date: Sat, 23 Apr 2022 16:20:09 -0500
Subject: [PATCH 59/66] fmt

---
 lang/rust/avro/src/writer.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lang/rust/avro/src/writer.rs b/lang/rust/avro/src/writer.rs
index 5d2c3b086e2..a15fd6f2f07 100644
--- a/lang/rust/avro/src/writer.rs
+++ b/lang/rust/avro/src/writer.rs
@@ -18,10 +18,11 @@
 //! Logic handling writing in Avro format at user level.
 use crate::{
     encode::{encode, encode_internal, encode_to_vec},
+    rabin::Rabin,
     schema::{AvroSchema, ResolvedOwnedSchema, ResolvedSchema, Schema},
     ser::Serializer,
     types::Value,
-    AvroResult, Codec, Error, rabin::Rabin,
+    AvroResult, Codec, Error,
 };
 use rand::random;
 use serde::Serialize;

From adc8add6fd625aea82d7bbe26345660efb396d14 Mon Sep 17 00:00:00 2001
From: Martin Tzvetanov Grigorov 
Date: Wed, 4 May 2022 08:16:06 +0300
Subject: [PATCH 60/66] AVRO-3506: Cleanup and minor improvements

Signed-off-by: Martin Tzvetanov Grigorov 
---
 ...a => TestInteropSingleObjectEncoding.java} | 27 +++++++++----------
 ...=> test_interop_single_object_encoding.rs} |  8 +++---
 lang/rust/avro/src/error.rs                   |  4 ++-
 lang/rust/avro/src/writer.rs                  | 15 ++++++-----
 lang/rust/build.sh                            |  6 ++---
 5 files changed, 32 insertions(+), 28 deletions(-)
 rename lang/java/avro/src/test/java/org/apache/avro/message/{TestInteropMessageData.java => TestInteropSingleObjectEncoding.java} (69%)
 rename lang/rust/avro/examples/{test_interop_message_data.rs => test_interop_single_object_encoding.rs} (85%)

diff --git a/lang/java/avro/src/test/java/org/apache/avro/message/TestInteropMessageData.java b/lang/java/avro/src/test/java/org/apache/avro/message/TestInteropSingleObjectEncoding.java
similarity index 69%
rename from lang/java/avro/src/test/java/org/apache/avro/message/TestInteropMessageData.java
rename to lang/java/avro/src/test/java/org/apache/avro/message/TestInteropSingleObjectEncoding.java
index 0a8335e7eeb..8bc42cbb70e 100644
--- a/lang/java/avro/src/test/java/org/apache/avro/message/TestInteropMessageData.java
+++ b/lang/java/avro/src/test/java/org/apache/avro/message/TestInteropSingleObjectEncoding.java
@@ -22,39 +22,38 @@
 import org.apache.avro.Schema;
 import org.apache.avro.generic.GenericData;
 import org.apache.avro.generic.GenericRecordBuilder;
-import org.junit.AfterClass;
 import org.junit.Assert;
-import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
 import java.io.File;
 import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.FileReader;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.file.Files;
 import java.util.Arrays;
-import java.util.logging.Logger;
 
-public class TestInteropMessageData {
-  private static final String inDir = System.getProperty("share.dir", "../../../share") + "/test/data/messageV1";
-  private static final File SCHEMA_FILE = new File(inDir + "/test_schema.avsc");
-  private static final File MESSAGE_FILE = new File(inDir + "/test_message.bin");
+/**
+ * Tests that test_message.bin is properly encoded
+ * single object
+ */
+public class TestInteropSingleObjectEncoding {
+  private static final String RESOURCES_FOLDER = System.getProperty("share.dir", "../../../share") + "/test/data/messageV1";
+  private static final File SCHEMA_FILE = new File(RESOURCES_FOLDER + "/test_schema.avsc");
+  private static final File MESSAGE_FILE = new File(RESOURCES_FOLDER + "/test_message.bin");
   private static Schema SCHEMA;
   private static GenericRecordBuilder BUILDER;
 
   @BeforeClass
   public static void setup() throws IOException {
-    final FileInputStream fileInputStream = new FileInputStream(SCHEMA_FILE);
-    SCHEMA = new Schema.Parser().parse(fileInputStream);
-    BUILDER = new GenericRecordBuilder(SCHEMA);
-    fileInputStream.close();
+    try (FileInputStream fileInputStream = new FileInputStream(SCHEMA_FILE)) {
+      SCHEMA = new Schema.Parser().parse(fileInputStream);
+      BUILDER = new GenericRecordBuilder(SCHEMA);
+    }
   }
 
   @Test
-  public void sanity_check() throws IOException {
+  public void checkSingleObjectEncoding() throws IOException {
     MessageEncoder encoder = new BinaryMessageEncoder<>(GenericData.get(), SCHEMA);
     ByteBuffer buffer = encoder.encode(
         BUILDER.set("id", 42L).set("name", "Bill").set("tags", Arrays.asList("dog_lover", "cat_hater")).build());
diff --git a/lang/rust/avro/examples/test_interop_message_data.rs b/lang/rust/avro/examples/test_interop_single_object_encoding.rs
similarity index 85%
rename from lang/rust/avro/examples/test_interop_message_data.rs
rename to lang/rust/avro/examples/test_interop_single_object_encoding.rs
index 0a72b25e47c..d5e3edafdee 100644
--- a/lang/rust/avro/examples/test_interop_message_data.rs
+++ b/lang/rust/avro/examples/test_interop_single_object_encoding.rs
@@ -17,11 +17,13 @@
 
 use apache_avro::{schema::AvroSchema, types::Value};
 
+const RESOURCES_FOLDER: &str = "../../share/test/data/messageV1";
+
 struct InteropMessage;
 
 impl AvroSchema for InteropMessage {
     fn get_schema() -> apache_avro::Schema {
-        let schema = std::fs::read_to_string("../../share/test/data/messageV1/test_schema.avsc")
+        let schema = std::fs::read_to_string(format!("{}/test_schema.avsc", RESOURCES_FOLDER))
             .expect("File should exist with schema inside");
         apache_avro::Schema::parse_str(schema.as_str())
             .expect("File should exist with schema inside")
@@ -47,8 +49,8 @@ impl From for Value {
 }
 
 fn main() {
-    let file_message = std::fs::read("../../share/test/data/messageV1/test_message.bin")
-        .expect("File not found or error reading");
+    let file_message = std::fs::read(format!("{}/test_message.bin", RESOURCES_FOLDER))
+        .expect("File with single object not found or error occurred while reading");
     let mut generated_encoding: Vec = Vec::new();
     apache_avro::SingleObjectWriter::::with_capacity(1024)
         .expect("resolve expected")
diff --git a/lang/rust/avro/src/error.rs b/lang/rust/avro/src/error.rs
index c5fe46f6b9a..670c6c4ec77 100644
--- a/lang/rust/avro/src/error.rs
+++ b/lang/rust/avro/src/error.rs
@@ -411,7 +411,9 @@ pub enum Error {
         value_kind: ValueKind,
         supported_schema: Vec,
     },
-    #[error("Internal buffer not drained properly. Reinitialize the writer struct")]
+    #[error(
+        "Internal buffer not drained properly. Re-initialize the single object writer struct!"
+    )]
     IllegalSingleObjectWriterState,
 }
 
diff --git a/lang/rust/avro/src/writer.rs b/lang/rust/avro/src/writer.rs
index a15fd6f2f07..849e91d974f 100644
--- a/lang/rust/avro/src/writer.rs
+++ b/lang/rust/avro/src/writer.rs
@@ -433,7 +433,8 @@ impl SingleObjectWriter
 where
     T: AvroSchema + Into,
 {
-    /// Write the Into to the provided Write object. Returns a result with the number of bytes written including the header
+    /// Write the Into to the provided Write object. Returns a result with the number
+    /// of bytes written including the header
     pub fn write_value(&mut self, data: T, writer: &mut W) -> AvroResult {
         let v: Value = data.into();
         self.inner.write_value_ref(&v, writer)
@@ -444,14 +445,16 @@ impl SingleObjectWriter
 where
     T: AvroSchema + Serialize,
 {
-    /// Write the referenced Serialize object to the provided Write object. Returns a result with the number of bytes written including the header
+    /// Write the referenced Serialize object to the provided Write object. Returns a result with
+    /// the number of bytes written including the header
     pub fn write_ref(&mut self, data: &T, writer: &mut W) -> AvroResult {
         let mut serializer = Serializer::default();
         let v = data.serialize(&mut serializer)?;
         self.inner.write_value_ref(&v, writer)
     }
 
-    /// Write the Serialize object to the provided Write object. Returns a result with the number of bytes writtern including the header
+    /// Write the Serialize object to the provided Write object. Returns a result with the number
+    /// of bytes written including the header
     pub fn write(&mut self, data: T, writer: &mut W) -> AvroResult {
         self.write_ref(&data, writer)
     }
@@ -1174,12 +1177,12 @@ mod tests {
             1024,
         )
         .expect("Should resolve schema");
-        let mut specifc_writer = SingleObjectWriter::::with_capacity(1024)
+        let mut specific_writer = SingleObjectWriter::::with_capacity(1024)
             .expect("Resolved should pass");
-        specifc_writer
+        specific_writer
             .write(obj1.clone(), &mut buf1)
             .expect("Serialization expected");
-        specifc_writer
+        specific_writer
             .write_value(obj1.clone(), &mut buf2)
             .expect("Serialization expected");
         generic_writer
diff --git a/lang/rust/build.sh b/lang/rust/build.sh
index a1429b7d97f..c44c05ad29e 100755
--- a/lang/rust/build.sh
+++ b/lang/rust/build.sh
@@ -17,7 +17,6 @@
 
 set -e  # exit on error
 
-root_dir=$(pwd)
 build_dir="../../build/rust"
 dist_dir="../../dist/rust"
 
@@ -35,7 +34,7 @@ function prepare_build {
   mkdir -p $build_dir
 }
 
-cd `dirname "$0"`
+cd $(dirname "$0")
 
 for target in "$@"
 do
@@ -60,13 +59,12 @@ do
       export RUST_LOG=apache_avro=debug
       export RUST_BACKTRACE=1
       cargo run --all-features --example generate_interop_data
-      cargo run --all-features --example generate_interop_data
       ;;
 
     interop-data-test)
       prepare_build
       cargo run --all-features --example test_interop_data
-      cargo run --all-features --example test_interop_message_data
+      cargo run --all-features --example test_interop_single_object_encoding
       ;;
     *)
       echo "Usage: $0 {lint|test|dist|clean|interop-data-generate|interop-data-test}" >&2

From 01600eccbef6f0491d09d9bdb46b297bc243f227 Mon Sep 17 00:00:00 2001
From: Martin Tzvetanov Grigorov 
Date: Wed, 4 May 2022 09:02:47 +0300
Subject: [PATCH 61/66] AVRO-3506: Cleanup

Give a better name to TestGenerateInteropSingleObjectEncoding
Remove useless lifetime in schema.rs
Remove .json files for the single object encoded test file

Signed-off-by: Martin Tzvetanov Grigorov 
---
 ...tGenerateInteropSingleObjectEncoding.java} | 28 +++++++++++--------
 lang/rust/avro/src/schema.rs                  |  2 +-
 lang/rust/avro/src/ser.rs                     |  4 +--
 share/test/data/messageV1/test_message.json   |  5 ----
 share/test/data/messageV1/test_schema.json    | 22 ---------------
 5 files changed, 20 insertions(+), 41 deletions(-)
 rename lang/java/avro/src/test/java/org/apache/avro/message/{TestInteropData.java => TestGenerateInteropSingleObjectEncoding.java} (51%)
 delete mode 100644 share/test/data/messageV1/test_message.json
 delete mode 100644 share/test/data/messageV1/test_schema.json

diff --git a/lang/java/avro/src/test/java/org/apache/avro/message/TestInteropData.java b/lang/java/avro/src/test/java/org/apache/avro/message/TestGenerateInteropSingleObjectEncoding.java
similarity index 51%
rename from lang/java/avro/src/test/java/org/apache/avro/message/TestInteropData.java
rename to lang/java/avro/src/test/java/org/apache/avro/message/TestGenerateInteropSingleObjectEncoding.java
index c7489c1a4ec..482416b1bf2 100644
--- a/lang/java/avro/src/test/java/org/apache/avro/message/TestInteropData.java
+++ b/lang/java/avro/src/test/java/org/apache/avro/message/TestGenerateInteropSingleObjectEncoding.java
@@ -3,6 +3,7 @@
 import org.apache.avro.Schema;
 import org.apache.avro.generic.GenericData;
 import org.apache.avro.generic.GenericRecordBuilder;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import java.io.File;
@@ -12,19 +13,24 @@
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 
-public class TestInteropData {
-  private static String inDir = System.getProperty("share.dir", "../../../share") + "/test/data/messageV1";
-  private static File SCHEMA_FILE = new File(inDir + "/test_schema.json");
-  private static File MESSAGE_FILE = new File(inDir + "/test_message.bin");
-  private static final Schema SCHEMA;
-  private static final GenericRecordBuilder BUILDER;
+/**
+ * Generates test_message.bin - a
+ * single object encoded
+ * Avro message.
+ */
+public class TestGenerateInteropSingleObjectEncoding {
+  private static final String RESOURCES_FOLDER = System.getProperty("share.dir", "../../../share") + "/test/data/messageV1";
+  private static final File SCHEMA_FILE = new File(RESOURCES_FOLDER + "/test_schema.json");
+  private static final File MESSAGE_FILE = new File(RESOURCES_FOLDER + "/test_message.bin");
+  private static Schema SCHEMA;
+  private static GenericRecordBuilder BUILDER;
 
-  static {
-    try {
-      SCHEMA = new Schema.Parser().parse(new FileInputStream(SCHEMA_FILE));
+
+  @BeforeClass
+  public static void setup() throws IOException {
+    try (FileInputStream fileInputStream = new FileInputStream(SCHEMA_FILE)) {
+      SCHEMA = new Schema.Parser().parse(fileInputStream);
       BUILDER = new GenericRecordBuilder(SCHEMA);
-    } catch (IOException e) {
-      throw new RuntimeException("Interop Message Data Schema not found");
     }
   }
 
diff --git a/lang/rust/avro/src/schema.rs b/lang/rust/avro/src/schema.rs
index 1b618bcbc85..52478890b4b 100644
--- a/lang/rust/avro/src/schema.rs
+++ b/lang/rust/avro/src/schema.rs
@@ -439,7 +439,7 @@ pub(crate) struct ResolvedOwnedSchema {
     root_schema: Schema,
 }
 
-impl<'s> TryFrom for ResolvedOwnedSchema {
+impl TryFrom for ResolvedOwnedSchema {
     type Error = Error;
 
     fn try_from(schema: Schema) -> AvroResult {
diff --git a/lang/rust/avro/src/ser.rs b/lang/rust/avro/src/ser.rs
index 9520ef69cbd..5cff13e5666 100644
--- a/lang/rust/avro/src/ser.rs
+++ b/lang/rust/avro/src/ser.rs
@@ -178,7 +178,7 @@ impl<'b> ser::Serializer for &'b mut Serializer {
     }
 
     fn serialize_none(self) -> Result {
-        Ok(Value::Null)
+        Ok(Value::from(None::))
     }
 
     fn serialize_some(self, value: &T) -> Result
@@ -186,7 +186,7 @@ impl<'b> ser::Serializer for &'b mut Serializer {
         T: Serialize,
     {
         let v = value.serialize(&mut Serializer::default())?;
-        Ok(v)
+        Ok(Value::from(Some(v)))
     }
 
     fn serialize_unit(self) -> Result {
diff --git a/share/test/data/messageV1/test_message.json b/share/test/data/messageV1/test_message.json
deleted file mode 100644
index e0c2b0d81ed..00000000000
--- a/share/test/data/messageV1/test_message.json
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-	"id": 42,
-	"name": "Bill", 
-	"tags": ["dog_lover", "cat_hater"]
-}
\ No newline at end of file
diff --git a/share/test/data/messageV1/test_schema.json b/share/test/data/messageV1/test_schema.json
deleted file mode 100644
index cd49b72a456..00000000000
--- a/share/test/data/messageV1/test_schema.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-	"type":"record",
-	"namespace":"org.apache.avro",
-	"name":"TestMessage",
-	"fields":[
-		{
-			"name":"id",
-			"type":"long"
-		},
-		{
-			"name":"name",
-			"type":"string"
-		},
-		{
-			"name":"tags",
-			"type":{
-				"type":"array",
-				"items":"string"
-			}
-		}
-	]
-}
\ No newline at end of file

From 8640187c043905a71f8d5a8c877b6b8c8c43bd56 Mon Sep 17 00:00:00 2001
From: Martin Tzvetanov Grigorov 
Date: Wed, 4 May 2022 09:17:29 +0300
Subject: [PATCH 62/66] AVRO-3506: Add licence header to
 TestGenerateInteropSingleObjectEncoding

Signed-off-by: Martin Tzvetanov Grigorov 
---
 ...stGenerateInteropSingleObjectEncoding.java | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/lang/java/avro/src/test/java/org/apache/avro/message/TestGenerateInteropSingleObjectEncoding.java b/lang/java/avro/src/test/java/org/apache/avro/message/TestGenerateInteropSingleObjectEncoding.java
index 482416b1bf2..8da781b052e 100644
--- a/lang/java/avro/src/test/java/org/apache/avro/message/TestGenerateInteropSingleObjectEncoding.java
+++ b/lang/java/avro/src/test/java/org/apache/avro/message/TestGenerateInteropSingleObjectEncoding.java
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 package org.apache.avro.message;
 
 import org.apache.avro.Schema;

From 77de905a415b4caa2ad8790327ad556c785d1223 Mon Sep 17 00:00:00 2001
From: Martin Tzvetanov Grigorov 
Date: Wed, 4 May 2022 09:18:40 +0300
Subject: [PATCH 63/66] AVRO-3506: Fix spotless issues in the new Java test
 classes

Signed-off-by: Martin Tzvetanov Grigorov 
---
 .../TestGenerateInteropSingleObjectEncoding.java       | 10 +++++-----
 .../avro/message/TestInteropSingleObjectEncoding.java  |  8 +++++---
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/lang/java/avro/src/test/java/org/apache/avro/message/TestGenerateInteropSingleObjectEncoding.java b/lang/java/avro/src/test/java/org/apache/avro/message/TestGenerateInteropSingleObjectEncoding.java
index 8da781b052e..67ec331e8d1 100644
--- a/lang/java/avro/src/test/java/org/apache/avro/message/TestGenerateInteropSingleObjectEncoding.java
+++ b/lang/java/avro/src/test/java/org/apache/avro/message/TestGenerateInteropSingleObjectEncoding.java
@@ -33,18 +33,18 @@
 import java.util.Arrays;
 
 /**
- * Generates test_message.bin - a
- * single object encoded
- * Avro message.
+ * Generates test_message.bin - a single
+ * object encoded Avro message.
  */
 public class TestGenerateInteropSingleObjectEncoding {
-  private static final String RESOURCES_FOLDER = System.getProperty("share.dir", "../../../share") + "/test/data/messageV1";
+  private static final String RESOURCES_FOLDER = System.getProperty("share.dir", "../../../share")
+      + "/test/data/messageV1";
   private static final File SCHEMA_FILE = new File(RESOURCES_FOLDER + "/test_schema.json");
   private static final File MESSAGE_FILE = new File(RESOURCES_FOLDER + "/test_message.bin");
   private static Schema SCHEMA;
   private static GenericRecordBuilder BUILDER;
 
-
   @BeforeClass
   public static void setup() throws IOException {
     try (FileInputStream fileInputStream = new FileInputStream(SCHEMA_FILE)) {
diff --git a/lang/java/avro/src/test/java/org/apache/avro/message/TestInteropSingleObjectEncoding.java b/lang/java/avro/src/test/java/org/apache/avro/message/TestInteropSingleObjectEncoding.java
index 8bc42cbb70e..df16817f515 100644
--- a/lang/java/avro/src/test/java/org/apache/avro/message/TestInteropSingleObjectEncoding.java
+++ b/lang/java/avro/src/test/java/org/apache/avro/message/TestInteropSingleObjectEncoding.java
@@ -34,11 +34,13 @@
 import java.util.Arrays;
 
 /**
- * Tests that test_message.bin is properly encoded
- * single object
+ * Tests that test_message.bin is properly encoded single
+ * object
  */
 public class TestInteropSingleObjectEncoding {
-  private static final String RESOURCES_FOLDER = System.getProperty("share.dir", "../../../share") + "/test/data/messageV1";
+  private static final String RESOURCES_FOLDER = System.getProperty("share.dir", "../../../share")
+      + "/test/data/messageV1";
   private static final File SCHEMA_FILE = new File(RESOURCES_FOLDER + "/test_schema.avsc");
   private static final File MESSAGE_FILE = new File(RESOURCES_FOLDER + "/test_message.bin");
   private static Schema SCHEMA;

From 651f7691e377dfc8d2ee59217a74eee975b07ecf Mon Sep 17 00:00:00 2001
From: Martin Tzvetanov Grigorov 
Date: Wed, 4 May 2022 09:55:29 +0300
Subject: [PATCH 64/66] AVRO-3506: Fix the path to the schema file

Signed-off-by: Martin Tzvetanov Grigorov 
---
 .../TestGenerateInteropSingleObjectEncoding.java  |   2 +-
 share/test/data/messageV1/test_message.bin        | Bin 38 -> 38 bytes
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/lang/java/avro/src/test/java/org/apache/avro/message/TestGenerateInteropSingleObjectEncoding.java b/lang/java/avro/src/test/java/org/apache/avro/message/TestGenerateInteropSingleObjectEncoding.java
index 67ec331e8d1..a71be9c7d30 100644
--- a/lang/java/avro/src/test/java/org/apache/avro/message/TestGenerateInteropSingleObjectEncoding.java
+++ b/lang/java/avro/src/test/java/org/apache/avro/message/TestGenerateInteropSingleObjectEncoding.java
@@ -40,7 +40,7 @@
 public class TestGenerateInteropSingleObjectEncoding {
   private static final String RESOURCES_FOLDER = System.getProperty("share.dir", "../../../share")
       + "/test/data/messageV1";
-  private static final File SCHEMA_FILE = new File(RESOURCES_FOLDER + "/test_schema.json");
+  private static final File SCHEMA_FILE = new File(RESOURCES_FOLDER + "/test_schema.avsc");
   private static final File MESSAGE_FILE = new File(RESOURCES_FOLDER + "/test_message.bin");
   private static Schema SCHEMA;
   private static GenericRecordBuilder BUILDER;
diff --git a/share/test/data/messageV1/test_message.bin b/share/test/data/messageV1/test_message.bin
index 609337eb91464ddf773f25c182a092b72e0ab08e..025d46151de381840b61ddc0679cccda9d98b9ef 100644
GIT binary patch
delta 17
ZcmY#W<37x|QupDHMNVIBxh8T;0{}aT2BH7}

delta 17
ZcmY#W<37x|QupDHMNVIBLnd-d0{}d+2J8R;


From 47d0f828e6a6df0f60fb3f0ddcbf9eb52bf925b4 Mon Sep 17 00:00:00 2001
From: Martin Tzvetanov Grigorov 
Date: Wed, 4 May 2022 10:02:33 +0300
Subject: [PATCH 65/66] AVRO-3506: Fix the id to match the expected value

Signed-off-by: Martin Tzvetanov Grigorov 
---
 .../TestGenerateInteropSingleObjectEncoding.java  |   4 ++--
 share/test/data/messageV1/test_message.bin        | Bin 38 -> 38 bytes
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lang/java/avro/src/test/java/org/apache/avro/message/TestGenerateInteropSingleObjectEncoding.java b/lang/java/avro/src/test/java/org/apache/avro/message/TestGenerateInteropSingleObjectEncoding.java
index a71be9c7d30..f3f65625ddf 100644
--- a/lang/java/avro/src/test/java/org/apache/avro/message/TestGenerateInteropSingleObjectEncoding.java
+++ b/lang/java/avro/src/test/java/org/apache/avro/message/TestGenerateInteropSingleObjectEncoding.java
@@ -56,9 +56,9 @@ public static void setup() throws IOException {
   @Test
   public void generateData() throws IOException {
     MessageEncoder encoder = new BinaryMessageEncoder<>(GenericData.get(), SCHEMA);
-    BUILDER.set("id", 5L).set("name", "Bill").set("tags", Arrays.asList("dog_lover", "cat_hater")).build();
+    BUILDER.set("id", 42L).set("name", "Bill").set("tags", Arrays.asList("dog_lover", "cat_hater")).build();
     ByteBuffer buffer = encoder
-        .encode(BUILDER.set("id", 5L).set("name", "Bill").set("tags", Arrays.asList("dog_lover", "cat_hater")).build());
+        .encode(BUILDER.set("id", 42L).set("name", "Bill").set("tags", Arrays.asList("dog_lover", "cat_hater")).build());
     new FileOutputStream(MESSAGE_FILE).write(buffer.array());
   }
 }
diff --git a/share/test/data/messageV1/test_message.bin b/share/test/data/messageV1/test_message.bin
index 025d46151de381840b61ddc0679cccda9d98b9ef..609337eb91464ddf773f25c182a092b72e0ab08e 100644
GIT binary patch
delta 17
ZcmY#W<37x|QupDHMNVIBLnd-d0{}d+2J8R;

delta 17
ZcmY#W<37x|QupDHMNVIBxh8T;0{}aT2BH7}


From a31a1955210769d6ace4e000b2a64f3233878623 Mon Sep 17 00:00:00 2001
From: Martin Tzvetanov Grigorov 
Date: Wed, 4 May 2022 10:19:47 +0300
Subject: [PATCH 66/66] AVRO-3506: Fix spotless again

Signed-off-by: Martin Tzvetanov Grigorov 
---
 .../avro/message/TestGenerateInteropSingleObjectEncoding.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lang/java/avro/src/test/java/org/apache/avro/message/TestGenerateInteropSingleObjectEncoding.java b/lang/java/avro/src/test/java/org/apache/avro/message/TestGenerateInteropSingleObjectEncoding.java
index f3f65625ddf..f41c69f8a01 100644
--- a/lang/java/avro/src/test/java/org/apache/avro/message/TestGenerateInteropSingleObjectEncoding.java
+++ b/lang/java/avro/src/test/java/org/apache/avro/message/TestGenerateInteropSingleObjectEncoding.java
@@ -57,8 +57,8 @@ public static void setup() throws IOException {
   public void generateData() throws IOException {
     MessageEncoder encoder = new BinaryMessageEncoder<>(GenericData.get(), SCHEMA);
     BUILDER.set("id", 42L).set("name", "Bill").set("tags", Arrays.asList("dog_lover", "cat_hater")).build();
-    ByteBuffer buffer = encoder
-        .encode(BUILDER.set("id", 42L).set("name", "Bill").set("tags", Arrays.asList("dog_lover", "cat_hater")).build());
+    ByteBuffer buffer = encoder.encode(
+        BUILDER.set("id", 42L).set("name", "Bill").set("tags", Arrays.asList("dog_lover", "cat_hater")).build());
     new FileOutputStream(MESSAGE_FILE).write(buffer.array());
   }
 }